Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/dev/e1000/if_igb.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /****************************************************************************** 2 3 Copyright (c) 2001-2015, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD: releng/10.3/sys/dev/e1000/if_igb.c 296073 2016-02-26 00:09:50Z marius $*/ 34 35 36 #include "opt_inet.h" 37 #include "opt_inet6.h" 38 39 #ifdef HAVE_KERNEL_OPTION_HEADERS 40 #include "opt_device_polling.h" 41 #include "opt_altq.h" 42 #endif 43 44 #include "if_igb.h" 45 46 /********************************************************************* 47 * Driver version: 48 *********************************************************************/ 49 char igb_driver_version[] = "2.5.3-k"; 50 51 52 /********************************************************************* 53 * PCI Device ID Table 54 * 55 * Used by probe to select devices to load on 56 * Last field stores an index into e1000_strings 57 * Last entry must be all 0s 58 * 59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 60 *********************************************************************/ 61 62 static igb_vendor_info_t igb_vendor_info_array[] = 63 { 64 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0}, 65 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0}, 66 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0}, 67 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0}, 68 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0}, 69 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0}, 70 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0}, 71 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0}, 72 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0}, 73 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0}, 74 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0}, 75 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0}, 76 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0}, 77 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0}, 78 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0}, 79 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0}, 80 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0}, 81 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0}, 82 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0}, 83 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0}, 84 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0}, 85 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0}, 86 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0}, 87 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0}, 88 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0}, 89 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0}, 90 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0}, 91 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0}, 92 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0}, 93 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0}, 94 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0}, 95 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0}, 96 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0}, 97 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0}, 98 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0}, 99 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0}, 100 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0}, 101 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0}, 102 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0}, 103 /* required last entry */ 104 {0, 0, 0, 0, 0} 105 }; 106 107 /********************************************************************* 108 * Table of branding strings for all supported NICs. 109 *********************************************************************/ 110 111 static char *igb_strings[] = { 112 "Intel(R) PRO/1000 Network Connection" 113 }; 114 115 /********************************************************************* 116 * Function prototypes 117 *********************************************************************/ 118 static int igb_probe(device_t); 119 static int igb_attach(device_t); 120 static int igb_detach(device_t); 121 static int igb_shutdown(device_t); 122 static int igb_suspend(device_t); 123 static int igb_resume(device_t); 124 #ifndef IGB_LEGACY_TX 125 static int igb_mq_start(struct ifnet *, struct mbuf *); 126 static int igb_mq_start_locked(struct ifnet *, struct tx_ring *); 127 static void igb_qflush(struct ifnet *); 128 static void igb_deferred_mq_start(void *, int); 129 #else 130 static void igb_start(struct ifnet *); 131 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); 132 #endif 133 static int igb_ioctl(struct ifnet *, u_long, caddr_t); 134 static void igb_init(void *); 135 static void igb_init_locked(struct adapter *); 136 static void igb_stop(void *); 137 static void igb_media_status(struct ifnet *, struct ifmediareq *); 138 static int igb_media_change(struct ifnet *); 139 static void igb_identify_hardware(struct adapter *); 140 static int igb_allocate_pci_resources(struct adapter *); 141 static int igb_allocate_msix(struct adapter *); 142 static int igb_allocate_legacy(struct adapter *); 143 static int igb_setup_msix(struct adapter *); 144 static void igb_free_pci_resources(struct adapter *); 145 static void igb_local_timer(void *); 146 static void igb_reset(struct adapter *); 147 static int igb_setup_interface(device_t, struct adapter *); 148 static int igb_allocate_queues(struct adapter *); 149 static void igb_configure_queues(struct adapter *); 150 151 static int igb_allocate_transmit_buffers(struct tx_ring *); 152 static void igb_setup_transmit_structures(struct adapter *); 153 static void igb_setup_transmit_ring(struct tx_ring *); 154 static void igb_initialize_transmit_units(struct adapter *); 155 static void igb_free_transmit_structures(struct adapter *); 156 static void igb_free_transmit_buffers(struct tx_ring *); 157 158 static int igb_allocate_receive_buffers(struct rx_ring *); 159 static int igb_setup_receive_structures(struct adapter *); 160 static int igb_setup_receive_ring(struct rx_ring *); 161 static void igb_initialize_receive_units(struct adapter *); 162 static void igb_free_receive_structures(struct adapter *); 163 static void igb_free_receive_buffers(struct rx_ring *); 164 static void igb_free_receive_ring(struct rx_ring *); 165 166 static void igb_enable_intr(struct adapter *); 167 static void igb_disable_intr(struct adapter *); 168 static void igb_update_stats_counters(struct adapter *); 169 static bool igb_txeof(struct tx_ring *); 170 171 static __inline void igb_rx_discard(struct rx_ring *, int); 172 static __inline void igb_rx_input(struct rx_ring *, 173 struct ifnet *, struct mbuf *, u32); 174 175 static bool igb_rxeof(struct igb_queue *, int, int *); 176 static void igb_rx_checksum(u32, struct mbuf *, u32); 177 static int igb_tx_ctx_setup(struct tx_ring *, 178 struct mbuf *, u32 *, u32 *); 179 static int igb_tso_setup(struct tx_ring *, 180 struct mbuf *, u32 *, u32 *); 181 static void igb_set_promisc(struct adapter *); 182 static void igb_disable_promisc(struct adapter *); 183 static void igb_set_multi(struct adapter *); 184 static void igb_update_link_status(struct adapter *); 185 static void igb_refresh_mbufs(struct rx_ring *, int); 186 187 static void igb_register_vlan(void *, struct ifnet *, u16); 188 static void igb_unregister_vlan(void *, struct ifnet *, u16); 189 static void igb_setup_vlan_hw_support(struct adapter *); 190 191 static int igb_xmit(struct tx_ring *, struct mbuf **); 192 static int igb_dma_malloc(struct adapter *, bus_size_t, 193 struct igb_dma_alloc *, int); 194 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); 195 static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 196 static void igb_print_nvm_info(struct adapter *); 197 static int igb_is_valid_ether_addr(u8 *); 198 static void igb_add_hw_stats(struct adapter *); 199 200 static void igb_vf_init_stats(struct adapter *); 201 static void igb_update_vf_stats_counters(struct adapter *); 202 203 /* Management and WOL Support */ 204 static void igb_init_manageability(struct adapter *); 205 static void igb_release_manageability(struct adapter *); 206 static void igb_get_hw_control(struct adapter *); 207 static void igb_release_hw_control(struct adapter *); 208 static void igb_enable_wakeup(device_t); 209 static void igb_led_func(void *, int); 210 211 static int igb_irq_fast(void *); 212 static void igb_msix_que(void *); 213 static void igb_msix_link(void *); 214 static void igb_handle_que(void *context, int pending); 215 static void igb_handle_link(void *context, int pending); 216 static void igb_handle_link_locked(struct adapter *); 217 218 static void igb_set_sysctl_value(struct adapter *, const char *, 219 const char *, int *, int); 220 static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS); 221 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); 222 static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS); 223 224 #ifdef DEVICE_POLLING 225 static poll_handler_t igb_poll; 226 #endif /* POLLING */ 227 228 /********************************************************************* 229 * FreeBSD Device Interface Entry Points 230 *********************************************************************/ 231 232 static device_method_t igb_methods[] = { 233 /* Device interface */ 234 DEVMETHOD(device_probe, igb_probe), 235 DEVMETHOD(device_attach, igb_attach), 236 DEVMETHOD(device_detach, igb_detach), 237 DEVMETHOD(device_shutdown, igb_shutdown), 238 DEVMETHOD(device_suspend, igb_suspend), 239 DEVMETHOD(device_resume, igb_resume), 240 DEVMETHOD_END 241 }; 242 243 static driver_t igb_driver = { 244 "igb", igb_methods, sizeof(struct adapter), 245 }; 246 247 static devclass_t igb_devclass; 248 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); 249 MODULE_DEPEND(igb, pci, 1, 1, 1); 250 MODULE_DEPEND(igb, ether, 1, 1, 1); 251 252 /********************************************************************* 253 * Tunable default values. 254 *********************************************************************/ 255 256 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters"); 257 258 /* Descriptor defaults */ 259 static int igb_rxd = IGB_DEFAULT_RXD; 260 static int igb_txd = IGB_DEFAULT_TXD; 261 TUNABLE_INT("hw.igb.rxd", &igb_rxd); 262 TUNABLE_INT("hw.igb.txd", &igb_txd); 263 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0, 264 "Number of receive descriptors per queue"); 265 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0, 266 "Number of transmit descriptors per queue"); 267 268 /* 269 ** AIM: Adaptive Interrupt Moderation 270 ** which means that the interrupt rate 271 ** is varied over time based on the 272 ** traffic for that interrupt vector 273 */ 274 static int igb_enable_aim = TRUE; 275 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); 276 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0, 277 "Enable adaptive interrupt moderation"); 278 279 /* 280 * MSIX should be the default for best performance, 281 * but this allows it to be forced off for testing. 282 */ 283 static int igb_enable_msix = 1; 284 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix); 285 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0, 286 "Enable MSI-X interrupts"); 287 288 /* 289 ** Tuneable Interrupt rate 290 */ 291 static int igb_max_interrupt_rate = 8000; 292 TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate); 293 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 294 &igb_max_interrupt_rate, 0, "Maximum interrupts per second"); 295 296 #ifndef IGB_LEGACY_TX 297 /* 298 ** Tuneable number of buffers in the buf-ring (drbr_xxx) 299 */ 300 static int igb_buf_ring_size = IGB_BR_SIZE; 301 TUNABLE_INT("hw.igb.buf_ring_size", &igb_buf_ring_size); 302 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN, 303 &igb_buf_ring_size, 0, "Size of the bufring"); 304 #endif 305 306 /* 307 ** Header split causes the packet header to 308 ** be dma'd to a seperate mbuf from the payload. 309 ** this can have memory alignment benefits. But 310 ** another plus is that small packets often fit 311 ** into the header and thus use no cluster. Its 312 ** a very workload dependent type feature. 313 */ 314 static int igb_header_split = FALSE; 315 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split); 316 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0, 317 "Enable receive mbuf header split"); 318 319 /* 320 ** This will autoconfigure based on the 321 ** number of CPUs and max supported 322 ** MSIX messages if left at 0. 323 */ 324 static int igb_num_queues = 0; 325 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues); 326 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0, 327 "Number of queues to configure, 0 indicates autoconfigure"); 328 329 /* 330 ** Global variable to store last used CPU when binding queues 331 ** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a 332 ** queue is bound to a cpu. 333 */ 334 static int igb_last_bind_cpu = -1; 335 336 /* How many packets rxeof tries to clean at a time */ 337 static int igb_rx_process_limit = 100; 338 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); 339 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 340 &igb_rx_process_limit, 0, 341 "Maximum number of received packets to process at a time, -1 means unlimited"); 342 343 /* How many packets txeof tries to clean at a time */ 344 static int igb_tx_process_limit = -1; 345 TUNABLE_INT("hw.igb.tx_process_limit", &igb_tx_process_limit); 346 SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 347 &igb_tx_process_limit, 0, 348 "Maximum number of sent packets to process at a time, -1 means unlimited"); 349 350 #ifdef DEV_NETMAP /* see ixgbe.c for details */ 351 #include <dev/netmap/if_igb_netmap.h> 352 #endif /* DEV_NETMAP */ 353 /********************************************************************* 354 * Device identification routine 355 * 356 * igb_probe determines if the driver should be loaded on 357 * adapter based on PCI vendor/device id of the adapter. 358 * 359 * return BUS_PROBE_DEFAULT on success, positive on failure 360 *********************************************************************/ 361 362 static int 363 igb_probe(device_t dev) 364 { 365 char adapter_name[256]; 366 uint16_t pci_vendor_id = 0; 367 uint16_t pci_device_id = 0; 368 uint16_t pci_subvendor_id = 0; 369 uint16_t pci_subdevice_id = 0; 370 igb_vendor_info_t *ent; 371 372 INIT_DEBUGOUT("igb_probe: begin"); 373 374 pci_vendor_id = pci_get_vendor(dev); 375 if (pci_vendor_id != IGB_INTEL_VENDOR_ID) 376 return (ENXIO); 377 378 pci_device_id = pci_get_device(dev); 379 pci_subvendor_id = pci_get_subvendor(dev); 380 pci_subdevice_id = pci_get_subdevice(dev); 381 382 ent = igb_vendor_info_array; 383 while (ent->vendor_id != 0) { 384 if ((pci_vendor_id == ent->vendor_id) && 385 (pci_device_id == ent->device_id) && 386 387 ((pci_subvendor_id == ent->subvendor_id) || 388 (ent->subvendor_id == 0)) && 389 390 ((pci_subdevice_id == ent->subdevice_id) || 391 (ent->subdevice_id == 0))) { 392 sprintf(adapter_name, "%s, Version - %s", 393 igb_strings[ent->index], 394 igb_driver_version); 395 device_set_desc_copy(dev, adapter_name); 396 return (BUS_PROBE_DEFAULT); 397 } 398 ent++; 399 } 400 return (ENXIO); 401 } 402 403 /********************************************************************* 404 * Device initialization routine 405 * 406 * The attach entry point is called when the driver is being loaded. 407 * This routine identifies the type of hardware, allocates all resources 408 * and initializes the hardware. 409 * 410 * return 0 on success, positive on failure 411 *********************************************************************/ 412 413 static int 414 igb_attach(device_t dev) 415 { 416 struct adapter *adapter; 417 int error = 0; 418 u16 eeprom_data; 419 420 INIT_DEBUGOUT("igb_attach: begin"); 421 422 if (resource_disabled("igb", device_get_unit(dev))) { 423 device_printf(dev, "Disabled by device hint\n"); 424 return (ENXIO); 425 } 426 427 adapter = device_get_softc(dev); 428 adapter->dev = adapter->osdep.dev = dev; 429 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 430 431 /* SYSCTLs */ 432 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 433 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 434 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 435 igb_sysctl_nvm_info, "I", "NVM Information"); 436 437 igb_set_sysctl_value(adapter, "enable_aim", 438 "Interrupt Moderation", &adapter->enable_aim, 439 igb_enable_aim); 440 441 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 442 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 443 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, 444 adapter, 0, igb_set_flowcntl, "I", "Flow Control"); 445 446 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 447 448 /* Determine hardware and mac info */ 449 igb_identify_hardware(adapter); 450 451 /* Setup PCI resources */ 452 if (igb_allocate_pci_resources(adapter)) { 453 device_printf(dev, "Allocation of PCI resources failed\n"); 454 error = ENXIO; 455 goto err_pci; 456 } 457 458 /* Do Shared Code initialization */ 459 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 460 device_printf(dev, "Setup of Shared code failed\n"); 461 error = ENXIO; 462 goto err_pci; 463 } 464 465 e1000_get_bus_info(&adapter->hw); 466 467 /* Sysctls for limiting the amount of work done in the taskqueues */ 468 igb_set_sysctl_value(adapter, "rx_processing_limit", 469 "max number of rx packets to process", 470 &adapter->rx_process_limit, igb_rx_process_limit); 471 472 igb_set_sysctl_value(adapter, "tx_processing_limit", 473 "max number of tx packets to process", 474 &adapter->tx_process_limit, igb_tx_process_limit); 475 476 /* 477 * Validate number of transmit and receive descriptors. It 478 * must not exceed hardware maximum, and must be multiple 479 * of E1000_DBA_ALIGN. 480 */ 481 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || 482 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { 483 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 484 IGB_DEFAULT_TXD, igb_txd); 485 adapter->num_tx_desc = IGB_DEFAULT_TXD; 486 } else 487 adapter->num_tx_desc = igb_txd; 488 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || 489 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { 490 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 491 IGB_DEFAULT_RXD, igb_rxd); 492 adapter->num_rx_desc = IGB_DEFAULT_RXD; 493 } else 494 adapter->num_rx_desc = igb_rxd; 495 496 adapter->hw.mac.autoneg = DO_AUTO_NEG; 497 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 498 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 499 500 /* Copper options */ 501 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 502 adapter->hw.phy.mdix = AUTO_ALL_MODES; 503 adapter->hw.phy.disable_polarity_correction = FALSE; 504 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; 505 } 506 507 /* 508 * Set the frame limits assuming 509 * standard ethernet sized frames. 510 */ 511 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 512 513 /* 514 ** Allocate and Setup Queues 515 */ 516 if (igb_allocate_queues(adapter)) { 517 error = ENOMEM; 518 goto err_pci; 519 } 520 521 /* Allocate the appropriate stats memory */ 522 if (adapter->vf_ifp) { 523 adapter->stats = 524 (struct e1000_vf_stats *)malloc(sizeof \ 525 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO); 526 igb_vf_init_stats(adapter); 527 } else 528 adapter->stats = 529 (struct e1000_hw_stats *)malloc(sizeof \ 530 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO); 531 if (adapter->stats == NULL) { 532 device_printf(dev, "Can not allocate stats memory\n"); 533 error = ENOMEM; 534 goto err_late; 535 } 536 537 /* Allocate multicast array memory. */ 538 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * 539 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 540 if (adapter->mta == NULL) { 541 device_printf(dev, "Can not allocate multicast setup array\n"); 542 error = ENOMEM; 543 goto err_late; 544 } 545 546 /* Some adapter-specific advanced features */ 547 if (adapter->hw.mac.type >= e1000_i350) { 548 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 549 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 550 OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW, 551 adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce"); 552 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 553 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 554 OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW, 555 adapter, 0, igb_sysctl_eee, "I", 556 "Disable Energy Efficient Ethernet"); 557 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 558 if (adapter->hw.mac.type == e1000_i354) 559 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); 560 else 561 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); 562 } 563 } 564 565 /* 566 ** Start from a known state, this is 567 ** important in reading the nvm and 568 ** mac from that. 569 */ 570 e1000_reset_hw(&adapter->hw); 571 572 /* Make sure we have a good EEPROM before we read from it */ 573 if (((adapter->hw.mac.type != e1000_i210) && 574 (adapter->hw.mac.type != e1000_i211)) && 575 (e1000_validate_nvm_checksum(&adapter->hw) < 0)) { 576 /* 577 ** Some PCI-E parts fail the first check due to 578 ** the link being in sleep state, call it again, 579 ** if it fails a second time its a real issue. 580 */ 581 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 582 device_printf(dev, 583 "The EEPROM Checksum Is Not Valid\n"); 584 error = EIO; 585 goto err_late; 586 } 587 } 588 589 /* 590 ** Copy the permanent MAC address out of the EEPROM 591 */ 592 if (e1000_read_mac_addr(&adapter->hw) < 0) { 593 device_printf(dev, "EEPROM read error while reading MAC" 594 " address\n"); 595 error = EIO; 596 goto err_late; 597 } 598 /* Check its sanity */ 599 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { 600 device_printf(dev, "Invalid MAC address\n"); 601 error = EIO; 602 goto err_late; 603 } 604 605 /* Setup OS specific network interface */ 606 if (igb_setup_interface(dev, adapter) != 0) 607 goto err_late; 608 609 /* Now get a good starting state */ 610 igb_reset(adapter); 611 612 /* Initialize statistics */ 613 igb_update_stats_counters(adapter); 614 615 adapter->hw.mac.get_link_status = 1; 616 igb_update_link_status(adapter); 617 618 /* Indicate SOL/IDER usage */ 619 if (e1000_check_reset_block(&adapter->hw)) 620 device_printf(dev, 621 "PHY reset is blocked due to SOL/IDER session.\n"); 622 623 /* Determine if we have to control management hardware */ 624 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 625 626 /* 627 * Setup Wake-on-Lan 628 */ 629 /* APME bit in EEPROM is mapped to WUC.APME */ 630 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; 631 if (eeprom_data) 632 adapter->wol = E1000_WUFC_MAG; 633 634 /* Register for VLAN events */ 635 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 636 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 637 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 638 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 639 640 igb_add_hw_stats(adapter); 641 642 /* Tell the stack that the interface is not active */ 643 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 644 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE; 645 646 adapter->led_dev = led_create(igb_led_func, adapter, 647 device_get_nameunit(dev)); 648 649 /* 650 ** Configure Interrupts 651 */ 652 if ((adapter->msix > 1) && (igb_enable_msix)) 653 error = igb_allocate_msix(adapter); 654 else /* MSI or Legacy */ 655 error = igb_allocate_legacy(adapter); 656 if (error) 657 goto err_late; 658 659 #ifdef DEV_NETMAP 660 igb_netmap_attach(adapter); 661 #endif /* DEV_NETMAP */ 662 INIT_DEBUGOUT("igb_attach: end"); 663 664 return (0); 665 666 err_late: 667 igb_detach(dev); 668 igb_free_transmit_structures(adapter); 669 igb_free_receive_structures(adapter); 670 igb_release_hw_control(adapter); 671 err_pci: 672 igb_free_pci_resources(adapter); 673 if (adapter->ifp != NULL) 674 if_free(adapter->ifp); 675 free(adapter->mta, M_DEVBUF); 676 IGB_CORE_LOCK_DESTROY(adapter); 677 678 return (error); 679 } 680 681 /********************************************************************* 682 * Device removal routine 683 * 684 * The detach entry point is called when the driver is being removed. 685 * This routine stops the adapter and deallocates all the resources 686 * that were allocated for driver operation. 687 * 688 * return 0 on success, positive on failure 689 *********************************************************************/ 690 691 static int 692 igb_detach(device_t dev) 693 { 694 struct adapter *adapter = device_get_softc(dev); 695 struct ifnet *ifp = adapter->ifp; 696 697 INIT_DEBUGOUT("igb_detach: begin"); 698 699 /* Make sure VLANS are not using driver */ 700 if (adapter->ifp->if_vlantrunk != NULL) { 701 device_printf(dev,"Vlan in use, detach first\n"); 702 return (EBUSY); 703 } 704 705 ether_ifdetach(adapter->ifp); 706 707 if (adapter->led_dev != NULL) 708 led_destroy(adapter->led_dev); 709 710 #ifdef DEVICE_POLLING 711 if (ifp->if_capenable & IFCAP_POLLING) 712 ether_poll_deregister(ifp); 713 #endif 714 715 IGB_CORE_LOCK(adapter); 716 adapter->in_detach = 1; 717 igb_stop(adapter); 718 IGB_CORE_UNLOCK(adapter); 719 720 e1000_phy_hw_reset(&adapter->hw); 721 722 /* Give control back to firmware */ 723 igb_release_manageability(adapter); 724 igb_release_hw_control(adapter); 725 726 if (adapter->wol) { 727 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 728 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 729 igb_enable_wakeup(dev); 730 } 731 732 /* Unregister VLAN events */ 733 if (adapter->vlan_attach != NULL) 734 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 735 if (adapter->vlan_detach != NULL) 736 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 737 738 callout_drain(&adapter->timer); 739 740 #ifdef DEV_NETMAP 741 netmap_detach(adapter->ifp); 742 #endif /* DEV_NETMAP */ 743 igb_free_pci_resources(adapter); 744 bus_generic_detach(dev); 745 if_free(ifp); 746 747 igb_free_transmit_structures(adapter); 748 igb_free_receive_structures(adapter); 749 if (adapter->mta != NULL) 750 free(adapter->mta, M_DEVBUF); 751 752 IGB_CORE_LOCK_DESTROY(adapter); 753 754 return (0); 755 } 756 757 /********************************************************************* 758 * 759 * Shutdown entry point 760 * 761 **********************************************************************/ 762 763 static int 764 igb_shutdown(device_t dev) 765 { 766 return igb_suspend(dev); 767 } 768 769 /* 770 * Suspend/resume device methods. 771 */ 772 static int 773 igb_suspend(device_t dev) 774 { 775 struct adapter *adapter = device_get_softc(dev); 776 777 IGB_CORE_LOCK(adapter); 778 779 igb_stop(adapter); 780 781 igb_release_manageability(adapter); 782 igb_release_hw_control(adapter); 783 784 if (adapter->wol) { 785 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 786 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 787 igb_enable_wakeup(dev); 788 } 789 790 IGB_CORE_UNLOCK(adapter); 791 792 return bus_generic_suspend(dev); 793 } 794 795 static int 796 igb_resume(device_t dev) 797 { 798 struct adapter *adapter = device_get_softc(dev); 799 struct tx_ring *txr = adapter->tx_rings; 800 struct ifnet *ifp = adapter->ifp; 801 802 IGB_CORE_LOCK(adapter); 803 igb_init_locked(adapter); 804 igb_init_manageability(adapter); 805 806 if ((ifp->if_flags & IFF_UP) && 807 (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) { 808 for (int i = 0; i < adapter->num_queues; i++, txr++) { 809 IGB_TX_LOCK(txr); 810 #ifndef IGB_LEGACY_TX 811 /* Process the stack queue only if not depleted */ 812 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && 813 !drbr_empty(ifp, txr->br)) 814 igb_mq_start_locked(ifp, txr); 815 #else 816 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 817 igb_start_locked(txr, ifp); 818 #endif 819 IGB_TX_UNLOCK(txr); 820 } 821 } 822 IGB_CORE_UNLOCK(adapter); 823 824 return bus_generic_resume(dev); 825 } 826 827 828 #ifdef IGB_LEGACY_TX 829 830 /********************************************************************* 831 * Transmit entry point 832 * 833 * igb_start is called by the stack to initiate a transmit. 834 * The driver will remain in this routine as long as there are 835 * packets to transmit and transmit resources are available. 836 * In case resources are not available stack is notified and 837 * the packet is requeued. 838 **********************************************************************/ 839 840 static void 841 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) 842 { 843 struct adapter *adapter = ifp->if_softc; 844 struct mbuf *m_head; 845 846 IGB_TX_LOCK_ASSERT(txr); 847 848 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 849 IFF_DRV_RUNNING) 850 return; 851 if (!adapter->link_active) 852 return; 853 854 /* Call cleanup if number of TX descriptors low */ 855 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 856 igb_txeof(txr); 857 858 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 859 if (txr->tx_avail <= IGB_MAX_SCATTER) { 860 txr->queue_status |= IGB_QUEUE_DEPLETED; 861 break; 862 } 863 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 864 if (m_head == NULL) 865 break; 866 /* 867 * Encapsulation can modify our pointer, and or make it 868 * NULL on failure. In that event, we can't requeue. 869 */ 870 if (igb_xmit(txr, &m_head)) { 871 if (m_head != NULL) 872 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 873 if (txr->tx_avail <= IGB_MAX_SCATTER) 874 txr->queue_status |= IGB_QUEUE_DEPLETED; 875 break; 876 } 877 878 /* Send a copy of the frame to the BPF listener */ 879 ETHER_BPF_MTAP(ifp, m_head); 880 881 /* Set watchdog on */ 882 txr->watchdog_time = ticks; 883 txr->queue_status |= IGB_QUEUE_WORKING; 884 } 885 } 886 887 /* 888 * Legacy TX driver routine, called from the 889 * stack, always uses tx[0], and spins for it. 890 * Should not be used with multiqueue tx 891 */ 892 static void 893 igb_start(struct ifnet *ifp) 894 { 895 struct adapter *adapter = ifp->if_softc; 896 struct tx_ring *txr = adapter->tx_rings; 897 898 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 899 IGB_TX_LOCK(txr); 900 igb_start_locked(txr, ifp); 901 IGB_TX_UNLOCK(txr); 902 } 903 return; 904 } 905 906 #else /* ~IGB_LEGACY_TX */ 907 908 /* 909 ** Multiqueue Transmit Entry: 910 ** quick turnaround to the stack 911 ** 912 */ 913 static int 914 igb_mq_start(struct ifnet *ifp, struct mbuf *m) 915 { 916 struct adapter *adapter = ifp->if_softc; 917 struct igb_queue *que; 918 struct tx_ring *txr; 919 int i, err = 0; 920 921 /* Which queue to use */ 922 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 923 i = m->m_pkthdr.flowid % adapter->num_queues; 924 else 925 i = curcpu % adapter->num_queues; 926 txr = &adapter->tx_rings[i]; 927 que = &adapter->queues[i]; 928 929 err = drbr_enqueue(ifp, txr->br, m); 930 if (err) 931 return (err); 932 if (IGB_TX_TRYLOCK(txr)) { 933 igb_mq_start_locked(ifp, txr); 934 IGB_TX_UNLOCK(txr); 935 } else 936 taskqueue_enqueue(que->tq, &txr->txq_task); 937 938 return (0); 939 } 940 941 static int 942 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) 943 { 944 struct adapter *adapter = txr->adapter; 945 struct mbuf *next; 946 int err = 0, enq = 0; 947 948 IGB_TX_LOCK_ASSERT(txr); 949 950 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || 951 adapter->link_active == 0) 952 return (ENETDOWN); 953 954 /* Process the queue */ 955 while ((next = drbr_peek(ifp, txr->br)) != NULL) { 956 if ((err = igb_xmit(txr, &next)) != 0) { 957 if (next == NULL) { 958 /* It was freed, move forward */ 959 drbr_advance(ifp, txr->br); 960 } else { 961 /* 962 * Still have one left, it may not be 963 * the same since the transmit function 964 * may have changed it. 965 */ 966 drbr_putback(ifp, txr->br, next); 967 } 968 break; 969 } 970 drbr_advance(ifp, txr->br); 971 enq++; 972 ifp->if_obytes += next->m_pkthdr.len; 973 if (next->m_flags & M_MCAST) 974 ifp->if_omcasts++; 975 ETHER_BPF_MTAP(ifp, next); 976 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 977 break; 978 } 979 if (enq > 0) { 980 /* Set the watchdog */ 981 txr->queue_status |= IGB_QUEUE_WORKING; 982 txr->watchdog_time = ticks; 983 } 984 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 985 igb_txeof(txr); 986 if (txr->tx_avail <= IGB_MAX_SCATTER) 987 txr->queue_status |= IGB_QUEUE_DEPLETED; 988 return (err); 989 } 990 991 /* 992 * Called from a taskqueue to drain queued transmit packets. 993 */ 994 static void 995 igb_deferred_mq_start(void *arg, int pending) 996 { 997 struct tx_ring *txr = arg; 998 struct adapter *adapter = txr->adapter; 999 struct ifnet *ifp = adapter->ifp; 1000 1001 IGB_TX_LOCK(txr); 1002 if (!drbr_empty(ifp, txr->br)) 1003 igb_mq_start_locked(ifp, txr); 1004 IGB_TX_UNLOCK(txr); 1005 } 1006 1007 /* 1008 ** Flush all ring buffers 1009 */ 1010 static void 1011 igb_qflush(struct ifnet *ifp) 1012 { 1013 struct adapter *adapter = ifp->if_softc; 1014 struct tx_ring *txr = adapter->tx_rings; 1015 struct mbuf *m; 1016 1017 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1018 IGB_TX_LOCK(txr); 1019 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 1020 m_freem(m); 1021 IGB_TX_UNLOCK(txr); 1022 } 1023 if_qflush(ifp); 1024 } 1025 #endif /* ~IGB_LEGACY_TX */ 1026 1027 /********************************************************************* 1028 * Ioctl entry point 1029 * 1030 * igb_ioctl is called when the user wants to configure the 1031 * interface. 1032 * 1033 * return 0 on success, positive on failure 1034 **********************************************************************/ 1035 1036 static int 1037 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 1038 { 1039 struct adapter *adapter = ifp->if_softc; 1040 struct ifreq *ifr = (struct ifreq *)data; 1041 #if defined(INET) || defined(INET6) 1042 struct ifaddr *ifa = (struct ifaddr *)data; 1043 #endif 1044 bool avoid_reset = FALSE; 1045 int error = 0; 1046 1047 if (adapter->in_detach) 1048 return (error); 1049 1050 switch (command) { 1051 case SIOCSIFADDR: 1052 #ifdef INET 1053 if (ifa->ifa_addr->sa_family == AF_INET) 1054 avoid_reset = TRUE; 1055 #endif 1056 #ifdef INET6 1057 if (ifa->ifa_addr->sa_family == AF_INET6) 1058 avoid_reset = TRUE; 1059 #endif 1060 /* 1061 ** Calling init results in link renegotiation, 1062 ** so we avoid doing it when possible. 1063 */ 1064 if (avoid_reset) { 1065 ifp->if_flags |= IFF_UP; 1066 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1067 igb_init(adapter); 1068 #ifdef INET 1069 if (!(ifp->if_flags & IFF_NOARP)) 1070 arp_ifinit(ifp, ifa); 1071 #endif 1072 } else 1073 error = ether_ioctl(ifp, command, data); 1074 break; 1075 case SIOCSIFMTU: 1076 { 1077 int max_frame_size; 1078 1079 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1080 1081 IGB_CORE_LOCK(adapter); 1082 max_frame_size = 9234; 1083 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1084 ETHER_CRC_LEN) { 1085 IGB_CORE_UNLOCK(adapter); 1086 error = EINVAL; 1087 break; 1088 } 1089 1090 ifp->if_mtu = ifr->ifr_mtu; 1091 adapter->max_frame_size = 1092 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1093 igb_init_locked(adapter); 1094 IGB_CORE_UNLOCK(adapter); 1095 break; 1096 } 1097 case SIOCSIFFLAGS: 1098 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1099 SIOCSIFFLAGS (Set Interface Flags)"); 1100 IGB_CORE_LOCK(adapter); 1101 if (ifp->if_flags & IFF_UP) { 1102 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1103 if ((ifp->if_flags ^ adapter->if_flags) & 1104 (IFF_PROMISC | IFF_ALLMULTI)) { 1105 igb_disable_promisc(adapter); 1106 igb_set_promisc(adapter); 1107 } 1108 } else 1109 igb_init_locked(adapter); 1110 } else 1111 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1112 igb_stop(adapter); 1113 adapter->if_flags = ifp->if_flags; 1114 IGB_CORE_UNLOCK(adapter); 1115 break; 1116 case SIOCADDMULTI: 1117 case SIOCDELMULTI: 1118 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1119 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1120 IGB_CORE_LOCK(adapter); 1121 igb_disable_intr(adapter); 1122 igb_set_multi(adapter); 1123 #ifdef DEVICE_POLLING 1124 if (!(ifp->if_capenable & IFCAP_POLLING)) 1125 #endif 1126 igb_enable_intr(adapter); 1127 IGB_CORE_UNLOCK(adapter); 1128 } 1129 break; 1130 case SIOCSIFMEDIA: 1131 /* Check SOL/IDER usage */ 1132 IGB_CORE_LOCK(adapter); 1133 if (e1000_check_reset_block(&adapter->hw)) { 1134 IGB_CORE_UNLOCK(adapter); 1135 device_printf(adapter->dev, "Media change is" 1136 " blocked due to SOL/IDER session.\n"); 1137 break; 1138 } 1139 IGB_CORE_UNLOCK(adapter); 1140 case SIOCGIFMEDIA: 1141 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1142 SIOCxIFMEDIA (Get/Set Interface Media)"); 1143 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1144 break; 1145 case SIOCSIFCAP: 1146 { 1147 int mask, reinit; 1148 1149 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1150 reinit = 0; 1151 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1152 #ifdef DEVICE_POLLING 1153 if (mask & IFCAP_POLLING) { 1154 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1155 error = ether_poll_register(igb_poll, ifp); 1156 if (error) 1157 return (error); 1158 IGB_CORE_LOCK(adapter); 1159 igb_disable_intr(adapter); 1160 ifp->if_capenable |= IFCAP_POLLING; 1161 IGB_CORE_UNLOCK(adapter); 1162 } else { 1163 error = ether_poll_deregister(ifp); 1164 /* Enable interrupt even in error case */ 1165 IGB_CORE_LOCK(adapter); 1166 igb_enable_intr(adapter); 1167 ifp->if_capenable &= ~IFCAP_POLLING; 1168 IGB_CORE_UNLOCK(adapter); 1169 } 1170 } 1171 #endif 1172 if (mask & IFCAP_HWCSUM) { 1173 ifp->if_capenable ^= IFCAP_HWCSUM; 1174 reinit = 1; 1175 } 1176 if (mask & IFCAP_TSO4) { 1177 ifp->if_capenable ^= IFCAP_TSO4; 1178 reinit = 1; 1179 } 1180 if (mask & IFCAP_TSO6) { 1181 ifp->if_capenable ^= IFCAP_TSO6; 1182 reinit = 1; 1183 } 1184 if (mask & IFCAP_VLAN_HWTAGGING) { 1185 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1186 reinit = 1; 1187 } 1188 if (mask & IFCAP_VLAN_HWFILTER) { 1189 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1190 reinit = 1; 1191 } 1192 if (mask & IFCAP_VLAN_HWTSO) { 1193 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1194 reinit = 1; 1195 } 1196 if (mask & IFCAP_LRO) { 1197 ifp->if_capenable ^= IFCAP_LRO; 1198 reinit = 1; 1199 } 1200 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1201 igb_init(adapter); 1202 VLAN_CAPABILITIES(ifp); 1203 break; 1204 } 1205 1206 default: 1207 error = ether_ioctl(ifp, command, data); 1208 break; 1209 } 1210 1211 return (error); 1212 } 1213 1214 1215 /********************************************************************* 1216 * Init entry point 1217 * 1218 * This routine is used in two ways. It is used by the stack as 1219 * init entry point in network interface structure. It is also used 1220 * by the driver as a hw/sw initialization routine to get to a 1221 * consistent state. 1222 * 1223 * return 0 on success, positive on failure 1224 **********************************************************************/ 1225 1226 static void 1227 igb_init_locked(struct adapter *adapter) 1228 { 1229 struct ifnet *ifp = adapter->ifp; 1230 device_t dev = adapter->dev; 1231 1232 INIT_DEBUGOUT("igb_init: begin"); 1233 1234 IGB_CORE_LOCK_ASSERT(adapter); 1235 1236 igb_disable_intr(adapter); 1237 callout_stop(&adapter->timer); 1238 1239 /* Get the latest mac address, User can use a LAA */ 1240 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1241 ETHER_ADDR_LEN); 1242 1243 /* Put the address into the Receive Address Array */ 1244 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1245 1246 igb_reset(adapter); 1247 igb_update_link_status(adapter); 1248 1249 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1250 1251 /* Set hardware offload abilities */ 1252 ifp->if_hwassist = 0; 1253 if (ifp->if_capenable & IFCAP_TXCSUM) { 1254 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1255 #if __FreeBSD_version >= 800000 1256 if ((adapter->hw.mac.type == e1000_82576) || 1257 (adapter->hw.mac.type == e1000_82580)) 1258 ifp->if_hwassist |= CSUM_SCTP; 1259 #endif 1260 } 1261 1262 if (ifp->if_capenable & IFCAP_TSO) 1263 ifp->if_hwassist |= CSUM_TSO; 1264 1265 /* Clear bad data from Rx FIFOs */ 1266 e1000_rx_fifo_flush_82575(&adapter->hw); 1267 1268 /* Configure for OS presence */ 1269 igb_init_manageability(adapter); 1270 1271 /* Prepare transmit descriptors and buffers */ 1272 igb_setup_transmit_structures(adapter); 1273 igb_initialize_transmit_units(adapter); 1274 1275 /* Setup Multicast table */ 1276 igb_set_multi(adapter); 1277 1278 /* 1279 ** Figure out the desired mbuf pool 1280 ** for doing jumbo/packetsplit 1281 */ 1282 if (adapter->max_frame_size <= 2048) 1283 adapter->rx_mbuf_sz = MCLBYTES; 1284 else if (adapter->max_frame_size <= 4096) 1285 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1286 else 1287 adapter->rx_mbuf_sz = MJUM9BYTES; 1288 1289 /* Prepare receive descriptors and buffers */ 1290 if (igb_setup_receive_structures(adapter)) { 1291 device_printf(dev, "Could not setup receive structures\n"); 1292 return; 1293 } 1294 igb_initialize_receive_units(adapter); 1295 1296 /* Enable VLAN support */ 1297 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1298 igb_setup_vlan_hw_support(adapter); 1299 1300 /* Don't lose promiscuous settings */ 1301 igb_set_promisc(adapter); 1302 1303 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1304 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1305 1306 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1307 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1308 1309 if (adapter->msix > 1) /* Set up queue routing */ 1310 igb_configure_queues(adapter); 1311 1312 /* this clears any pending interrupts */ 1313 E1000_READ_REG(&adapter->hw, E1000_ICR); 1314 #ifdef DEVICE_POLLING 1315 /* 1316 * Only enable interrupts if we are not polling, make sure 1317 * they are off otherwise. 1318 */ 1319 if (ifp->if_capenable & IFCAP_POLLING) 1320 igb_disable_intr(adapter); 1321 else 1322 #endif /* DEVICE_POLLING */ 1323 { 1324 igb_enable_intr(adapter); 1325 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); 1326 } 1327 1328 /* Set Energy Efficient Ethernet */ 1329 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 1330 if (adapter->hw.mac.type == e1000_i354) 1331 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); 1332 else 1333 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); 1334 } 1335 } 1336 1337 static void 1338 igb_init(void *arg) 1339 { 1340 struct adapter *adapter = arg; 1341 1342 IGB_CORE_LOCK(adapter); 1343 igb_init_locked(adapter); 1344 IGB_CORE_UNLOCK(adapter); 1345 } 1346 1347 1348 static void 1349 igb_handle_que(void *context, int pending) 1350 { 1351 struct igb_queue *que = context; 1352 struct adapter *adapter = que->adapter; 1353 struct tx_ring *txr = que->txr; 1354 struct ifnet *ifp = adapter->ifp; 1355 1356 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1357 bool more; 1358 1359 more = igb_rxeof(que, adapter->rx_process_limit, NULL); 1360 1361 IGB_TX_LOCK(txr); 1362 igb_txeof(txr); 1363 #ifndef IGB_LEGACY_TX 1364 /* Process the stack queue only if not depleted */ 1365 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && 1366 !drbr_empty(ifp, txr->br)) 1367 igb_mq_start_locked(ifp, txr); 1368 #else 1369 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1370 igb_start_locked(txr, ifp); 1371 #endif 1372 IGB_TX_UNLOCK(txr); 1373 /* Do we need another? */ 1374 if (more) { 1375 taskqueue_enqueue(que->tq, &que->que_task); 1376 return; 1377 } 1378 } 1379 1380 #ifdef DEVICE_POLLING 1381 if (ifp->if_capenable & IFCAP_POLLING) 1382 return; 1383 #endif 1384 /* Reenable this interrupt */ 1385 if (que->eims) 1386 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1387 else 1388 igb_enable_intr(adapter); 1389 } 1390 1391 /* Deal with link in a sleepable context */ 1392 static void 1393 igb_handle_link(void *context, int pending) 1394 { 1395 struct adapter *adapter = context; 1396 1397 IGB_CORE_LOCK(adapter); 1398 igb_handle_link_locked(adapter); 1399 IGB_CORE_UNLOCK(adapter); 1400 } 1401 1402 static void 1403 igb_handle_link_locked(struct adapter *adapter) 1404 { 1405 struct tx_ring *txr = adapter->tx_rings; 1406 struct ifnet *ifp = adapter->ifp; 1407 1408 IGB_CORE_LOCK_ASSERT(adapter); 1409 adapter->hw.mac.get_link_status = 1; 1410 igb_update_link_status(adapter); 1411 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) { 1412 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1413 IGB_TX_LOCK(txr); 1414 #ifndef IGB_LEGACY_TX 1415 /* Process the stack queue only if not depleted */ 1416 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && 1417 !drbr_empty(ifp, txr->br)) 1418 igb_mq_start_locked(ifp, txr); 1419 #else 1420 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1421 igb_start_locked(txr, ifp); 1422 #endif 1423 IGB_TX_UNLOCK(txr); 1424 } 1425 } 1426 } 1427 1428 /********************************************************************* 1429 * 1430 * MSI/Legacy Deferred 1431 * Interrupt Service routine 1432 * 1433 *********************************************************************/ 1434 static int 1435 igb_irq_fast(void *arg) 1436 { 1437 struct adapter *adapter = arg; 1438 struct igb_queue *que = adapter->queues; 1439 u32 reg_icr; 1440 1441 1442 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1443 1444 /* Hot eject? */ 1445 if (reg_icr == 0xffffffff) 1446 return FILTER_STRAY; 1447 1448 /* Definitely not our interrupt. */ 1449 if (reg_icr == 0x0) 1450 return FILTER_STRAY; 1451 1452 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1453 return FILTER_STRAY; 1454 1455 /* 1456 * Mask interrupts until the taskqueue is finished running. This is 1457 * cheap, just assume that it is needed. This also works around the 1458 * MSI message reordering errata on certain systems. 1459 */ 1460 igb_disable_intr(adapter); 1461 taskqueue_enqueue(que->tq, &que->que_task); 1462 1463 /* Link status change */ 1464 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1465 taskqueue_enqueue(que->tq, &adapter->link_task); 1466 1467 if (reg_icr & E1000_ICR_RXO) 1468 adapter->rx_overruns++; 1469 return FILTER_HANDLED; 1470 } 1471 1472 #ifdef DEVICE_POLLING 1473 #if __FreeBSD_version >= 800000 1474 #define POLL_RETURN_COUNT(a) (a) 1475 static int 1476 #else 1477 #define POLL_RETURN_COUNT(a) 1478 static void 1479 #endif 1480 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1481 { 1482 struct adapter *adapter = ifp->if_softc; 1483 struct igb_queue *que; 1484 struct tx_ring *txr; 1485 u32 reg_icr, rx_done = 0; 1486 u32 loop = IGB_MAX_LOOP; 1487 bool more; 1488 1489 IGB_CORE_LOCK(adapter); 1490 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1491 IGB_CORE_UNLOCK(adapter); 1492 return POLL_RETURN_COUNT(rx_done); 1493 } 1494 1495 if (cmd == POLL_AND_CHECK_STATUS) { 1496 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1497 /* Link status change */ 1498 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1499 igb_handle_link_locked(adapter); 1500 1501 if (reg_icr & E1000_ICR_RXO) 1502 adapter->rx_overruns++; 1503 } 1504 IGB_CORE_UNLOCK(adapter); 1505 1506 for (int i = 0; i < adapter->num_queues; i++) { 1507 que = &adapter->queues[i]; 1508 txr = que->txr; 1509 1510 igb_rxeof(que, count, &rx_done); 1511 1512 IGB_TX_LOCK(txr); 1513 do { 1514 more = igb_txeof(txr); 1515 } while (loop-- && more); 1516 #ifndef IGB_LEGACY_TX 1517 if (!drbr_empty(ifp, txr->br)) 1518 igb_mq_start_locked(ifp, txr); 1519 #else 1520 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1521 igb_start_locked(txr, ifp); 1522 #endif 1523 IGB_TX_UNLOCK(txr); 1524 } 1525 1526 return POLL_RETURN_COUNT(rx_done); 1527 } 1528 #endif /* DEVICE_POLLING */ 1529 1530 /********************************************************************* 1531 * 1532 * MSIX Que Interrupt Service routine 1533 * 1534 **********************************************************************/ 1535 static void 1536 igb_msix_que(void *arg) 1537 { 1538 struct igb_queue *que = arg; 1539 struct adapter *adapter = que->adapter; 1540 struct ifnet *ifp = adapter->ifp; 1541 struct tx_ring *txr = que->txr; 1542 struct rx_ring *rxr = que->rxr; 1543 u32 newitr = 0; 1544 bool more_rx; 1545 1546 /* Ignore spurious interrupts */ 1547 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1548 return; 1549 1550 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims); 1551 ++que->irqs; 1552 1553 IGB_TX_LOCK(txr); 1554 igb_txeof(txr); 1555 #ifndef IGB_LEGACY_TX 1556 /* Process the stack queue only if not depleted */ 1557 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && 1558 !drbr_empty(ifp, txr->br)) 1559 igb_mq_start_locked(ifp, txr); 1560 #else 1561 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1562 igb_start_locked(txr, ifp); 1563 #endif 1564 IGB_TX_UNLOCK(txr); 1565 1566 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL); 1567 1568 if (adapter->enable_aim == FALSE) 1569 goto no_calc; 1570 /* 1571 ** Do Adaptive Interrupt Moderation: 1572 ** - Write out last calculated setting 1573 ** - Calculate based on average size over 1574 ** the last interval. 1575 */ 1576 if (que->eitr_setting) 1577 E1000_WRITE_REG(&adapter->hw, 1578 E1000_EITR(que->msix), que->eitr_setting); 1579 1580 que->eitr_setting = 0; 1581 1582 /* Idle, do nothing */ 1583 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1584 goto no_calc; 1585 1586 /* Used half Default if sub-gig */ 1587 if (adapter->link_speed != 1000) 1588 newitr = IGB_DEFAULT_ITR / 2; 1589 else { 1590 if ((txr->bytes) && (txr->packets)) 1591 newitr = txr->bytes/txr->packets; 1592 if ((rxr->bytes) && (rxr->packets)) 1593 newitr = max(newitr, 1594 (rxr->bytes / rxr->packets)); 1595 newitr += 24; /* account for hardware frame, crc */ 1596 /* set an upper boundary */ 1597 newitr = min(newitr, 3000); 1598 /* Be nice to the mid range */ 1599 if ((newitr > 300) && (newitr < 1200)) 1600 newitr = (newitr / 3); 1601 else 1602 newitr = (newitr / 2); 1603 } 1604 newitr &= 0x7FFC; /* Mask invalid bits */ 1605 if (adapter->hw.mac.type == e1000_82575) 1606 newitr |= newitr << 16; 1607 else 1608 newitr |= E1000_EITR_CNT_IGNR; 1609 1610 /* save for next interrupt */ 1611 que->eitr_setting = newitr; 1612 1613 /* Reset state */ 1614 txr->bytes = 0; 1615 txr->packets = 0; 1616 rxr->bytes = 0; 1617 rxr->packets = 0; 1618 1619 no_calc: 1620 /* Schedule a clean task if needed*/ 1621 if (more_rx) 1622 taskqueue_enqueue(que->tq, &que->que_task); 1623 else 1624 /* Reenable this interrupt */ 1625 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1626 return; 1627 } 1628 1629 1630 /********************************************************************* 1631 * 1632 * MSIX Link Interrupt Service routine 1633 * 1634 **********************************************************************/ 1635 1636 static void 1637 igb_msix_link(void *arg) 1638 { 1639 struct adapter *adapter = arg; 1640 u32 icr; 1641 1642 ++adapter->link_irq; 1643 icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1644 if (!(icr & E1000_ICR_LSC)) 1645 goto spurious; 1646 igb_handle_link(adapter, 0); 1647 1648 spurious: 1649 /* Rearm */ 1650 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); 1651 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); 1652 return; 1653 } 1654 1655 1656 /********************************************************************* 1657 * 1658 * Media Ioctl callback 1659 * 1660 * This routine is called whenever the user queries the status of 1661 * the interface using ifconfig. 1662 * 1663 **********************************************************************/ 1664 static void 1665 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1666 { 1667 struct adapter *adapter = ifp->if_softc; 1668 1669 INIT_DEBUGOUT("igb_media_status: begin"); 1670 1671 IGB_CORE_LOCK(adapter); 1672 igb_update_link_status(adapter); 1673 1674 ifmr->ifm_status = IFM_AVALID; 1675 ifmr->ifm_active = IFM_ETHER; 1676 1677 if (!adapter->link_active) { 1678 IGB_CORE_UNLOCK(adapter); 1679 return; 1680 } 1681 1682 ifmr->ifm_status |= IFM_ACTIVE; 1683 1684 switch (adapter->link_speed) { 1685 case 10: 1686 ifmr->ifm_active |= IFM_10_T; 1687 break; 1688 case 100: 1689 /* 1690 ** Support for 100Mb SFP - these are Fiber 1691 ** but the media type appears as serdes 1692 */ 1693 if (adapter->hw.phy.media_type == 1694 e1000_media_type_internal_serdes) 1695 ifmr->ifm_active |= IFM_100_FX; 1696 else 1697 ifmr->ifm_active |= IFM_100_TX; 1698 break; 1699 case 1000: 1700 ifmr->ifm_active |= IFM_1000_T; 1701 break; 1702 case 2500: 1703 ifmr->ifm_active |= IFM_2500_SX; 1704 break; 1705 } 1706 1707 if (adapter->link_duplex == FULL_DUPLEX) 1708 ifmr->ifm_active |= IFM_FDX; 1709 else 1710 ifmr->ifm_active |= IFM_HDX; 1711 1712 IGB_CORE_UNLOCK(adapter); 1713 } 1714 1715 /********************************************************************* 1716 * 1717 * Media Ioctl callback 1718 * 1719 * This routine is called when the user changes speed/duplex using 1720 * media/mediopt option with ifconfig. 1721 * 1722 **********************************************************************/ 1723 static int 1724 igb_media_change(struct ifnet *ifp) 1725 { 1726 struct adapter *adapter = ifp->if_softc; 1727 struct ifmedia *ifm = &adapter->media; 1728 1729 INIT_DEBUGOUT("igb_media_change: begin"); 1730 1731 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1732 return (EINVAL); 1733 1734 IGB_CORE_LOCK(adapter); 1735 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1736 case IFM_AUTO: 1737 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1738 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1739 break; 1740 case IFM_1000_LX: 1741 case IFM_1000_SX: 1742 case IFM_1000_T: 1743 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1744 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1745 break; 1746 case IFM_100_TX: 1747 adapter->hw.mac.autoneg = FALSE; 1748 adapter->hw.phy.autoneg_advertised = 0; 1749 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1750 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1751 else 1752 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1753 break; 1754 case IFM_10_T: 1755 adapter->hw.mac.autoneg = FALSE; 1756 adapter->hw.phy.autoneg_advertised = 0; 1757 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1758 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1759 else 1760 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1761 break; 1762 default: 1763 device_printf(adapter->dev, "Unsupported media type\n"); 1764 } 1765 1766 igb_init_locked(adapter); 1767 IGB_CORE_UNLOCK(adapter); 1768 1769 return (0); 1770 } 1771 1772 1773 /********************************************************************* 1774 * 1775 * This routine maps the mbufs to Advanced TX descriptors. 1776 * 1777 **********************************************************************/ 1778 static int 1779 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1780 { 1781 struct adapter *adapter = txr->adapter; 1782 u32 olinfo_status = 0, cmd_type_len; 1783 int i, j, error, nsegs; 1784 int first; 1785 bool remap = TRUE; 1786 struct mbuf *m_head; 1787 bus_dma_segment_t segs[IGB_MAX_SCATTER]; 1788 bus_dmamap_t map; 1789 struct igb_tx_buf *txbuf; 1790 union e1000_adv_tx_desc *txd = NULL; 1791 1792 m_head = *m_headp; 1793 1794 /* Basic descriptor defines */ 1795 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | 1796 E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); 1797 1798 if (m_head->m_flags & M_VLANTAG) 1799 cmd_type_len |= E1000_ADVTXD_DCMD_VLE; 1800 1801 /* 1802 * Important to capture the first descriptor 1803 * used because it will contain the index of 1804 * the one we tell the hardware to report back 1805 */ 1806 first = txr->next_avail_desc; 1807 txbuf = &txr->tx_buffers[first]; 1808 map = txbuf->map; 1809 1810 /* 1811 * Map the packet for DMA. 1812 */ 1813 retry: 1814 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1815 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1816 1817 if (__predict_false(error)) { 1818 struct mbuf *m; 1819 1820 switch (error) { 1821 case EFBIG: 1822 /* Try it again? - one try */ 1823 if (remap == TRUE) { 1824 remap = FALSE; 1825 m = m_collapse(*m_headp, M_NOWAIT, 1826 IGB_MAX_SCATTER); 1827 if (m == NULL) { 1828 adapter->mbuf_defrag_failed++; 1829 m_freem(*m_headp); 1830 *m_headp = NULL; 1831 return (ENOBUFS); 1832 } 1833 *m_headp = m; 1834 goto retry; 1835 } else 1836 return (error); 1837 default: 1838 txr->no_tx_dma_setup++; 1839 m_freem(*m_headp); 1840 *m_headp = NULL; 1841 return (error); 1842 } 1843 } 1844 1845 /* Make certain there are enough descriptors */ 1846 if (nsegs > txr->tx_avail - 2) { 1847 txr->no_desc_avail++; 1848 bus_dmamap_unload(txr->txtag, map); 1849 return (ENOBUFS); 1850 } 1851 m_head = *m_headp; 1852 1853 /* 1854 ** Set up the appropriate offload context 1855 ** this will consume the first descriptor 1856 */ 1857 error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1858 if (__predict_false(error)) { 1859 m_freem(*m_headp); 1860 *m_headp = NULL; 1861 return (error); 1862 } 1863 1864 /* 82575 needs the queue index added */ 1865 if (adapter->hw.mac.type == e1000_82575) 1866 olinfo_status |= txr->me << 4; 1867 1868 i = txr->next_avail_desc; 1869 for (j = 0; j < nsegs; j++) { 1870 bus_size_t seglen; 1871 bus_addr_t segaddr; 1872 1873 txbuf = &txr->tx_buffers[i]; 1874 txd = &txr->tx_base[i]; 1875 seglen = segs[j].ds_len; 1876 segaddr = htole64(segs[j].ds_addr); 1877 1878 txd->read.buffer_addr = segaddr; 1879 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS | 1880 cmd_type_len | seglen); 1881 txd->read.olinfo_status = htole32(olinfo_status); 1882 1883 if (++i == txr->num_desc) 1884 i = 0; 1885 } 1886 1887 txd->read.cmd_type_len |= 1888 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 1889 txr->tx_avail -= nsegs; 1890 txr->next_avail_desc = i; 1891 1892 txbuf->m_head = m_head; 1893 /* 1894 ** Here we swap the map so the last descriptor, 1895 ** which gets the completion interrupt has the 1896 ** real map, and the first descriptor gets the 1897 ** unused map from this descriptor. 1898 */ 1899 txr->tx_buffers[first].map = txbuf->map; 1900 txbuf->map = map; 1901 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1902 1903 /* Set the EOP descriptor that will be marked done */ 1904 txbuf = &txr->tx_buffers[first]; 1905 txbuf->eop = txd; 1906 1907 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1908 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1909 /* 1910 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1911 * hardware that this frame is available to transmit. 1912 */ 1913 ++txr->total_packets; 1914 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1915 1916 return (0); 1917 } 1918 static void 1919 igb_set_promisc(struct adapter *adapter) 1920 { 1921 struct ifnet *ifp = adapter->ifp; 1922 struct e1000_hw *hw = &adapter->hw; 1923 u32 reg; 1924 1925 if (adapter->vf_ifp) { 1926 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 1927 return; 1928 } 1929 1930 reg = E1000_READ_REG(hw, E1000_RCTL); 1931 if (ifp->if_flags & IFF_PROMISC) { 1932 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1933 E1000_WRITE_REG(hw, E1000_RCTL, reg); 1934 } else if (ifp->if_flags & IFF_ALLMULTI) { 1935 reg |= E1000_RCTL_MPE; 1936 reg &= ~E1000_RCTL_UPE; 1937 E1000_WRITE_REG(hw, E1000_RCTL, reg); 1938 } 1939 } 1940 1941 static void 1942 igb_disable_promisc(struct adapter *adapter) 1943 { 1944 struct e1000_hw *hw = &adapter->hw; 1945 struct ifnet *ifp = adapter->ifp; 1946 u32 reg; 1947 int mcnt = 0; 1948 1949 if (adapter->vf_ifp) { 1950 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 1951 return; 1952 } 1953 reg = E1000_READ_REG(hw, E1000_RCTL); 1954 reg &= (~E1000_RCTL_UPE); 1955 if (ifp->if_flags & IFF_ALLMULTI) 1956 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 1957 else { 1958 struct ifmultiaddr *ifma; 1959 #if __FreeBSD_version < 800000 1960 IF_ADDR_LOCK(ifp); 1961 #else 1962 if_maddr_rlock(ifp); 1963 #endif 1964 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1965 if (ifma->ifma_addr->sa_family != AF_LINK) 1966 continue; 1967 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1968 break; 1969 mcnt++; 1970 } 1971 #if __FreeBSD_version < 800000 1972 IF_ADDR_UNLOCK(ifp); 1973 #else 1974 if_maddr_runlock(ifp); 1975 #endif 1976 } 1977 /* Don't disable if in MAX groups */ 1978 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 1979 reg &= (~E1000_RCTL_MPE); 1980 E1000_WRITE_REG(hw, E1000_RCTL, reg); 1981 } 1982 1983 1984 /********************************************************************* 1985 * Multicast Update 1986 * 1987 * This routine is called whenever multicast address list is updated. 1988 * 1989 **********************************************************************/ 1990 1991 static void 1992 igb_set_multi(struct adapter *adapter) 1993 { 1994 struct ifnet *ifp = adapter->ifp; 1995 struct ifmultiaddr *ifma; 1996 u32 reg_rctl = 0; 1997 u8 *mta; 1998 1999 int mcnt = 0; 2000 2001 IOCTL_DEBUGOUT("igb_set_multi: begin"); 2002 2003 mta = adapter->mta; 2004 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN * 2005 MAX_NUM_MULTICAST_ADDRESSES); 2006 2007 #if __FreeBSD_version < 800000 2008 IF_ADDR_LOCK(ifp); 2009 #else 2010 if_maddr_rlock(ifp); 2011 #endif 2012 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2013 if (ifma->ifma_addr->sa_family != AF_LINK) 2014 continue; 2015 2016 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2017 break; 2018 2019 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2020 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2021 mcnt++; 2022 } 2023 #if __FreeBSD_version < 800000 2024 IF_ADDR_UNLOCK(ifp); 2025 #else 2026 if_maddr_runlock(ifp); 2027 #endif 2028 2029 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2030 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2031 reg_rctl |= E1000_RCTL_MPE; 2032 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2033 } else 2034 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2035 } 2036 2037 2038 /********************************************************************* 2039 * Timer routine: 2040 * This routine checks for link status, 2041 * updates statistics, and does the watchdog. 2042 * 2043 **********************************************************************/ 2044 2045 static void 2046 igb_local_timer(void *arg) 2047 { 2048 struct adapter *adapter = arg; 2049 device_t dev = adapter->dev; 2050 struct ifnet *ifp = adapter->ifp; 2051 struct tx_ring *txr = adapter->tx_rings; 2052 struct igb_queue *que = adapter->queues; 2053 int hung = 0, busy = 0; 2054 2055 2056 IGB_CORE_LOCK_ASSERT(adapter); 2057 2058 igb_update_link_status(adapter); 2059 igb_update_stats_counters(adapter); 2060 2061 /* 2062 ** Check the TX queues status 2063 ** - central locked handling of OACTIVE 2064 ** - watchdog only if all queues show hung 2065 */ 2066 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 2067 if ((txr->queue_status & IGB_QUEUE_HUNG) && 2068 (adapter->pause_frames == 0)) 2069 ++hung; 2070 if (txr->queue_status & IGB_QUEUE_DEPLETED) 2071 ++busy; 2072 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0) 2073 taskqueue_enqueue(que->tq, &que->que_task); 2074 } 2075 if (hung == adapter->num_queues) 2076 goto timeout; 2077 if (busy == adapter->num_queues) 2078 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 2079 else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) && 2080 (busy < adapter->num_queues)) 2081 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2082 2083 adapter->pause_frames = 0; 2084 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 2085 #ifndef DEVICE_POLLING 2086 /* Schedule all queue interrupts - deadlock protection */ 2087 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask); 2088 #endif 2089 return; 2090 2091 timeout: 2092 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2093 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2094 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), 2095 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); 2096 device_printf(dev,"TX(%d) desc avail = %d," 2097 "Next TX to Clean = %d\n", 2098 txr->me, txr->tx_avail, txr->next_to_clean); 2099 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2100 adapter->watchdog_events++; 2101 igb_init_locked(adapter); 2102 } 2103 2104 static void 2105 igb_update_link_status(struct adapter *adapter) 2106 { 2107 struct e1000_hw *hw = &adapter->hw; 2108 struct e1000_fc_info *fc = &hw->fc; 2109 struct ifnet *ifp = adapter->ifp; 2110 device_t dev = adapter->dev; 2111 struct tx_ring *txr = adapter->tx_rings; 2112 u32 link_check, thstat, ctrl; 2113 char *flowctl = NULL; 2114 2115 link_check = thstat = ctrl = 0; 2116 2117 /* Get the cached link value or read for real */ 2118 switch (hw->phy.media_type) { 2119 case e1000_media_type_copper: 2120 if (hw->mac.get_link_status) { 2121 /* Do the work to read phy */ 2122 e1000_check_for_link(hw); 2123 link_check = !hw->mac.get_link_status; 2124 } else 2125 link_check = TRUE; 2126 break; 2127 case e1000_media_type_fiber: 2128 e1000_check_for_link(hw); 2129 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2130 E1000_STATUS_LU); 2131 break; 2132 case e1000_media_type_internal_serdes: 2133 e1000_check_for_link(hw); 2134 link_check = adapter->hw.mac.serdes_has_link; 2135 break; 2136 /* VF device is type_unknown */ 2137 case e1000_media_type_unknown: 2138 e1000_check_for_link(hw); 2139 link_check = !hw->mac.get_link_status; 2140 /* Fall thru */ 2141 default: 2142 break; 2143 } 2144 2145 /* Check for thermal downshift or shutdown */ 2146 if (hw->mac.type == e1000_i350) { 2147 thstat = E1000_READ_REG(hw, E1000_THSTAT); 2148 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT); 2149 } 2150 2151 /* Get the flow control for display */ 2152 switch (fc->current_mode) { 2153 case e1000_fc_rx_pause: 2154 flowctl = "RX"; 2155 break; 2156 case e1000_fc_tx_pause: 2157 flowctl = "TX"; 2158 break; 2159 case e1000_fc_full: 2160 flowctl = "Full"; 2161 break; 2162 case e1000_fc_none: 2163 default: 2164 flowctl = "None"; 2165 break; 2166 } 2167 2168 /* Now we check if a transition has happened */ 2169 if (link_check && (adapter->link_active == 0)) { 2170 e1000_get_speed_and_duplex(&adapter->hw, 2171 &adapter->link_speed, &adapter->link_duplex); 2172 if (bootverbose) 2173 device_printf(dev, "Link is up %d Mbps %s," 2174 " Flow Control: %s\n", 2175 adapter->link_speed, 2176 ((adapter->link_duplex == FULL_DUPLEX) ? 2177 "Full Duplex" : "Half Duplex"), flowctl); 2178 adapter->link_active = 1; 2179 ifp->if_baudrate = adapter->link_speed * 1000000; 2180 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) && 2181 (thstat & E1000_THSTAT_LINK_THROTTLE)) 2182 device_printf(dev, "Link: thermal downshift\n"); 2183 /* Delay Link Up for Phy update */ 2184 if (((hw->mac.type == e1000_i210) || 2185 (hw->mac.type == e1000_i211)) && 2186 (hw->phy.id == I210_I_PHY_ID)) 2187 msec_delay(I210_LINK_DELAY); 2188 /* Reset if the media type changed. */ 2189 if (hw->dev_spec._82575.media_changed) { 2190 hw->dev_spec._82575.media_changed = false; 2191 adapter->flags |= IGB_MEDIA_RESET; 2192 igb_reset(adapter); 2193 } 2194 /* This can sleep */ 2195 if_link_state_change(ifp, LINK_STATE_UP); 2196 } else if (!link_check && (adapter->link_active == 1)) { 2197 ifp->if_baudrate = adapter->link_speed = 0; 2198 adapter->link_duplex = 0; 2199 if (bootverbose) 2200 device_printf(dev, "Link is Down\n"); 2201 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) && 2202 (thstat & E1000_THSTAT_PWR_DOWN)) 2203 device_printf(dev, "Link: thermal shutdown\n"); 2204 adapter->link_active = 0; 2205 /* This can sleep */ 2206 if_link_state_change(ifp, LINK_STATE_DOWN); 2207 /* Reset queue state */ 2208 for (int i = 0; i < adapter->num_queues; i++, txr++) 2209 txr->queue_status = IGB_QUEUE_IDLE; 2210 } 2211 } 2212 2213 /********************************************************************* 2214 * 2215 * This routine disables all traffic on the adapter by issuing a 2216 * global reset on the MAC and deallocates TX/RX buffers. 2217 * 2218 **********************************************************************/ 2219 2220 static void 2221 igb_stop(void *arg) 2222 { 2223 struct adapter *adapter = arg; 2224 struct ifnet *ifp = adapter->ifp; 2225 struct tx_ring *txr = adapter->tx_rings; 2226 2227 IGB_CORE_LOCK_ASSERT(adapter); 2228 2229 INIT_DEBUGOUT("igb_stop: begin"); 2230 2231 igb_disable_intr(adapter); 2232 2233 callout_stop(&adapter->timer); 2234 2235 /* Tell the stack that the interface is no longer active */ 2236 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2237 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 2238 2239 /* Disarm watchdog timer. */ 2240 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2241 IGB_TX_LOCK(txr); 2242 txr->queue_status = IGB_QUEUE_IDLE; 2243 IGB_TX_UNLOCK(txr); 2244 } 2245 2246 e1000_reset_hw(&adapter->hw); 2247 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2248 2249 e1000_led_off(&adapter->hw); 2250 e1000_cleanup_led(&adapter->hw); 2251 } 2252 2253 2254 /********************************************************************* 2255 * 2256 * Determine hardware revision. 2257 * 2258 **********************************************************************/ 2259 static void 2260 igb_identify_hardware(struct adapter *adapter) 2261 { 2262 device_t dev = adapter->dev; 2263 2264 /* Make sure our PCI config space has the necessary stuff set */ 2265 pci_enable_busmaster(dev); 2266 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2267 2268 /* Save off the information about this board */ 2269 adapter->hw.vendor_id = pci_get_vendor(dev); 2270 adapter->hw.device_id = pci_get_device(dev); 2271 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2272 adapter->hw.subsystem_vendor_id = 2273 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2274 adapter->hw.subsystem_device_id = 2275 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2276 2277 /* Set MAC type early for PCI setup */ 2278 e1000_set_mac_type(&adapter->hw); 2279 2280 /* Are we a VF device? */ 2281 if ((adapter->hw.mac.type == e1000_vfadapt) || 2282 (adapter->hw.mac.type == e1000_vfadapt_i350)) 2283 adapter->vf_ifp = 1; 2284 else 2285 adapter->vf_ifp = 0; 2286 } 2287 2288 static int 2289 igb_allocate_pci_resources(struct adapter *adapter) 2290 { 2291 device_t dev = adapter->dev; 2292 int rid; 2293 2294 rid = PCIR_BAR(0); 2295 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2296 &rid, RF_ACTIVE); 2297 if (adapter->pci_mem == NULL) { 2298 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2299 return (ENXIO); 2300 } 2301 adapter->osdep.mem_bus_space_tag = 2302 rman_get_bustag(adapter->pci_mem); 2303 adapter->osdep.mem_bus_space_handle = 2304 rman_get_bushandle(adapter->pci_mem); 2305 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2306 2307 adapter->num_queues = 1; /* Defaults for Legacy or MSI */ 2308 2309 /* This will setup either MSI/X or MSI */ 2310 adapter->msix = igb_setup_msix(adapter); 2311 adapter->hw.back = &adapter->osdep; 2312 2313 return (0); 2314 } 2315 2316 /********************************************************************* 2317 * 2318 * Setup the Legacy or MSI Interrupt handler 2319 * 2320 **********************************************************************/ 2321 static int 2322 igb_allocate_legacy(struct adapter *adapter) 2323 { 2324 device_t dev = adapter->dev; 2325 struct igb_queue *que = adapter->queues; 2326 #ifndef IGB_LEGACY_TX 2327 struct tx_ring *txr = adapter->tx_rings; 2328 #endif 2329 int error, rid = 0; 2330 2331 /* Turn off all interrupts */ 2332 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2333 2334 /* MSI RID is 1 */ 2335 if (adapter->msix == 1) 2336 rid = 1; 2337 2338 /* We allocate a single interrupt resource */ 2339 adapter->res = bus_alloc_resource_any(dev, 2340 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2341 if (adapter->res == NULL) { 2342 device_printf(dev, "Unable to allocate bus resource: " 2343 "interrupt\n"); 2344 return (ENXIO); 2345 } 2346 2347 #ifndef IGB_LEGACY_TX 2348 TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr); 2349 #endif 2350 2351 /* 2352 * Try allocating a fast interrupt and the associated deferred 2353 * processing contexts. 2354 */ 2355 TASK_INIT(&que->que_task, 0, igb_handle_que, que); 2356 /* Make tasklet for deferred link handling */ 2357 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); 2358 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2359 taskqueue_thread_enqueue, &que->tq); 2360 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq", 2361 device_get_nameunit(adapter->dev)); 2362 if ((error = bus_setup_intr(dev, adapter->res, 2363 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, 2364 adapter, &adapter->tag)) != 0) { 2365 device_printf(dev, "Failed to register fast interrupt " 2366 "handler: %d\n", error); 2367 taskqueue_free(que->tq); 2368 que->tq = NULL; 2369 return (error); 2370 } 2371 2372 return (0); 2373 } 2374 2375 2376 /********************************************************************* 2377 * 2378 * Setup the MSIX Queue Interrupt handlers: 2379 * 2380 **********************************************************************/ 2381 static int 2382 igb_allocate_msix(struct adapter *adapter) 2383 { 2384 device_t dev = adapter->dev; 2385 struct igb_queue *que = adapter->queues; 2386 int error, rid, vector = 0; 2387 2388 /* Be sure to start with all interrupts disabled */ 2389 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); 2390 E1000_WRITE_FLUSH(&adapter->hw); 2391 2392 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2393 rid = vector +1; 2394 que->res = bus_alloc_resource_any(dev, 2395 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2396 if (que->res == NULL) { 2397 device_printf(dev, 2398 "Unable to allocate bus resource: " 2399 "MSIX Queue Interrupt\n"); 2400 return (ENXIO); 2401 } 2402 error = bus_setup_intr(dev, que->res, 2403 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2404 igb_msix_que, que, &que->tag); 2405 if (error) { 2406 que->res = NULL; 2407 device_printf(dev, "Failed to register Queue handler"); 2408 return (error); 2409 } 2410 #if __FreeBSD_version >= 800504 2411 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2412 #endif 2413 que->msix = vector; 2414 if (adapter->hw.mac.type == e1000_82575) 2415 que->eims = E1000_EICR_TX_QUEUE0 << i; 2416 else 2417 que->eims = 1 << vector; 2418 /* 2419 ** Bind the msix vector, and thus the 2420 ** rings to the corresponding cpu. 2421 */ 2422 if (adapter->num_queues > 1) { 2423 if (igb_last_bind_cpu < 0) 2424 igb_last_bind_cpu = CPU_FIRST(); 2425 bus_bind_intr(dev, que->res, igb_last_bind_cpu); 2426 device_printf(dev, 2427 "Bound queue %d to cpu %d\n", 2428 i,igb_last_bind_cpu); 2429 igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu); 2430 } 2431 #ifndef IGB_LEGACY_TX 2432 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start, 2433 que->txr); 2434 #endif 2435 /* Make tasklet for deferred handling */ 2436 TASK_INIT(&que->que_task, 0, igb_handle_que, que); 2437 que->tq = taskqueue_create("igb_que", M_NOWAIT, 2438 taskqueue_thread_enqueue, &que->tq); 2439 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 2440 device_get_nameunit(adapter->dev)); 2441 } 2442 2443 /* And Link */ 2444 rid = vector + 1; 2445 adapter->res = bus_alloc_resource_any(dev, 2446 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2447 if (adapter->res == NULL) { 2448 device_printf(dev, 2449 "Unable to allocate bus resource: " 2450 "MSIX Link Interrupt\n"); 2451 return (ENXIO); 2452 } 2453 if ((error = bus_setup_intr(dev, adapter->res, 2454 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2455 igb_msix_link, adapter, &adapter->tag)) != 0) { 2456 device_printf(dev, "Failed to register Link handler"); 2457 return (error); 2458 } 2459 #if __FreeBSD_version >= 800504 2460 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2461 #endif 2462 adapter->linkvec = vector; 2463 2464 return (0); 2465 } 2466 2467 2468 static void 2469 igb_configure_queues(struct adapter *adapter) 2470 { 2471 struct e1000_hw *hw = &adapter->hw; 2472 struct igb_queue *que; 2473 u32 tmp, ivar = 0, newitr = 0; 2474 2475 /* First turn on RSS capability */ 2476 if (adapter->hw.mac.type != e1000_82575) 2477 E1000_WRITE_REG(hw, E1000_GPIE, 2478 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 2479 E1000_GPIE_PBA | E1000_GPIE_NSICR); 2480 2481 /* Turn on MSIX */ 2482 switch (adapter->hw.mac.type) { 2483 case e1000_82580: 2484 case e1000_i350: 2485 case e1000_i354: 2486 case e1000_i210: 2487 case e1000_i211: 2488 case e1000_vfadapt: 2489 case e1000_vfadapt_i350: 2490 /* RX entries */ 2491 for (int i = 0; i < adapter->num_queues; i++) { 2492 u32 index = i >> 1; 2493 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2494 que = &adapter->queues[i]; 2495 if (i & 1) { 2496 ivar &= 0xFF00FFFF; 2497 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2498 } else { 2499 ivar &= 0xFFFFFF00; 2500 ivar |= que->msix | E1000_IVAR_VALID; 2501 } 2502 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2503 } 2504 /* TX entries */ 2505 for (int i = 0; i < adapter->num_queues; i++) { 2506 u32 index = i >> 1; 2507 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2508 que = &adapter->queues[i]; 2509 if (i & 1) { 2510 ivar &= 0x00FFFFFF; 2511 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2512 } else { 2513 ivar &= 0xFFFF00FF; 2514 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2515 } 2516 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2517 adapter->que_mask |= que->eims; 2518 } 2519 2520 /* And for the link interrupt */ 2521 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2522 adapter->link_mask = 1 << adapter->linkvec; 2523 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2524 break; 2525 case e1000_82576: 2526 /* RX entries */ 2527 for (int i = 0; i < adapter->num_queues; i++) { 2528 u32 index = i & 0x7; /* Each IVAR has two entries */ 2529 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2530 que = &adapter->queues[i]; 2531 if (i < 8) { 2532 ivar &= 0xFFFFFF00; 2533 ivar |= que->msix | E1000_IVAR_VALID; 2534 } else { 2535 ivar &= 0xFF00FFFF; 2536 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2537 } 2538 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2539 adapter->que_mask |= que->eims; 2540 } 2541 /* TX entries */ 2542 for (int i = 0; i < adapter->num_queues; i++) { 2543 u32 index = i & 0x7; /* Each IVAR has two entries */ 2544 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2545 que = &adapter->queues[i]; 2546 if (i < 8) { 2547 ivar &= 0xFFFF00FF; 2548 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2549 } else { 2550 ivar &= 0x00FFFFFF; 2551 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2552 } 2553 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2554 adapter->que_mask |= que->eims; 2555 } 2556 2557 /* And for the link interrupt */ 2558 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2559 adapter->link_mask = 1 << adapter->linkvec; 2560 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2561 break; 2562 2563 case e1000_82575: 2564 /* enable MSI-X support*/ 2565 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 2566 tmp |= E1000_CTRL_EXT_PBA_CLR; 2567 /* Auto-Mask interrupts upon ICR read. */ 2568 tmp |= E1000_CTRL_EXT_EIAME; 2569 tmp |= E1000_CTRL_EXT_IRCA; 2570 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 2571 2572 /* Queues */ 2573 for (int i = 0; i < adapter->num_queues; i++) { 2574 que = &adapter->queues[i]; 2575 tmp = E1000_EICR_RX_QUEUE0 << i; 2576 tmp |= E1000_EICR_TX_QUEUE0 << i; 2577 que->eims = tmp; 2578 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 2579 i, que->eims); 2580 adapter->que_mask |= que->eims; 2581 } 2582 2583 /* Link */ 2584 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), 2585 E1000_EIMS_OTHER); 2586 adapter->link_mask |= E1000_EIMS_OTHER; 2587 default: 2588 break; 2589 } 2590 2591 /* Set the starting interrupt rate */ 2592 if (igb_max_interrupt_rate > 0) 2593 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC; 2594 2595 if (hw->mac.type == e1000_82575) 2596 newitr |= newitr << 16; 2597 else 2598 newitr |= E1000_EITR_CNT_IGNR; 2599 2600 for (int i = 0; i < adapter->num_queues; i++) { 2601 que = &adapter->queues[i]; 2602 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr); 2603 } 2604 2605 return; 2606 } 2607 2608 2609 static void 2610 igb_free_pci_resources(struct adapter *adapter) 2611 { 2612 struct igb_queue *que = adapter->queues; 2613 device_t dev = adapter->dev; 2614 int rid; 2615 2616 /* 2617 ** There is a slight possibility of a failure mode 2618 ** in attach that will result in entering this function 2619 ** before interrupt resources have been initialized, and 2620 ** in that case we do not want to execute the loops below 2621 ** We can detect this reliably by the state of the adapter 2622 ** res pointer. 2623 */ 2624 if (adapter->res == NULL) 2625 goto mem; 2626 2627 /* 2628 * First release all the interrupt resources: 2629 */ 2630 for (int i = 0; i < adapter->num_queues; i++, que++) { 2631 rid = que->msix + 1; 2632 if (que->tag != NULL) { 2633 bus_teardown_intr(dev, que->res, que->tag); 2634 que->tag = NULL; 2635 } 2636 if (que->res != NULL) 2637 bus_release_resource(dev, 2638 SYS_RES_IRQ, rid, que->res); 2639 } 2640 2641 /* Clean the Legacy or Link interrupt last */ 2642 if (adapter->linkvec) /* we are doing MSIX */ 2643 rid = adapter->linkvec + 1; 2644 else 2645 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2646 2647 que = adapter->queues; 2648 if (adapter->tag != NULL) { 2649 taskqueue_drain(que->tq, &adapter->link_task); 2650 bus_teardown_intr(dev, adapter->res, adapter->tag); 2651 adapter->tag = NULL; 2652 } 2653 if (adapter->res != NULL) 2654 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2655 2656 for (int i = 0; i < adapter->num_queues; i++, que++) { 2657 if (que->tq != NULL) { 2658 #ifndef IGB_LEGACY_TX 2659 taskqueue_drain(que->tq, &que->txr->txq_task); 2660 #endif 2661 taskqueue_drain(que->tq, &que->que_task); 2662 taskqueue_free(que->tq); 2663 } 2664 } 2665 mem: 2666 if (adapter->msix) 2667 pci_release_msi(dev); 2668 2669 if (adapter->msix_mem != NULL) 2670 bus_release_resource(dev, SYS_RES_MEMORY, 2671 adapter->memrid, adapter->msix_mem); 2672 2673 if (adapter->pci_mem != NULL) 2674 bus_release_resource(dev, SYS_RES_MEMORY, 2675 PCIR_BAR(0), adapter->pci_mem); 2676 2677 } 2678 2679 /* 2680 * Setup Either MSI/X or MSI 2681 */ 2682 static int 2683 igb_setup_msix(struct adapter *adapter) 2684 { 2685 device_t dev = adapter->dev; 2686 int bar, want, queues, msgs, maxqueues; 2687 2688 /* tuneable override */ 2689 if (igb_enable_msix == 0) 2690 goto msi; 2691 2692 /* First try MSI/X */ 2693 msgs = pci_msix_count(dev); 2694 if (msgs == 0) 2695 goto msi; 2696 /* 2697 ** Some new devices, as with ixgbe, now may 2698 ** use a different BAR, so we need to keep 2699 ** track of which is used. 2700 */ 2701 adapter->memrid = PCIR_BAR(IGB_MSIX_BAR); 2702 bar = pci_read_config(dev, adapter->memrid, 4); 2703 if (bar == 0) /* use next bar */ 2704 adapter->memrid += 4; 2705 adapter->msix_mem = bus_alloc_resource_any(dev, 2706 SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE); 2707 if (adapter->msix_mem == NULL) { 2708 /* May not be enabled */ 2709 device_printf(adapter->dev, 2710 "Unable to map MSIX table \n"); 2711 goto msi; 2712 } 2713 2714 /* Figure out a reasonable auto config value */ 2715 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2716 2717 /* Manual override */ 2718 if (igb_num_queues != 0) 2719 queues = igb_num_queues; 2720 2721 /* Sanity check based on HW */ 2722 switch (adapter->hw.mac.type) { 2723 case e1000_82575: 2724 maxqueues = 4; 2725 break; 2726 case e1000_82576: 2727 case e1000_82580: 2728 case e1000_i350: 2729 case e1000_i354: 2730 maxqueues = 8; 2731 break; 2732 case e1000_i210: 2733 maxqueues = 4; 2734 break; 2735 case e1000_i211: 2736 maxqueues = 2; 2737 break; 2738 default: /* VF interfaces */ 2739 maxqueues = 1; 2740 break; 2741 } 2742 if (queues > maxqueues) 2743 queues = maxqueues; 2744 2745 /* Manual override */ 2746 if (igb_num_queues != 0) 2747 queues = igb_num_queues; 2748 2749 /* 2750 ** One vector (RX/TX pair) per queue 2751 ** plus an additional for Link interrupt 2752 */ 2753 want = queues + 1; 2754 if (msgs >= want) 2755 msgs = want; 2756 else { 2757 device_printf(adapter->dev, 2758 "MSIX Configuration Problem, " 2759 "%d vectors configured, but %d queues wanted!\n", 2760 msgs, want); 2761 goto msi; 2762 } 2763 if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) { 2764 device_printf(adapter->dev, 2765 "Using MSIX interrupts with %d vectors\n", msgs); 2766 adapter->num_queues = queues; 2767 return (msgs); 2768 } 2769 /* 2770 ** If MSIX alloc failed or provided us with 2771 ** less than needed, free and fall through to MSI 2772 */ 2773 pci_release_msi(dev); 2774 2775 msi: 2776 if (adapter->msix_mem != NULL) { 2777 bus_release_resource(dev, SYS_RES_MEMORY, 2778 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2779 adapter->msix_mem = NULL; 2780 } 2781 msgs = 1; 2782 if (pci_alloc_msi(dev, &msgs) == 0) { 2783 device_printf(adapter->dev," Using an MSI interrupt\n"); 2784 return (msgs); 2785 } 2786 device_printf(adapter->dev," Using a Legacy interrupt\n"); 2787 return (0); 2788 } 2789 2790 /********************************************************************* 2791 * 2792 * Initialize the DMA Coalescing feature 2793 * 2794 **********************************************************************/ 2795 static void 2796 igb_init_dmac(struct adapter *adapter, u32 pba) 2797 { 2798 device_t dev = adapter->dev; 2799 struct e1000_hw *hw = &adapter->hw; 2800 u32 dmac, reg = ~E1000_DMACR_DMAC_EN; 2801 u16 hwm; 2802 2803 if (hw->mac.type == e1000_i211) 2804 return; 2805 2806 if (hw->mac.type > e1000_82580) { 2807 2808 if (adapter->dmac == 0) { /* Disabling it */ 2809 E1000_WRITE_REG(hw, E1000_DMACR, reg); 2810 return; 2811 } else 2812 device_printf(dev, "DMA Coalescing enabled\n"); 2813 2814 /* Set starting threshold */ 2815 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0); 2816 2817 hwm = 64 * pba - adapter->max_frame_size / 16; 2818 if (hwm < 64 * (pba - 6)) 2819 hwm = 64 * (pba - 6); 2820 reg = E1000_READ_REG(hw, E1000_FCRTC); 2821 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 2822 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 2823 & E1000_FCRTC_RTH_COAL_MASK); 2824 E1000_WRITE_REG(hw, E1000_FCRTC, reg); 2825 2826 2827 dmac = pba - adapter->max_frame_size / 512; 2828 if (dmac < pba - 10) 2829 dmac = pba - 10; 2830 reg = E1000_READ_REG(hw, E1000_DMACR); 2831 reg &= ~E1000_DMACR_DMACTHR_MASK; 2832 reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT) 2833 & E1000_DMACR_DMACTHR_MASK); 2834 2835 /* transition to L0x or L1 if available..*/ 2836 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 2837 2838 /* Check if status is 2.5Gb backplane connection 2839 * before configuration of watchdog timer, which is 2840 * in msec values in 12.8usec intervals 2841 * watchdog timer= msec values in 32usec intervals 2842 * for non 2.5Gb connection 2843 */ 2844 if (hw->mac.type == e1000_i354) { 2845 int status = E1000_READ_REG(hw, E1000_STATUS); 2846 if ((status & E1000_STATUS_2P5_SKU) && 2847 (!(status & E1000_STATUS_2P5_SKU_OVER))) 2848 reg |= ((adapter->dmac * 5) >> 6); 2849 else 2850 reg |= (adapter->dmac >> 5); 2851 } else { 2852 reg |= (adapter->dmac >> 5); 2853 } 2854 2855 E1000_WRITE_REG(hw, E1000_DMACR, reg); 2856 2857 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); 2858 2859 /* Set the interval before transition */ 2860 reg = E1000_READ_REG(hw, E1000_DMCTLX); 2861 if (hw->mac.type == e1000_i350) 2862 reg |= IGB_DMCTLX_DCFLUSH_DIS; 2863 /* 2864 ** in 2.5Gb connection, TTLX unit is 0.4 usec 2865 ** which is 0x4*2 = 0xA. But delay is still 4 usec 2866 */ 2867 if (hw->mac.type == e1000_i354) { 2868 int status = E1000_READ_REG(hw, E1000_STATUS); 2869 if ((status & E1000_STATUS_2P5_SKU) && 2870 (!(status & E1000_STATUS_2P5_SKU_OVER))) 2871 reg |= 0xA; 2872 else 2873 reg |= 0x4; 2874 } else { 2875 reg |= 0x4; 2876 } 2877 2878 E1000_WRITE_REG(hw, E1000_DMCTLX, reg); 2879 2880 /* free space in tx packet buffer to wake from DMA coal */ 2881 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE - 2882 (2 * adapter->max_frame_size)) >> 6); 2883 2884 /* make low power state decision controlled by DMA coal */ 2885 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 2886 reg &= ~E1000_PCIEMISC_LX_DECISION; 2887 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg); 2888 2889 } else if (hw->mac.type == e1000_82580) { 2890 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 2891 E1000_WRITE_REG(hw, E1000_PCIEMISC, 2892 reg & ~E1000_PCIEMISC_LX_DECISION); 2893 E1000_WRITE_REG(hw, E1000_DMACR, 0); 2894 } 2895 } 2896 2897 2898 /********************************************************************* 2899 * 2900 * Set up an fresh starting state 2901 * 2902 **********************************************************************/ 2903 static void 2904 igb_reset(struct adapter *adapter) 2905 { 2906 device_t dev = adapter->dev; 2907 struct e1000_hw *hw = &adapter->hw; 2908 struct e1000_fc_info *fc = &hw->fc; 2909 struct ifnet *ifp = adapter->ifp; 2910 u32 pba = 0; 2911 u16 hwm; 2912 2913 INIT_DEBUGOUT("igb_reset: begin"); 2914 2915 /* Let the firmware know the OS is in control */ 2916 igb_get_hw_control(adapter); 2917 2918 /* 2919 * Packet Buffer Allocation (PBA) 2920 * Writing PBA sets the receive portion of the buffer 2921 * the remainder is used for the transmit buffer. 2922 */ 2923 switch (hw->mac.type) { 2924 case e1000_82575: 2925 pba = E1000_PBA_32K; 2926 break; 2927 case e1000_82576: 2928 case e1000_vfadapt: 2929 pba = E1000_READ_REG(hw, E1000_RXPBS); 2930 pba &= E1000_RXPBS_SIZE_MASK_82576; 2931 break; 2932 case e1000_82580: 2933 case e1000_i350: 2934 case e1000_i354: 2935 case e1000_vfadapt_i350: 2936 pba = E1000_READ_REG(hw, E1000_RXPBS); 2937 pba = e1000_rxpbs_adjust_82580(pba); 2938 break; 2939 case e1000_i210: 2940 case e1000_i211: 2941 pba = E1000_PBA_34K; 2942 default: 2943 break; 2944 } 2945 2946 /* Special needs in case of Jumbo frames */ 2947 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { 2948 u32 tx_space, min_tx, min_rx; 2949 pba = E1000_READ_REG(hw, E1000_PBA); 2950 tx_space = pba >> 16; 2951 pba &= 0xffff; 2952 min_tx = (adapter->max_frame_size + 2953 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 2954 min_tx = roundup2(min_tx, 1024); 2955 min_tx >>= 10; 2956 min_rx = adapter->max_frame_size; 2957 min_rx = roundup2(min_rx, 1024); 2958 min_rx >>= 10; 2959 if (tx_space < min_tx && 2960 ((min_tx - tx_space) < pba)) { 2961 pba = pba - (min_tx - tx_space); 2962 /* 2963 * if short on rx space, rx wins 2964 * and must trump tx adjustment 2965 */ 2966 if (pba < min_rx) 2967 pba = min_rx; 2968 } 2969 E1000_WRITE_REG(hw, E1000_PBA, pba); 2970 } 2971 2972 INIT_DEBUGOUT1("igb_init: pba=%dK",pba); 2973 2974 /* 2975 * These parameters control the automatic generation (Tx) and 2976 * response (Rx) to Ethernet PAUSE frames. 2977 * - High water mark should allow for at least two frames to be 2978 * received after sending an XOFF. 2979 * - Low water mark works best when it is very near the high water mark. 2980 * This allows the receiver to restart by sending XON when it has 2981 * drained a bit. 2982 */ 2983 hwm = min(((pba << 10) * 9 / 10), 2984 ((pba << 10) - 2 * adapter->max_frame_size)); 2985 2986 if (hw->mac.type < e1000_82576) { 2987 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */ 2988 fc->low_water = fc->high_water - 8; 2989 } else { 2990 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ 2991 fc->low_water = fc->high_water - 16; 2992 } 2993 2994 fc->pause_time = IGB_FC_PAUSE_TIME; 2995 fc->send_xon = TRUE; 2996 if (adapter->fc) 2997 fc->requested_mode = adapter->fc; 2998 else 2999 fc->requested_mode = e1000_fc_default; 3000 3001 /* Issue a global reset */ 3002 e1000_reset_hw(hw); 3003 E1000_WRITE_REG(hw, E1000_WUC, 0); 3004 3005 /* Reset for AutoMediaDetect */ 3006 if (adapter->flags & IGB_MEDIA_RESET) { 3007 e1000_setup_init_funcs(hw, TRUE); 3008 e1000_get_bus_info(hw); 3009 adapter->flags &= ~IGB_MEDIA_RESET; 3010 } 3011 3012 if (e1000_init_hw(hw) < 0) 3013 device_printf(dev, "Hardware Initialization Failed\n"); 3014 3015 /* Setup DMA Coalescing */ 3016 igb_init_dmac(adapter, pba); 3017 3018 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 3019 e1000_get_phy_info(hw); 3020 e1000_check_for_link(hw); 3021 return; 3022 } 3023 3024 /********************************************************************* 3025 * 3026 * Setup networking device structure and register an interface. 3027 * 3028 **********************************************************************/ 3029 static int 3030 igb_setup_interface(device_t dev, struct adapter *adapter) 3031 { 3032 struct ifnet *ifp; 3033 3034 INIT_DEBUGOUT("igb_setup_interface: begin"); 3035 3036 ifp = adapter->ifp = if_alloc(IFT_ETHER); 3037 if (ifp == NULL) { 3038 device_printf(dev, "can not allocate ifnet structure\n"); 3039 return (-1); 3040 } 3041 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 3042 ifp->if_init = igb_init; 3043 ifp->if_softc = adapter; 3044 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 3045 ifp->if_ioctl = igb_ioctl; 3046 3047 /* TSO parameters */ 3048 ifp->if_hw_tsomax = IP_MAXPACKET; 3049 ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER; 3050 ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE; 3051 3052 #ifndef IGB_LEGACY_TX 3053 ifp->if_transmit = igb_mq_start; 3054 ifp->if_qflush = igb_qflush; 3055 #else 3056 ifp->if_start = igb_start; 3057 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 3058 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 3059 IFQ_SET_READY(&ifp->if_snd); 3060 #endif 3061 3062 ether_ifattach(ifp, adapter->hw.mac.addr); 3063 3064 ifp->if_capabilities = ifp->if_capenable = 0; 3065 3066 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 3067 ifp->if_capabilities |= IFCAP_TSO; 3068 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 3069 ifp->if_capenable = ifp->if_capabilities; 3070 3071 /* Don't enable LRO by default */ 3072 ifp->if_capabilities |= IFCAP_LRO; 3073 3074 #ifdef DEVICE_POLLING 3075 ifp->if_capabilities |= IFCAP_POLLING; 3076 #endif 3077 3078 /* 3079 * Tell the upper layer(s) we 3080 * support full VLAN capability. 3081 */ 3082 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 3083 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 3084 | IFCAP_VLAN_HWTSO 3085 | IFCAP_VLAN_MTU; 3086 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING 3087 | IFCAP_VLAN_HWTSO 3088 | IFCAP_VLAN_MTU; 3089 3090 /* 3091 ** Don't turn this on by default, if vlans are 3092 ** created on another pseudo device (eg. lagg) 3093 ** then vlan events are not passed thru, breaking 3094 ** operation, but with HW FILTER off it works. If 3095 ** using vlans directly on the igb driver you can 3096 ** enable this and get full hardware tag filtering. 3097 */ 3098 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 3099 3100 /* 3101 * Specify the media types supported by this adapter and register 3102 * callbacks to update media and link information 3103 */ 3104 ifmedia_init(&adapter->media, IFM_IMASK, 3105 igb_media_change, igb_media_status); 3106 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3107 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3108 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 3109 0, NULL); 3110 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 3111 } else { 3112 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 3113 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 3114 0, NULL); 3115 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 3116 0, NULL); 3117 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 3118 0, NULL); 3119 if (adapter->hw.phy.type != e1000_phy_ife) { 3120 ifmedia_add(&adapter->media, 3121 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 3122 ifmedia_add(&adapter->media, 3123 IFM_ETHER | IFM_1000_T, 0, NULL); 3124 } 3125 } 3126 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 3127 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 3128 return (0); 3129 } 3130 3131 3132 /* 3133 * Manage DMA'able memory. 3134 */ 3135 static void 3136 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3137 { 3138 if (error) 3139 return; 3140 *(bus_addr_t *) arg = segs[0].ds_addr; 3141 } 3142 3143 static int 3144 igb_dma_malloc(struct adapter *adapter, bus_size_t size, 3145 struct igb_dma_alloc *dma, int mapflags) 3146 { 3147 int error; 3148 3149 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 3150 IGB_DBA_ALIGN, 0, /* alignment, bounds */ 3151 BUS_SPACE_MAXADDR, /* lowaddr */ 3152 BUS_SPACE_MAXADDR, /* highaddr */ 3153 NULL, NULL, /* filter, filterarg */ 3154 size, /* maxsize */ 3155 1, /* nsegments */ 3156 size, /* maxsegsize */ 3157 0, /* flags */ 3158 NULL, /* lockfunc */ 3159 NULL, /* lockarg */ 3160 &dma->dma_tag); 3161 if (error) { 3162 device_printf(adapter->dev, 3163 "%s: bus_dma_tag_create failed: %d\n", 3164 __func__, error); 3165 goto fail_0; 3166 } 3167 3168 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 3169 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 3170 if (error) { 3171 device_printf(adapter->dev, 3172 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 3173 __func__, (uintmax_t)size, error); 3174 goto fail_2; 3175 } 3176 3177 dma->dma_paddr = 0; 3178 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3179 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 3180 if (error || dma->dma_paddr == 0) { 3181 device_printf(adapter->dev, 3182 "%s: bus_dmamap_load failed: %d\n", 3183 __func__, error); 3184 goto fail_3; 3185 } 3186 3187 return (0); 3188 3189 fail_3: 3190 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3191 fail_2: 3192 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3193 bus_dma_tag_destroy(dma->dma_tag); 3194 fail_0: 3195 dma->dma_tag = NULL; 3196 3197 return (error); 3198 } 3199 3200 static void 3201 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) 3202 { 3203 if (dma->dma_tag == NULL) 3204 return; 3205 if (dma->dma_paddr != 0) { 3206 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3207 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3208 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3209 dma->dma_paddr = 0; 3210 } 3211 if (dma->dma_vaddr != NULL) { 3212 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3213 dma->dma_vaddr = NULL; 3214 } 3215 bus_dma_tag_destroy(dma->dma_tag); 3216 dma->dma_tag = NULL; 3217 } 3218 3219 3220 /********************************************************************* 3221 * 3222 * Allocate memory for the transmit and receive rings, and then 3223 * the descriptors associated with each, called only once at attach. 3224 * 3225 **********************************************************************/ 3226 static int 3227 igb_allocate_queues(struct adapter *adapter) 3228 { 3229 device_t dev = adapter->dev; 3230 struct igb_queue *que = NULL; 3231 struct tx_ring *txr = NULL; 3232 struct rx_ring *rxr = NULL; 3233 int rsize, tsize, error = E1000_SUCCESS; 3234 int txconf = 0, rxconf = 0; 3235 3236 /* First allocate the top level queue structs */ 3237 if (!(adapter->queues = 3238 (struct igb_queue *) malloc(sizeof(struct igb_queue) * 3239 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3240 device_printf(dev, "Unable to allocate queue memory\n"); 3241 error = ENOMEM; 3242 goto fail; 3243 } 3244 3245 /* Next allocate the TX ring struct memory */ 3246 if (!(adapter->tx_rings = 3247 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 3248 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3249 device_printf(dev, "Unable to allocate TX ring memory\n"); 3250 error = ENOMEM; 3251 goto tx_fail; 3252 } 3253 3254 /* Now allocate the RX */ 3255 if (!(adapter->rx_rings = 3256 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 3257 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3258 device_printf(dev, "Unable to allocate RX ring memory\n"); 3259 error = ENOMEM; 3260 goto rx_fail; 3261 } 3262 3263 tsize = roundup2(adapter->num_tx_desc * 3264 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); 3265 /* 3266 * Now set up the TX queues, txconf is needed to handle the 3267 * possibility that things fail midcourse and we need to 3268 * undo memory gracefully 3269 */ 3270 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 3271 /* Set up some basics */ 3272 txr = &adapter->tx_rings[i]; 3273 txr->adapter = adapter; 3274 txr->me = i; 3275 txr->num_desc = adapter->num_tx_desc; 3276 3277 /* Initialize the TX lock */ 3278 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 3279 device_get_nameunit(dev), txr->me); 3280 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 3281 3282 if (igb_dma_malloc(adapter, tsize, 3283 &txr->txdma, BUS_DMA_NOWAIT)) { 3284 device_printf(dev, 3285 "Unable to allocate TX Descriptor memory\n"); 3286 error = ENOMEM; 3287 goto err_tx_desc; 3288 } 3289 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr; 3290 bzero((void *)txr->tx_base, tsize); 3291 3292 /* Now allocate transmit buffers for the ring */ 3293 if (igb_allocate_transmit_buffers(txr)) { 3294 device_printf(dev, 3295 "Critical Failure setting up transmit buffers\n"); 3296 error = ENOMEM; 3297 goto err_tx_desc; 3298 } 3299 #ifndef IGB_LEGACY_TX 3300 /* Allocate a buf ring */ 3301 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF, 3302 M_WAITOK, &txr->tx_mtx); 3303 #endif 3304 } 3305 3306 /* 3307 * Next the RX queues... 3308 */ 3309 rsize = roundup2(adapter->num_rx_desc * 3310 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 3311 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 3312 rxr = &adapter->rx_rings[i]; 3313 rxr->adapter = adapter; 3314 rxr->me = i; 3315 3316 /* Initialize the RX lock */ 3317 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 3318 device_get_nameunit(dev), txr->me); 3319 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 3320 3321 if (igb_dma_malloc(adapter, rsize, 3322 &rxr->rxdma, BUS_DMA_NOWAIT)) { 3323 device_printf(dev, 3324 "Unable to allocate RxDescriptor memory\n"); 3325 error = ENOMEM; 3326 goto err_rx_desc; 3327 } 3328 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; 3329 bzero((void *)rxr->rx_base, rsize); 3330 3331 /* Allocate receive buffers for the ring*/ 3332 if (igb_allocate_receive_buffers(rxr)) { 3333 device_printf(dev, 3334 "Critical Failure setting up receive buffers\n"); 3335 error = ENOMEM; 3336 goto err_rx_desc; 3337 } 3338 } 3339 3340 /* 3341 ** Finally set up the queue holding structs 3342 */ 3343 for (int i = 0; i < adapter->num_queues; i++) { 3344 que = &adapter->queues[i]; 3345 que->adapter = adapter; 3346 que->txr = &adapter->tx_rings[i]; 3347 que->rxr = &adapter->rx_rings[i]; 3348 } 3349 3350 return (0); 3351 3352 err_rx_desc: 3353 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 3354 igb_dma_free(adapter, &rxr->rxdma); 3355 err_tx_desc: 3356 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 3357 igb_dma_free(adapter, &txr->txdma); 3358 free(adapter->rx_rings, M_DEVBUF); 3359 rx_fail: 3360 #ifndef IGB_LEGACY_TX 3361 buf_ring_free(txr->br, M_DEVBUF); 3362 #endif 3363 free(adapter->tx_rings, M_DEVBUF); 3364 tx_fail: 3365 free(adapter->queues, M_DEVBUF); 3366 fail: 3367 return (error); 3368 } 3369 3370 /********************************************************************* 3371 * 3372 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3373 * the information needed to transmit a packet on the wire. This is 3374 * called only once at attach, setup is done every reset. 3375 * 3376 **********************************************************************/ 3377 static int 3378 igb_allocate_transmit_buffers(struct tx_ring *txr) 3379 { 3380 struct adapter *adapter = txr->adapter; 3381 device_t dev = adapter->dev; 3382 struct igb_tx_buf *txbuf; 3383 int error, i; 3384 3385 /* 3386 * Setup DMA descriptor areas. 3387 */ 3388 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3389 1, 0, /* alignment, bounds */ 3390 BUS_SPACE_MAXADDR, /* lowaddr */ 3391 BUS_SPACE_MAXADDR, /* highaddr */ 3392 NULL, NULL, /* filter, filterarg */ 3393 IGB_TSO_SIZE, /* maxsize */ 3394 IGB_MAX_SCATTER, /* nsegments */ 3395 PAGE_SIZE, /* maxsegsize */ 3396 0, /* flags */ 3397 NULL, /* lockfunc */ 3398 NULL, /* lockfuncarg */ 3399 &txr->txtag))) { 3400 device_printf(dev,"Unable to allocate TX DMA tag\n"); 3401 goto fail; 3402 } 3403 3404 if (!(txr->tx_buffers = 3405 (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) * 3406 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3407 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3408 error = ENOMEM; 3409 goto fail; 3410 } 3411 3412 /* Create the descriptor buffer dma maps */ 3413 txbuf = txr->tx_buffers; 3414 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3415 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 3416 if (error != 0) { 3417 device_printf(dev, "Unable to create TX DMA map\n"); 3418 goto fail; 3419 } 3420 } 3421 3422 return 0; 3423 fail: 3424 /* We free all, it handles case where we are in the middle */ 3425 igb_free_transmit_structures(adapter); 3426 return (error); 3427 } 3428 3429 /********************************************************************* 3430 * 3431 * Initialize a transmit ring. 3432 * 3433 **********************************************************************/ 3434 static void 3435 igb_setup_transmit_ring(struct tx_ring *txr) 3436 { 3437 struct adapter *adapter = txr->adapter; 3438 struct igb_tx_buf *txbuf; 3439 int i; 3440 #ifdef DEV_NETMAP 3441 struct netmap_adapter *na = NA(adapter->ifp); 3442 struct netmap_slot *slot; 3443 #endif /* DEV_NETMAP */ 3444 3445 /* Clear the old descriptor contents */ 3446 IGB_TX_LOCK(txr); 3447 #ifdef DEV_NETMAP 3448 slot = netmap_reset(na, NR_TX, txr->me, 0); 3449 #endif /* DEV_NETMAP */ 3450 bzero((void *)txr->tx_base, 3451 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); 3452 /* Reset indices */ 3453 txr->next_avail_desc = 0; 3454 txr->next_to_clean = 0; 3455 3456 /* Free any existing tx buffers. */ 3457 txbuf = txr->tx_buffers; 3458 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3459 if (txbuf->m_head != NULL) { 3460 bus_dmamap_sync(txr->txtag, txbuf->map, 3461 BUS_DMASYNC_POSTWRITE); 3462 bus_dmamap_unload(txr->txtag, txbuf->map); 3463 m_freem(txbuf->m_head); 3464 txbuf->m_head = NULL; 3465 } 3466 #ifdef DEV_NETMAP 3467 if (slot) { 3468 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); 3469 /* no need to set the address */ 3470 netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si)); 3471 } 3472 #endif /* DEV_NETMAP */ 3473 /* clear the watch index */ 3474 txbuf->eop = NULL; 3475 } 3476 3477 /* Set number of descriptors available */ 3478 txr->tx_avail = adapter->num_tx_desc; 3479 3480 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3481 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3482 IGB_TX_UNLOCK(txr); 3483 } 3484 3485 /********************************************************************* 3486 * 3487 * Initialize all transmit rings. 3488 * 3489 **********************************************************************/ 3490 static void 3491 igb_setup_transmit_structures(struct adapter *adapter) 3492 { 3493 struct tx_ring *txr = adapter->tx_rings; 3494 3495 for (int i = 0; i < adapter->num_queues; i++, txr++) 3496 igb_setup_transmit_ring(txr); 3497 3498 return; 3499 } 3500 3501 /********************************************************************* 3502 * 3503 * Enable transmit unit. 3504 * 3505 **********************************************************************/ 3506 static void 3507 igb_initialize_transmit_units(struct adapter *adapter) 3508 { 3509 struct tx_ring *txr = adapter->tx_rings; 3510 struct e1000_hw *hw = &adapter->hw; 3511 u32 tctl, txdctl; 3512 3513 INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); 3514 tctl = txdctl = 0; 3515 3516 /* Setup the Tx Descriptor Rings */ 3517 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3518 u64 bus_addr = txr->txdma.dma_paddr; 3519 3520 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3521 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3522 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3523 (uint32_t)(bus_addr >> 32)); 3524 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3525 (uint32_t)bus_addr); 3526 3527 /* Setup the HW Tx Head and Tail descriptor pointers */ 3528 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3529 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3530 3531 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3532 E1000_READ_REG(hw, E1000_TDBAL(i)), 3533 E1000_READ_REG(hw, E1000_TDLEN(i))); 3534 3535 txr->queue_status = IGB_QUEUE_IDLE; 3536 3537 txdctl |= IGB_TX_PTHRESH; 3538 txdctl |= IGB_TX_HTHRESH << 8; 3539 txdctl |= IGB_TX_WTHRESH << 16; 3540 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 3541 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); 3542 } 3543 3544 if (adapter->vf_ifp) 3545 return; 3546 3547 e1000_config_collision_dist(hw); 3548 3549 /* Program the Transmit Control Register */ 3550 tctl = E1000_READ_REG(hw, E1000_TCTL); 3551 tctl &= ~E1000_TCTL_CT; 3552 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3553 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3554 3555 /* This write will effectively turn on the transmit unit. */ 3556 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 3557 } 3558 3559 /********************************************************************* 3560 * 3561 * Free all transmit rings. 3562 * 3563 **********************************************************************/ 3564 static void 3565 igb_free_transmit_structures(struct adapter *adapter) 3566 { 3567 struct tx_ring *txr = adapter->tx_rings; 3568 3569 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3570 IGB_TX_LOCK(txr); 3571 igb_free_transmit_buffers(txr); 3572 igb_dma_free(adapter, &txr->txdma); 3573 IGB_TX_UNLOCK(txr); 3574 IGB_TX_LOCK_DESTROY(txr); 3575 } 3576 free(adapter->tx_rings, M_DEVBUF); 3577 } 3578 3579 /********************************************************************* 3580 * 3581 * Free transmit ring related data structures. 3582 * 3583 **********************************************************************/ 3584 static void 3585 igb_free_transmit_buffers(struct tx_ring *txr) 3586 { 3587 struct adapter *adapter = txr->adapter; 3588 struct igb_tx_buf *tx_buffer; 3589 int i; 3590 3591 INIT_DEBUGOUT("free_transmit_ring: begin"); 3592 3593 if (txr->tx_buffers == NULL) 3594 return; 3595 3596 tx_buffer = txr->tx_buffers; 3597 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3598 if (tx_buffer->m_head != NULL) { 3599 bus_dmamap_sync(txr->txtag, tx_buffer->map, 3600 BUS_DMASYNC_POSTWRITE); 3601 bus_dmamap_unload(txr->txtag, 3602 tx_buffer->map); 3603 m_freem(tx_buffer->m_head); 3604 tx_buffer->m_head = NULL; 3605 if (tx_buffer->map != NULL) { 3606 bus_dmamap_destroy(txr->txtag, 3607 tx_buffer->map); 3608 tx_buffer->map = NULL; 3609 } 3610 } else if (tx_buffer->map != NULL) { 3611 bus_dmamap_unload(txr->txtag, 3612 tx_buffer->map); 3613 bus_dmamap_destroy(txr->txtag, 3614 tx_buffer->map); 3615 tx_buffer->map = NULL; 3616 } 3617 } 3618 #ifndef IGB_LEGACY_TX 3619 if (txr->br != NULL) 3620 buf_ring_free(txr->br, M_DEVBUF); 3621 #endif 3622 if (txr->tx_buffers != NULL) { 3623 free(txr->tx_buffers, M_DEVBUF); 3624 txr->tx_buffers = NULL; 3625 } 3626 if (txr->txtag != NULL) { 3627 bus_dma_tag_destroy(txr->txtag); 3628 txr->txtag = NULL; 3629 } 3630 return; 3631 } 3632 3633 /********************************************************************** 3634 * 3635 * Setup work for hardware segmentation offload (TSO) on 3636 * adapters using advanced tx descriptors 3637 * 3638 **********************************************************************/ 3639 static int 3640 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, 3641 u32 *cmd_type_len, u32 *olinfo_status) 3642 { 3643 struct adapter *adapter = txr->adapter; 3644 struct e1000_adv_tx_context_desc *TXD; 3645 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3646 u32 mss_l4len_idx = 0, paylen; 3647 u16 vtag = 0, eh_type; 3648 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3649 struct ether_vlan_header *eh; 3650 #ifdef INET6 3651 struct ip6_hdr *ip6; 3652 #endif 3653 #ifdef INET 3654 struct ip *ip; 3655 #endif 3656 struct tcphdr *th; 3657 3658 3659 /* 3660 * Determine where frame payload starts. 3661 * Jump over vlan headers if already present 3662 */ 3663 eh = mtod(mp, struct ether_vlan_header *); 3664 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3665 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3666 eh_type = eh->evl_proto; 3667 } else { 3668 ehdrlen = ETHER_HDR_LEN; 3669 eh_type = eh->evl_encap_proto; 3670 } 3671 3672 switch (ntohs(eh_type)) { 3673 #ifdef INET6 3674 case ETHERTYPE_IPV6: 3675 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3676 /* XXX-BZ For now we do not pretend to support ext. hdrs. */ 3677 if (ip6->ip6_nxt != IPPROTO_TCP) 3678 return (ENXIO); 3679 ip_hlen = sizeof(struct ip6_hdr); 3680 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3681 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3682 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 3683 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 3684 break; 3685 #endif 3686 #ifdef INET 3687 case ETHERTYPE_IP: 3688 ip = (struct ip *)(mp->m_data + ehdrlen); 3689 if (ip->ip_p != IPPROTO_TCP) 3690 return (ENXIO); 3691 ip->ip_sum = 0; 3692 ip_hlen = ip->ip_hl << 2; 3693 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3694 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3695 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3696 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3697 /* Tell transmit desc to also do IPv4 checksum. */ 3698 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 3699 break; 3700 #endif 3701 default: 3702 panic("%s: CSUM_TSO but no supported IP version (0x%04x)", 3703 __func__, ntohs(eh_type)); 3704 break; 3705 } 3706 3707 ctxd = txr->next_avail_desc; 3708 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3709 3710 tcp_hlen = th->th_off << 2; 3711 3712 /* This is used in the transmit desc in encap */ 3713 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 3714 3715 /* VLAN MACLEN IPLEN */ 3716 if (mp->m_flags & M_VLANTAG) { 3717 vtag = htole16(mp->m_pkthdr.ether_vtag); 3718 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3719 } 3720 3721 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 3722 vlan_macip_lens |= ip_hlen; 3723 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 3724 3725 /* ADV DTYPE TUCMD */ 3726 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3727 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3728 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 3729 3730 /* MSS L4LEN IDX */ 3731 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); 3732 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); 3733 /* 82575 needs the queue index added */ 3734 if (adapter->hw.mac.type == e1000_82575) 3735 mss_l4len_idx |= txr->me << 4; 3736 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3737 3738 TXD->seqnum_seed = htole32(0); 3739 3740 if (++ctxd == txr->num_desc) 3741 ctxd = 0; 3742 3743 txr->tx_avail--; 3744 txr->next_avail_desc = ctxd; 3745 *cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 3746 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 3747 *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT; 3748 ++txr->tso_tx; 3749 return (0); 3750 } 3751 3752 /********************************************************************* 3753 * 3754 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3755 * 3756 **********************************************************************/ 3757 3758 static int 3759 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 3760 u32 *cmd_type_len, u32 *olinfo_status) 3761 { 3762 struct e1000_adv_tx_context_desc *TXD; 3763 struct adapter *adapter = txr->adapter; 3764 struct ether_vlan_header *eh; 3765 struct ip *ip; 3766 struct ip6_hdr *ip6; 3767 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0; 3768 int ehdrlen, ip_hlen = 0; 3769 u16 etype; 3770 u8 ipproto = 0; 3771 int offload = TRUE; 3772 int ctxd = txr->next_avail_desc; 3773 u16 vtag = 0; 3774 3775 /* First check if TSO is to be used */ 3776 if (mp->m_pkthdr.csum_flags & CSUM_TSO) 3777 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status)); 3778 3779 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 3780 offload = FALSE; 3781 3782 /* Indicate the whole packet as payload when not doing TSO */ 3783 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT; 3784 3785 /* Now ready a context descriptor */ 3786 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3787 3788 /* 3789 ** In advanced descriptors the vlan tag must 3790 ** be placed into the context descriptor. Hence 3791 ** we need to make one even if not doing offloads. 3792 */ 3793 if (mp->m_flags & M_VLANTAG) { 3794 vtag = htole16(mp->m_pkthdr.ether_vtag); 3795 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3796 } else if (offload == FALSE) /* ... no offload to do */ 3797 return (0); 3798 3799 /* 3800 * Determine where frame payload starts. 3801 * Jump over vlan headers if already present, 3802 * helpful for QinQ too. 3803 */ 3804 eh = mtod(mp, struct ether_vlan_header *); 3805 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3806 etype = ntohs(eh->evl_proto); 3807 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3808 } else { 3809 etype = ntohs(eh->evl_encap_proto); 3810 ehdrlen = ETHER_HDR_LEN; 3811 } 3812 3813 /* Set the ether header length */ 3814 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 3815 3816 switch (etype) { 3817 case ETHERTYPE_IP: 3818 ip = (struct ip *)(mp->m_data + ehdrlen); 3819 ip_hlen = ip->ip_hl << 2; 3820 ipproto = ip->ip_p; 3821 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3822 break; 3823 case ETHERTYPE_IPV6: 3824 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3825 ip_hlen = sizeof(struct ip6_hdr); 3826 /* XXX-BZ this will go badly in case of ext hdrs. */ 3827 ipproto = ip6->ip6_nxt; 3828 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 3829 break; 3830 default: 3831 offload = FALSE; 3832 break; 3833 } 3834 3835 vlan_macip_lens |= ip_hlen; 3836 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3837 3838 switch (ipproto) { 3839 case IPPROTO_TCP: 3840 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 3841 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3842 break; 3843 case IPPROTO_UDP: 3844 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 3845 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; 3846 break; 3847 3848 #if __FreeBSD_version >= 800000 3849 case IPPROTO_SCTP: 3850 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 3851 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; 3852 break; 3853 #endif 3854 default: 3855 offload = FALSE; 3856 break; 3857 } 3858 3859 if (offload) /* For the TX descriptor setup */ 3860 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 3861 3862 /* 82575 needs the queue index added */ 3863 if (adapter->hw.mac.type == e1000_82575) 3864 mss_l4len_idx = txr->me << 4; 3865 3866 /* Now copy bits into descriptor */ 3867 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 3868 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 3869 TXD->seqnum_seed = htole32(0); 3870 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3871 3872 /* We've consumed the first desc, adjust counters */ 3873 if (++ctxd == txr->num_desc) 3874 ctxd = 0; 3875 txr->next_avail_desc = ctxd; 3876 --txr->tx_avail; 3877 3878 return (0); 3879 } 3880 3881 /********************************************************************** 3882 * 3883 * Examine each tx_buffer in the used queue. If the hardware is done 3884 * processing the packet then free associated resources. The 3885 * tx_buffer is put back on the free queue. 3886 * 3887 * TRUE return means there's work in the ring to clean, FALSE its empty. 3888 **********************************************************************/ 3889 static bool 3890 igb_txeof(struct tx_ring *txr) 3891 { 3892 struct adapter *adapter = txr->adapter; 3893 struct ifnet *ifp = adapter->ifp; 3894 u32 work, processed = 0; 3895 int limit = adapter->tx_process_limit; 3896 struct igb_tx_buf *buf; 3897 union e1000_adv_tx_desc *txd; 3898 3899 mtx_assert(&txr->tx_mtx, MA_OWNED); 3900 3901 #ifdef DEV_NETMAP 3902 if (netmap_tx_irq(ifp, txr->me)) 3903 return (FALSE); 3904 #endif /* DEV_NETMAP */ 3905 3906 if (txr->tx_avail == txr->num_desc) { 3907 txr->queue_status = IGB_QUEUE_IDLE; 3908 return FALSE; 3909 } 3910 3911 /* Get work starting point */ 3912 work = txr->next_to_clean; 3913 buf = &txr->tx_buffers[work]; 3914 txd = &txr->tx_base[work]; 3915 work -= txr->num_desc; /* The distance to ring end */ 3916 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3917 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3918 do { 3919 union e1000_adv_tx_desc *eop = buf->eop; 3920 if (eop == NULL) /* No work */ 3921 break; 3922 3923 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0) 3924 break; /* I/O not complete */ 3925 3926 if (buf->m_head) { 3927 txr->bytes += 3928 buf->m_head->m_pkthdr.len; 3929 bus_dmamap_sync(txr->txtag, 3930 buf->map, 3931 BUS_DMASYNC_POSTWRITE); 3932 bus_dmamap_unload(txr->txtag, 3933 buf->map); 3934 m_freem(buf->m_head); 3935 buf->m_head = NULL; 3936 } 3937 buf->eop = NULL; 3938 ++txr->tx_avail; 3939 3940 /* We clean the range if multi segment */ 3941 while (txd != eop) { 3942 ++txd; 3943 ++buf; 3944 ++work; 3945 /* wrap the ring? */ 3946 if (__predict_false(!work)) { 3947 work -= txr->num_desc; 3948 buf = txr->tx_buffers; 3949 txd = txr->tx_base; 3950 } 3951 if (buf->m_head) { 3952 txr->bytes += 3953 buf->m_head->m_pkthdr.len; 3954 bus_dmamap_sync(txr->txtag, 3955 buf->map, 3956 BUS_DMASYNC_POSTWRITE); 3957 bus_dmamap_unload(txr->txtag, 3958 buf->map); 3959 m_freem(buf->m_head); 3960 buf->m_head = NULL; 3961 } 3962 ++txr->tx_avail; 3963 buf->eop = NULL; 3964 3965 } 3966 ++txr->packets; 3967 ++processed; 3968 ++ifp->if_opackets; 3969 txr->watchdog_time = ticks; 3970 3971 /* Try the next packet */ 3972 ++txd; 3973 ++buf; 3974 ++work; 3975 /* reset with a wrap */ 3976 if (__predict_false(!work)) { 3977 work -= txr->num_desc; 3978 buf = txr->tx_buffers; 3979 txd = txr->tx_base; 3980 } 3981 prefetch(txd); 3982 } while (__predict_true(--limit)); 3983 3984 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3985 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3986 3987 work += txr->num_desc; 3988 txr->next_to_clean = work; 3989 3990 /* 3991 ** Watchdog calculation, we know there's 3992 ** work outstanding or the first return 3993 ** would have been taken, so none processed 3994 ** for too long indicates a hang. 3995 */ 3996 if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG)) 3997 txr->queue_status |= IGB_QUEUE_HUNG; 3998 3999 if (txr->tx_avail >= IGB_QUEUE_THRESHOLD) 4000 txr->queue_status &= ~IGB_QUEUE_DEPLETED; 4001 4002 if (txr->tx_avail == txr->num_desc) { 4003 txr->queue_status = IGB_QUEUE_IDLE; 4004 return (FALSE); 4005 } 4006 4007 return (TRUE); 4008 } 4009 4010 /********************************************************************* 4011 * 4012 * Refresh mbuf buffers for RX descriptor rings 4013 * - now keeps its own state so discards due to resource 4014 * exhaustion are unnecessary, if an mbuf cannot be obtained 4015 * it just returns, keeping its placeholder, thus it can simply 4016 * be recalled to try again. 4017 * 4018 **********************************************************************/ 4019 static void 4020 igb_refresh_mbufs(struct rx_ring *rxr, int limit) 4021 { 4022 struct adapter *adapter = rxr->adapter; 4023 bus_dma_segment_t hseg[1]; 4024 bus_dma_segment_t pseg[1]; 4025 struct igb_rx_buf *rxbuf; 4026 struct mbuf *mh, *mp; 4027 int i, j, nsegs, error; 4028 bool refreshed = FALSE; 4029 4030 i = j = rxr->next_to_refresh; 4031 /* 4032 ** Get one descriptor beyond 4033 ** our work mark to control 4034 ** the loop. 4035 */ 4036 if (++j == adapter->num_rx_desc) 4037 j = 0; 4038 4039 while (j != limit) { 4040 rxbuf = &rxr->rx_buffers[i]; 4041 /* No hdr mbuf used with header split off */ 4042 if (rxr->hdr_split == FALSE) 4043 goto no_split; 4044 if (rxbuf->m_head == NULL) { 4045 mh = m_gethdr(M_NOWAIT, MT_DATA); 4046 if (mh == NULL) 4047 goto update; 4048 } else 4049 mh = rxbuf->m_head; 4050 4051 mh->m_pkthdr.len = mh->m_len = MHLEN; 4052 mh->m_len = MHLEN; 4053 mh->m_flags |= M_PKTHDR; 4054 /* Get the memory mapping */ 4055 error = bus_dmamap_load_mbuf_sg(rxr->htag, 4056 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 4057 if (error != 0) { 4058 printf("Refresh mbufs: hdr dmamap load" 4059 " failure - %d\n", error); 4060 m_free(mh); 4061 rxbuf->m_head = NULL; 4062 goto update; 4063 } 4064 rxbuf->m_head = mh; 4065 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 4066 BUS_DMASYNC_PREREAD); 4067 rxr->rx_base[i].read.hdr_addr = 4068 htole64(hseg[0].ds_addr); 4069 no_split: 4070 if (rxbuf->m_pack == NULL) { 4071 mp = m_getjcl(M_NOWAIT, MT_DATA, 4072 M_PKTHDR, adapter->rx_mbuf_sz); 4073 if (mp == NULL) 4074 goto update; 4075 } else 4076 mp = rxbuf->m_pack; 4077 4078 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 4079 /* Get the memory mapping */ 4080 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 4081 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 4082 if (error != 0) { 4083 printf("Refresh mbufs: payload dmamap load" 4084 " failure - %d\n", error); 4085 m_free(mp); 4086 rxbuf->m_pack = NULL; 4087 goto update; 4088 } 4089 rxbuf->m_pack = mp; 4090 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 4091 BUS_DMASYNC_PREREAD); 4092 rxr->rx_base[i].read.pkt_addr = 4093 htole64(pseg[0].ds_addr); 4094 refreshed = TRUE; /* I feel wefreshed :) */ 4095 4096 i = j; /* our next is precalculated */ 4097 rxr->next_to_refresh = i; 4098 if (++j == adapter->num_rx_desc) 4099 j = 0; 4100 } 4101 update: 4102 if (refreshed) /* update tail */ 4103 E1000_WRITE_REG(&adapter->hw, 4104 E1000_RDT(rxr->me), rxr->next_to_refresh); 4105 return; 4106 } 4107 4108 4109 /********************************************************************* 4110 * 4111 * Allocate memory for rx_buffer structures. Since we use one 4112 * rx_buffer per received packet, the maximum number of rx_buffer's 4113 * that we'll need is equal to the number of receive descriptors 4114 * that we've allocated. 4115 * 4116 **********************************************************************/ 4117 static int 4118 igb_allocate_receive_buffers(struct rx_ring *rxr) 4119 { 4120 struct adapter *adapter = rxr->adapter; 4121 device_t dev = adapter->dev; 4122 struct igb_rx_buf *rxbuf; 4123 int i, bsize, error; 4124 4125 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc; 4126 if (!(rxr->rx_buffers = 4127 (struct igb_rx_buf *) malloc(bsize, 4128 M_DEVBUF, M_NOWAIT | M_ZERO))) { 4129 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 4130 error = ENOMEM; 4131 goto fail; 4132 } 4133 4134 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 4135 1, 0, /* alignment, bounds */ 4136 BUS_SPACE_MAXADDR, /* lowaddr */ 4137 BUS_SPACE_MAXADDR, /* highaddr */ 4138 NULL, NULL, /* filter, filterarg */ 4139 MSIZE, /* maxsize */ 4140 1, /* nsegments */ 4141 MSIZE, /* maxsegsize */ 4142 0, /* flags */ 4143 NULL, /* lockfunc */ 4144 NULL, /* lockfuncarg */ 4145 &rxr->htag))) { 4146 device_printf(dev, "Unable to create RX DMA tag\n"); 4147 goto fail; 4148 } 4149 4150 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 4151 1, 0, /* alignment, bounds */ 4152 BUS_SPACE_MAXADDR, /* lowaddr */ 4153 BUS_SPACE_MAXADDR, /* highaddr */ 4154 NULL, NULL, /* filter, filterarg */ 4155 MJUM9BYTES, /* maxsize */ 4156 1, /* nsegments */ 4157 MJUM9BYTES, /* maxsegsize */ 4158 0, /* flags */ 4159 NULL, /* lockfunc */ 4160 NULL, /* lockfuncarg */ 4161 &rxr->ptag))) { 4162 device_printf(dev, "Unable to create RX payload DMA tag\n"); 4163 goto fail; 4164 } 4165 4166 for (i = 0; i < adapter->num_rx_desc; i++) { 4167 rxbuf = &rxr->rx_buffers[i]; 4168 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap); 4169 if (error) { 4170 device_printf(dev, 4171 "Unable to create RX head DMA maps\n"); 4172 goto fail; 4173 } 4174 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap); 4175 if (error) { 4176 device_printf(dev, 4177 "Unable to create RX packet DMA maps\n"); 4178 goto fail; 4179 } 4180 } 4181 4182 return (0); 4183 4184 fail: 4185 /* Frees all, but can handle partial completion */ 4186 igb_free_receive_structures(adapter); 4187 return (error); 4188 } 4189 4190 4191 static void 4192 igb_free_receive_ring(struct rx_ring *rxr) 4193 { 4194 struct adapter *adapter = rxr->adapter; 4195 struct igb_rx_buf *rxbuf; 4196 4197 4198 for (int i = 0; i < adapter->num_rx_desc; i++) { 4199 rxbuf = &rxr->rx_buffers[i]; 4200 if (rxbuf->m_head != NULL) { 4201 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 4202 BUS_DMASYNC_POSTREAD); 4203 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 4204 rxbuf->m_head->m_flags |= M_PKTHDR; 4205 m_freem(rxbuf->m_head); 4206 } 4207 if (rxbuf->m_pack != NULL) { 4208 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 4209 BUS_DMASYNC_POSTREAD); 4210 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 4211 rxbuf->m_pack->m_flags |= M_PKTHDR; 4212 m_freem(rxbuf->m_pack); 4213 } 4214 rxbuf->m_head = NULL; 4215 rxbuf->m_pack = NULL; 4216 } 4217 } 4218 4219 4220 /********************************************************************* 4221 * 4222 * Initialize a receive ring and its buffers. 4223 * 4224 **********************************************************************/ 4225 static int 4226 igb_setup_receive_ring(struct rx_ring *rxr) 4227 { 4228 struct adapter *adapter; 4229 struct ifnet *ifp; 4230 device_t dev; 4231 struct igb_rx_buf *rxbuf; 4232 bus_dma_segment_t pseg[1], hseg[1]; 4233 struct lro_ctrl *lro = &rxr->lro; 4234 int rsize, nsegs, error = 0; 4235 #ifdef DEV_NETMAP 4236 struct netmap_adapter *na = NA(rxr->adapter->ifp); 4237 struct netmap_slot *slot; 4238 #endif /* DEV_NETMAP */ 4239 4240 adapter = rxr->adapter; 4241 dev = adapter->dev; 4242 ifp = adapter->ifp; 4243 4244 /* Clear the ring contents */ 4245 IGB_RX_LOCK(rxr); 4246 #ifdef DEV_NETMAP 4247 slot = netmap_reset(na, NR_RX, rxr->me, 0); 4248 #endif /* DEV_NETMAP */ 4249 rsize = roundup2(adapter->num_rx_desc * 4250 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 4251 bzero((void *)rxr->rx_base, rsize); 4252 4253 /* 4254 ** Free current RX buffer structures and their mbufs 4255 */ 4256 igb_free_receive_ring(rxr); 4257 4258 /* Configure for header split? */ 4259 if (igb_header_split) 4260 rxr->hdr_split = TRUE; 4261 4262 /* Now replenish the ring mbufs */ 4263 for (int j = 0; j < adapter->num_rx_desc; ++j) { 4264 struct mbuf *mh, *mp; 4265 4266 rxbuf = &rxr->rx_buffers[j]; 4267 #ifdef DEV_NETMAP 4268 if (slot) { 4269 /* slot sj is mapped to the i-th NIC-ring entry */ 4270 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); 4271 uint64_t paddr; 4272 void *addr; 4273 4274 addr = PNMB(na, slot + sj, &paddr); 4275 netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); 4276 /* Update descriptor */ 4277 rxr->rx_base[j].read.pkt_addr = htole64(paddr); 4278 continue; 4279 } 4280 #endif /* DEV_NETMAP */ 4281 if (rxr->hdr_split == FALSE) 4282 goto skip_head; 4283 4284 /* First the header */ 4285 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); 4286 if (rxbuf->m_head == NULL) { 4287 error = ENOBUFS; 4288 goto fail; 4289 } 4290 m_adj(rxbuf->m_head, ETHER_ALIGN); 4291 mh = rxbuf->m_head; 4292 mh->m_len = mh->m_pkthdr.len = MHLEN; 4293 mh->m_flags |= M_PKTHDR; 4294 /* Get the memory mapping */ 4295 error = bus_dmamap_load_mbuf_sg(rxr->htag, 4296 rxbuf->hmap, rxbuf->m_head, hseg, 4297 &nsegs, BUS_DMA_NOWAIT); 4298 if (error != 0) /* Nothing elegant to do here */ 4299 goto fail; 4300 bus_dmamap_sync(rxr->htag, 4301 rxbuf->hmap, BUS_DMASYNC_PREREAD); 4302 /* Update descriptor */ 4303 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 4304 4305 skip_head: 4306 /* Now the payload cluster */ 4307 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, 4308 M_PKTHDR, adapter->rx_mbuf_sz); 4309 if (rxbuf->m_pack == NULL) { 4310 error = ENOBUFS; 4311 goto fail; 4312 } 4313 mp = rxbuf->m_pack; 4314 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 4315 /* Get the memory mapping */ 4316 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 4317 rxbuf->pmap, mp, pseg, 4318 &nsegs, BUS_DMA_NOWAIT); 4319 if (error != 0) 4320 goto fail; 4321 bus_dmamap_sync(rxr->ptag, 4322 rxbuf->pmap, BUS_DMASYNC_PREREAD); 4323 /* Update descriptor */ 4324 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 4325 } 4326 4327 /* Setup our descriptor indices */ 4328 rxr->next_to_check = 0; 4329 rxr->next_to_refresh = adapter->num_rx_desc - 1; 4330 rxr->lro_enabled = FALSE; 4331 rxr->rx_split_packets = 0; 4332 rxr->rx_bytes = 0; 4333 4334 rxr->fmp = NULL; 4335 rxr->lmp = NULL; 4336 4337 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4338 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4339 4340 /* 4341 ** Now set up the LRO interface, we 4342 ** also only do head split when LRO 4343 ** is enabled, since so often they 4344 ** are undesireable in similar setups. 4345 */ 4346 if (ifp->if_capenable & IFCAP_LRO) { 4347 error = tcp_lro_init(lro); 4348 if (error) { 4349 device_printf(dev, "LRO Initialization failed!\n"); 4350 goto fail; 4351 } 4352 INIT_DEBUGOUT("RX LRO Initialized\n"); 4353 rxr->lro_enabled = TRUE; 4354 lro->ifp = adapter->ifp; 4355 } 4356 4357 IGB_RX_UNLOCK(rxr); 4358 return (0); 4359 4360 fail: 4361 igb_free_receive_ring(rxr); 4362 IGB_RX_UNLOCK(rxr); 4363 return (error); 4364 } 4365 4366 4367 /********************************************************************* 4368 * 4369 * Initialize all receive rings. 4370 * 4371 **********************************************************************/ 4372 static int 4373 igb_setup_receive_structures(struct adapter *adapter) 4374 { 4375 struct rx_ring *rxr = adapter->rx_rings; 4376 int i; 4377 4378 for (i = 0; i < adapter->num_queues; i++, rxr++) 4379 if (igb_setup_receive_ring(rxr)) 4380 goto fail; 4381 4382 return (0); 4383 fail: 4384 /* 4385 * Free RX buffers allocated so far, we will only handle 4386 * the rings that completed, the failing case will have 4387 * cleaned up for itself. 'i' is the endpoint. 4388 */ 4389 for (int j = 0; j < i; ++j) { 4390 rxr = &adapter->rx_rings[j]; 4391 IGB_RX_LOCK(rxr); 4392 igb_free_receive_ring(rxr); 4393 IGB_RX_UNLOCK(rxr); 4394 } 4395 4396 return (ENOBUFS); 4397 } 4398 4399 /********************************************************************* 4400 * 4401 * Enable receive unit. 4402 * 4403 **********************************************************************/ 4404 static void 4405 igb_initialize_receive_units(struct adapter *adapter) 4406 { 4407 struct rx_ring *rxr = adapter->rx_rings; 4408 struct ifnet *ifp = adapter->ifp; 4409 struct e1000_hw *hw = &adapter->hw; 4410 u32 rctl, rxcsum, psize, srrctl = 0; 4411 4412 INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); 4413 4414 /* 4415 * Make sure receives are disabled while setting 4416 * up the descriptor ring 4417 */ 4418 rctl = E1000_READ_REG(hw, E1000_RCTL); 4419 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4420 4421 /* 4422 ** Set up for header split 4423 */ 4424 if (igb_header_split) { 4425 /* Use a standard mbuf for the header */ 4426 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 4427 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4428 } else 4429 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 4430 4431 /* 4432 ** Set up for jumbo frames 4433 */ 4434 if (ifp->if_mtu > ETHERMTU) { 4435 rctl |= E1000_RCTL_LPE; 4436 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) { 4437 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 4438 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 4439 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) { 4440 srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 4441 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; 4442 } 4443 /* Set maximum packet len */ 4444 psize = adapter->max_frame_size; 4445 /* are we on a vlan? */ 4446 if (adapter->ifp->if_vlantrunk != NULL) 4447 psize += VLAN_TAG_SIZE; 4448 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); 4449 } else { 4450 rctl &= ~E1000_RCTL_LPE; 4451 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 4452 rctl |= E1000_RCTL_SZ_2048; 4453 } 4454 4455 /* 4456 * If TX flow control is disabled and there's >1 queue defined, 4457 * enable DROP. 4458 * 4459 * This drops frames rather than hanging the RX MAC for all queues. 4460 */ 4461 if ((adapter->num_queues > 1) && 4462 (adapter->fc == e1000_fc_none || 4463 adapter->fc == e1000_fc_rx_pause)) { 4464 srrctl |= E1000_SRRCTL_DROP_EN; 4465 } 4466 4467 /* Setup the Base and Length of the Rx Descriptor Rings */ 4468 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4469 u64 bus_addr = rxr->rxdma.dma_paddr; 4470 u32 rxdctl; 4471 4472 E1000_WRITE_REG(hw, E1000_RDLEN(i), 4473 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4474 E1000_WRITE_REG(hw, E1000_RDBAH(i), 4475 (uint32_t)(bus_addr >> 32)); 4476 E1000_WRITE_REG(hw, E1000_RDBAL(i), 4477 (uint32_t)bus_addr); 4478 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); 4479 /* Enable this Queue */ 4480 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 4481 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 4482 rxdctl &= 0xFFF00000; 4483 rxdctl |= IGB_RX_PTHRESH; 4484 rxdctl |= IGB_RX_HTHRESH << 8; 4485 rxdctl |= IGB_RX_WTHRESH << 16; 4486 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 4487 } 4488 4489 /* 4490 ** Setup for RX MultiQueue 4491 */ 4492 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 4493 if (adapter->num_queues >1) { 4494 u32 random[10], mrqc, shift = 0; 4495 union igb_reta { 4496 u32 dword; 4497 u8 bytes[4]; 4498 } reta; 4499 4500 arc4rand(&random, sizeof(random), 0); 4501 if (adapter->hw.mac.type == e1000_82575) 4502 shift = 6; 4503 /* Warning FM follows */ 4504 for (int i = 0; i < 128; i++) { 4505 reta.bytes[i & 3] = 4506 (i % adapter->num_queues) << shift; 4507 if ((i & 3) == 3) 4508 E1000_WRITE_REG(hw, 4509 E1000_RETA(i >> 2), reta.dword); 4510 } 4511 /* Now fill in hash table */ 4512 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 4513 for (int i = 0; i < 10; i++) 4514 E1000_WRITE_REG_ARRAY(hw, 4515 E1000_RSSRK(0), i, random[i]); 4516 4517 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 4518 E1000_MRQC_RSS_FIELD_IPV4_TCP); 4519 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 4520 E1000_MRQC_RSS_FIELD_IPV6_TCP); 4521 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 4522 E1000_MRQC_RSS_FIELD_IPV6_UDP); 4523 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 4524 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 4525 4526 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 4527 4528 /* 4529 ** NOTE: Receive Full-Packet Checksum Offload 4530 ** is mutually exclusive with Multiqueue. However 4531 ** this is not the same as TCP/IP checksums which 4532 ** still work. 4533 */ 4534 rxcsum |= E1000_RXCSUM_PCSD; 4535 #if __FreeBSD_version >= 800000 4536 /* For SCTP Offload */ 4537 if (((hw->mac.type == e1000_82576) || 4538 (hw->mac.type == e1000_82580)) && 4539 (ifp->if_capenable & IFCAP_RXCSUM)) 4540 rxcsum |= E1000_RXCSUM_CRCOFL; 4541 #endif 4542 } else { 4543 /* Non RSS setup */ 4544 if (ifp->if_capenable & IFCAP_RXCSUM) { 4545 rxcsum |= E1000_RXCSUM_IPPCSE; 4546 #if __FreeBSD_version >= 800000 4547 if ((adapter->hw.mac.type == e1000_82576) || 4548 (adapter->hw.mac.type == e1000_82580)) 4549 rxcsum |= E1000_RXCSUM_CRCOFL; 4550 #endif 4551 } else 4552 rxcsum &= ~E1000_RXCSUM_TUOFL; 4553 } 4554 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 4555 4556 /* Setup the Receive Control Register */ 4557 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4558 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4559 E1000_RCTL_RDMTS_HALF | 4560 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4561 /* Strip CRC bytes. */ 4562 rctl |= E1000_RCTL_SECRC; 4563 /* Make sure VLAN Filters are off */ 4564 rctl &= ~E1000_RCTL_VFE; 4565 /* Don't store bad packets */ 4566 rctl &= ~E1000_RCTL_SBP; 4567 4568 /* Enable Receives */ 4569 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4570 4571 /* 4572 * Setup the HW Rx Head and Tail Descriptor Pointers 4573 * - needs to be after enable 4574 */ 4575 for (int i = 0; i < adapter->num_queues; i++) { 4576 rxr = &adapter->rx_rings[i]; 4577 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check); 4578 #ifdef DEV_NETMAP 4579 /* 4580 * an init() while a netmap client is active must 4581 * preserve the rx buffers passed to userspace. 4582 * In this driver it means we adjust RDT to 4583 * something different from next_to_refresh 4584 * (which is not used in netmap mode). 4585 */ 4586 if (ifp->if_capenable & IFCAP_NETMAP) { 4587 struct netmap_adapter *na = NA(adapter->ifp); 4588 struct netmap_kring *kring = &na->rx_rings[i]; 4589 int t = rxr->next_to_refresh - nm_kr_rxspace(kring); 4590 4591 if (t >= adapter->num_rx_desc) 4592 t -= adapter->num_rx_desc; 4593 else if (t < 0) 4594 t += adapter->num_rx_desc; 4595 E1000_WRITE_REG(hw, E1000_RDT(i), t); 4596 } else 4597 #endif /* DEV_NETMAP */ 4598 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh); 4599 } 4600 return; 4601 } 4602 4603 /********************************************************************* 4604 * 4605 * Free receive rings. 4606 * 4607 **********************************************************************/ 4608 static void 4609 igb_free_receive_structures(struct adapter *adapter) 4610 { 4611 struct rx_ring *rxr = adapter->rx_rings; 4612 4613 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4614 struct lro_ctrl *lro = &rxr->lro; 4615 igb_free_receive_buffers(rxr); 4616 tcp_lro_free(lro); 4617 igb_dma_free(adapter, &rxr->rxdma); 4618 } 4619 4620 free(adapter->rx_rings, M_DEVBUF); 4621 } 4622 4623 /********************************************************************* 4624 * 4625 * Free receive ring data structures. 4626 * 4627 **********************************************************************/ 4628 static void 4629 igb_free_receive_buffers(struct rx_ring *rxr) 4630 { 4631 struct adapter *adapter = rxr->adapter; 4632 struct igb_rx_buf *rxbuf; 4633 int i; 4634 4635 INIT_DEBUGOUT("free_receive_structures: begin"); 4636 4637 /* Cleanup any existing buffers */ 4638 if (rxr->rx_buffers != NULL) { 4639 for (i = 0; i < adapter->num_rx_desc; i++) { 4640 rxbuf = &rxr->rx_buffers[i]; 4641 if (rxbuf->m_head != NULL) { 4642 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 4643 BUS_DMASYNC_POSTREAD); 4644 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 4645 rxbuf->m_head->m_flags |= M_PKTHDR; 4646 m_freem(rxbuf->m_head); 4647 } 4648 if (rxbuf->m_pack != NULL) { 4649 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 4650 BUS_DMASYNC_POSTREAD); 4651 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 4652 rxbuf->m_pack->m_flags |= M_PKTHDR; 4653 m_freem(rxbuf->m_pack); 4654 } 4655 rxbuf->m_head = NULL; 4656 rxbuf->m_pack = NULL; 4657 if (rxbuf->hmap != NULL) { 4658 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 4659 rxbuf->hmap = NULL; 4660 } 4661 if (rxbuf->pmap != NULL) { 4662 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4663 rxbuf->pmap = NULL; 4664 } 4665 } 4666 if (rxr->rx_buffers != NULL) { 4667 free(rxr->rx_buffers, M_DEVBUF); 4668 rxr->rx_buffers = NULL; 4669 } 4670 } 4671 4672 if (rxr->htag != NULL) { 4673 bus_dma_tag_destroy(rxr->htag); 4674 rxr->htag = NULL; 4675 } 4676 if (rxr->ptag != NULL) { 4677 bus_dma_tag_destroy(rxr->ptag); 4678 rxr->ptag = NULL; 4679 } 4680 } 4681 4682 static __inline void 4683 igb_rx_discard(struct rx_ring *rxr, int i) 4684 { 4685 struct igb_rx_buf *rbuf; 4686 4687 rbuf = &rxr->rx_buffers[i]; 4688 4689 /* Partially received? Free the chain */ 4690 if (rxr->fmp != NULL) { 4691 rxr->fmp->m_flags |= M_PKTHDR; 4692 m_freem(rxr->fmp); 4693 rxr->fmp = NULL; 4694 rxr->lmp = NULL; 4695 } 4696 4697 /* 4698 ** With advanced descriptors the writeback 4699 ** clobbers the buffer addrs, so its easier 4700 ** to just free the existing mbufs and take 4701 ** the normal refresh path to get new buffers 4702 ** and mapping. 4703 */ 4704 if (rbuf->m_head) { 4705 m_free(rbuf->m_head); 4706 rbuf->m_head = NULL; 4707 bus_dmamap_unload(rxr->htag, rbuf->hmap); 4708 } 4709 4710 if (rbuf->m_pack) { 4711 m_free(rbuf->m_pack); 4712 rbuf->m_pack = NULL; 4713 bus_dmamap_unload(rxr->ptag, rbuf->pmap); 4714 } 4715 4716 return; 4717 } 4718 4719 static __inline void 4720 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4721 { 4722 4723 /* 4724 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4725 * should be computed by hardware. Also it should not have VLAN tag in 4726 * ethernet header. 4727 */ 4728 if (rxr->lro_enabled && 4729 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4730 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 4731 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) == 4732 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) && 4733 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4734 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4735 /* 4736 * Send to the stack if: 4737 ** - LRO not enabled, or 4738 ** - no LRO resources, or 4739 ** - lro enqueue fails 4740 */ 4741 if (rxr->lro.lro_cnt != 0) 4742 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4743 return; 4744 } 4745 IGB_RX_UNLOCK(rxr); 4746 (*ifp->if_input)(ifp, m); 4747 IGB_RX_LOCK(rxr); 4748 } 4749 4750 /********************************************************************* 4751 * 4752 * This routine executes in interrupt context. It replenishes 4753 * the mbufs in the descriptor and sends data which has been 4754 * dma'ed into host memory to upper layer. 4755 * 4756 * We loop at most count times if count is > 0, or until done if 4757 * count < 0. 4758 * 4759 * Return TRUE if more to clean, FALSE otherwise 4760 *********************************************************************/ 4761 static bool 4762 igb_rxeof(struct igb_queue *que, int count, int *done) 4763 { 4764 struct adapter *adapter = que->adapter; 4765 struct rx_ring *rxr = que->rxr; 4766 struct ifnet *ifp = adapter->ifp; 4767 struct lro_ctrl *lro = &rxr->lro; 4768 struct lro_entry *queued; 4769 int i, processed = 0, rxdone = 0; 4770 u32 ptype, staterr = 0; 4771 union e1000_adv_rx_desc *cur; 4772 4773 IGB_RX_LOCK(rxr); 4774 /* Sync the ring. */ 4775 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4776 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4777 4778 #ifdef DEV_NETMAP 4779 if (netmap_rx_irq(ifp, rxr->me, &processed)) { 4780 IGB_RX_UNLOCK(rxr); 4781 return (FALSE); 4782 } 4783 #endif /* DEV_NETMAP */ 4784 4785 /* Main clean loop */ 4786 for (i = rxr->next_to_check; count != 0;) { 4787 struct mbuf *sendmp, *mh, *mp; 4788 struct igb_rx_buf *rxbuf; 4789 u16 hlen, plen, hdr, vtag; 4790 bool eop = FALSE; 4791 4792 cur = &rxr->rx_base[i]; 4793 staterr = le32toh(cur->wb.upper.status_error); 4794 if ((staterr & E1000_RXD_STAT_DD) == 0) 4795 break; 4796 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4797 break; 4798 count--; 4799 sendmp = mh = mp = NULL; 4800 cur->wb.upper.status_error = 0; 4801 rxbuf = &rxr->rx_buffers[i]; 4802 plen = le16toh(cur->wb.upper.length); 4803 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; 4804 if (((adapter->hw.mac.type == e1000_i350) || 4805 (adapter->hw.mac.type == e1000_i354)) && 4806 (staterr & E1000_RXDEXT_STATERR_LB)) 4807 vtag = be16toh(cur->wb.upper.vlan); 4808 else 4809 vtag = le16toh(cur->wb.upper.vlan); 4810 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4811 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); 4812 4813 /* 4814 * Free the frame (all segments) if we're at EOP and 4815 * it's an error. 4816 * 4817 * The datasheet states that EOP + status is only valid for 4818 * the final segment in a multi-segment frame. 4819 */ 4820 if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) { 4821 adapter->dropped_pkts++; 4822 ++rxr->rx_discarded; 4823 igb_rx_discard(rxr, i); 4824 goto next_desc; 4825 } 4826 4827 /* 4828 ** The way the hardware is configured to 4829 ** split, it will ONLY use the header buffer 4830 ** when header split is enabled, otherwise we 4831 ** get normal behavior, ie, both header and 4832 ** payload are DMA'd into the payload buffer. 4833 ** 4834 ** The fmp test is to catch the case where a 4835 ** packet spans multiple descriptors, in that 4836 ** case only the first header is valid. 4837 */ 4838 if (rxr->hdr_split && rxr->fmp == NULL) { 4839 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 4840 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> 4841 E1000_RXDADV_HDRBUFLEN_SHIFT; 4842 if (hlen > IGB_HDR_BUF) 4843 hlen = IGB_HDR_BUF; 4844 mh = rxr->rx_buffers[i].m_head; 4845 mh->m_len = hlen; 4846 /* clear buf pointer for refresh */ 4847 rxbuf->m_head = NULL; 4848 /* 4849 ** Get the payload length, this 4850 ** could be zero if its a small 4851 ** packet. 4852 */ 4853 if (plen > 0) { 4854 mp = rxr->rx_buffers[i].m_pack; 4855 mp->m_len = plen; 4856 mh->m_next = mp; 4857 /* clear buf pointer */ 4858 rxbuf->m_pack = NULL; 4859 rxr->rx_split_packets++; 4860 } 4861 } else { 4862 /* 4863 ** Either no header split, or a 4864 ** secondary piece of a fragmented 4865 ** split packet. 4866 */ 4867 mh = rxr->rx_buffers[i].m_pack; 4868 mh->m_len = plen; 4869 /* clear buf info for refresh */ 4870 rxbuf->m_pack = NULL; 4871 } 4872 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 4873 4874 ++processed; /* So we know when to refresh */ 4875 4876 /* Initial frame - setup */ 4877 if (rxr->fmp == NULL) { 4878 mh->m_pkthdr.len = mh->m_len; 4879 /* Save the head of the chain */ 4880 rxr->fmp = mh; 4881 rxr->lmp = mh; 4882 if (mp != NULL) { 4883 /* Add payload if split */ 4884 mh->m_pkthdr.len += mp->m_len; 4885 rxr->lmp = mh->m_next; 4886 } 4887 } else { 4888 /* Chain mbuf's together */ 4889 rxr->lmp->m_next = mh; 4890 rxr->lmp = rxr->lmp->m_next; 4891 rxr->fmp->m_pkthdr.len += mh->m_len; 4892 } 4893 4894 if (eop) { 4895 rxr->fmp->m_pkthdr.rcvif = ifp; 4896 ifp->if_ipackets++; 4897 rxr->rx_packets++; 4898 /* capture data for AIM */ 4899 rxr->packets++; 4900 rxr->bytes += rxr->fmp->m_pkthdr.len; 4901 rxr->rx_bytes += rxr->fmp->m_pkthdr.len; 4902 4903 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 4904 igb_rx_checksum(staterr, rxr->fmp, ptype); 4905 4906 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4907 (staterr & E1000_RXD_STAT_VP) != 0) { 4908 rxr->fmp->m_pkthdr.ether_vtag = vtag; 4909 rxr->fmp->m_flags |= M_VLANTAG; 4910 } 4911 4912 /* 4913 * In case of multiqueue, we have RXCSUM.PCSD bit set 4914 * and never cleared. This means we have RSS hash 4915 * available to be used. 4916 */ 4917 if (adapter->num_queues > 1) { 4918 rxr->fmp->m_pkthdr.flowid = 4919 le32toh(cur->wb.lower.hi_dword.rss); 4920 /* 4921 * Full RSS support is not avilable in 4922 * FreeBSD 10 so setting the hash type to 4923 * OPAQUE. 4924 */ 4925 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE); 4926 } else { 4927 #ifndef IGB_LEGACY_TX 4928 rxr->fmp->m_pkthdr.flowid = que->msix; 4929 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE); 4930 #endif 4931 } 4932 sendmp = rxr->fmp; 4933 /* Make sure to set M_PKTHDR. */ 4934 sendmp->m_flags |= M_PKTHDR; 4935 rxr->fmp = NULL; 4936 rxr->lmp = NULL; 4937 } 4938 4939 next_desc: 4940 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4941 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4942 4943 /* Advance our pointers to the next descriptor. */ 4944 if (++i == adapter->num_rx_desc) 4945 i = 0; 4946 /* 4947 ** Send to the stack or LRO 4948 */ 4949 if (sendmp != NULL) { 4950 rxr->next_to_check = i; 4951 igb_rx_input(rxr, ifp, sendmp, ptype); 4952 i = rxr->next_to_check; 4953 rxdone++; 4954 } 4955 4956 /* Every 8 descriptors we go to refresh mbufs */ 4957 if (processed == 8) { 4958 igb_refresh_mbufs(rxr, i); 4959 processed = 0; 4960 } 4961 } 4962 4963 /* Catch any remainders */ 4964 if (igb_rx_unrefreshed(rxr)) 4965 igb_refresh_mbufs(rxr, i); 4966 4967 rxr->next_to_check = i; 4968 4969 /* 4970 * Flush any outstanding LRO work 4971 */ 4972 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4973 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4974 tcp_lro_flush(lro, queued); 4975 } 4976 4977 if (done != NULL) 4978 *done += rxdone; 4979 4980 IGB_RX_UNLOCK(rxr); 4981 return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE); 4982 } 4983 4984 /********************************************************************* 4985 * 4986 * Verify that the hardware indicated that the checksum is valid. 4987 * Inform the stack about the status of checksum so that stack 4988 * doesn't spend time verifying the checksum. 4989 * 4990 *********************************************************************/ 4991 static void 4992 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype) 4993 { 4994 u16 status = (u16)staterr; 4995 u8 errors = (u8) (staterr >> 24); 4996 int sctp; 4997 4998 /* Ignore Checksum bit is set */ 4999 if (status & E1000_RXD_STAT_IXSM) { 5000 mp->m_pkthdr.csum_flags = 0; 5001 return; 5002 } 5003 5004 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 5005 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) 5006 sctp = 1; 5007 else 5008 sctp = 0; 5009 if (status & E1000_RXD_STAT_IPCS) { 5010 /* Did it pass? */ 5011 if (!(errors & E1000_RXD_ERR_IPE)) { 5012 /* IP Checksum Good */ 5013 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 5014 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 5015 } else 5016 mp->m_pkthdr.csum_flags = 0; 5017 } 5018 5019 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { 5020 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 5021 #if __FreeBSD_version >= 800000 5022 if (sctp) /* reassign */ 5023 type = CSUM_SCTP_VALID; 5024 #endif 5025 /* Did it pass? */ 5026 if (!(errors & E1000_RXD_ERR_TCPE)) { 5027 mp->m_pkthdr.csum_flags |= type; 5028 if (sctp == 0) 5029 mp->m_pkthdr.csum_data = htons(0xffff); 5030 } 5031 } 5032 return; 5033 } 5034 5035 /* 5036 * This routine is run via an vlan 5037 * config EVENT 5038 */ 5039 static void 5040 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 5041 { 5042 struct adapter *adapter = ifp->if_softc; 5043 u32 index, bit; 5044 5045 if (ifp->if_softc != arg) /* Not our event */ 5046 return; 5047 5048 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 5049 return; 5050 5051 IGB_CORE_LOCK(adapter); 5052 index = (vtag >> 5) & 0x7F; 5053 bit = vtag & 0x1F; 5054 adapter->shadow_vfta[index] |= (1 << bit); 5055 ++adapter->num_vlans; 5056 /* Change hw filter setting */ 5057 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 5058 igb_setup_vlan_hw_support(adapter); 5059 IGB_CORE_UNLOCK(adapter); 5060 } 5061 5062 /* 5063 * This routine is run via an vlan 5064 * unconfig EVENT 5065 */ 5066 static void 5067 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 5068 { 5069 struct adapter *adapter = ifp->if_softc; 5070 u32 index, bit; 5071 5072 if (ifp->if_softc != arg) 5073 return; 5074 5075 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 5076 return; 5077 5078 IGB_CORE_LOCK(adapter); 5079 index = (vtag >> 5) & 0x7F; 5080 bit = vtag & 0x1F; 5081 adapter->shadow_vfta[index] &= ~(1 << bit); 5082 --adapter->num_vlans; 5083 /* Change hw filter setting */ 5084 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 5085 igb_setup_vlan_hw_support(adapter); 5086 IGB_CORE_UNLOCK(adapter); 5087 } 5088 5089 static void 5090 igb_setup_vlan_hw_support(struct adapter *adapter) 5091 { 5092 struct e1000_hw *hw = &adapter->hw; 5093 struct ifnet *ifp = adapter->ifp; 5094 u32 reg; 5095 5096 if (adapter->vf_ifp) { 5097 e1000_rlpml_set_vf(hw, 5098 adapter->max_frame_size + VLAN_TAG_SIZE); 5099 return; 5100 } 5101 5102 reg = E1000_READ_REG(hw, E1000_CTRL); 5103 reg |= E1000_CTRL_VME; 5104 E1000_WRITE_REG(hw, E1000_CTRL, reg); 5105 5106 /* Enable the Filter Table */ 5107 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 5108 reg = E1000_READ_REG(hw, E1000_RCTL); 5109 reg &= ~E1000_RCTL_CFIEN; 5110 reg |= E1000_RCTL_VFE; 5111 E1000_WRITE_REG(hw, E1000_RCTL, reg); 5112 } 5113 5114 /* Update the frame size */ 5115 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 5116 adapter->max_frame_size + VLAN_TAG_SIZE); 5117 5118 /* Don't bother with table if no vlans */ 5119 if ((adapter->num_vlans == 0) || 5120 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) 5121 return; 5122 /* 5123 ** A soft reset zero's out the VFTA, so 5124 ** we need to repopulate it now. 5125 */ 5126 for (int i = 0; i < IGB_VFTA_SIZE; i++) 5127 if (adapter->shadow_vfta[i] != 0) { 5128 if (adapter->vf_ifp) 5129 e1000_vfta_set_vf(hw, 5130 adapter->shadow_vfta[i], TRUE); 5131 else 5132 e1000_write_vfta(hw, 5133 i, adapter->shadow_vfta[i]); 5134 } 5135 } 5136 5137 static void 5138 igb_enable_intr(struct adapter *adapter) 5139 { 5140 /* With RSS set up what to auto clear */ 5141 if (adapter->msix_mem) { 5142 u32 mask = (adapter->que_mask | adapter->link_mask); 5143 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask); 5144 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask); 5145 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask); 5146 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 5147 E1000_IMS_LSC); 5148 } else { 5149 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 5150 IMS_ENABLE_MASK); 5151 } 5152 E1000_WRITE_FLUSH(&adapter->hw); 5153 5154 return; 5155 } 5156 5157 static void 5158 igb_disable_intr(struct adapter *adapter) 5159 { 5160 if (adapter->msix_mem) { 5161 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); 5162 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); 5163 } 5164 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); 5165 E1000_WRITE_FLUSH(&adapter->hw); 5166 return; 5167 } 5168 5169 /* 5170 * Bit of a misnomer, what this really means is 5171 * to enable OS management of the system... aka 5172 * to disable special hardware management features 5173 */ 5174 static void 5175 igb_init_manageability(struct adapter *adapter) 5176 { 5177 if (adapter->has_manage) { 5178 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 5179 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 5180 5181 /* disable hardware interception of ARP */ 5182 manc &= ~(E1000_MANC_ARP_EN); 5183 5184 /* enable receiving management packets to the host */ 5185 manc |= E1000_MANC_EN_MNG2HOST; 5186 manc2h |= 1 << 5; /* Mng Port 623 */ 5187 manc2h |= 1 << 6; /* Mng Port 664 */ 5188 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 5189 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 5190 } 5191 } 5192 5193 /* 5194 * Give control back to hardware management 5195 * controller if there is one. 5196 */ 5197 static void 5198 igb_release_manageability(struct adapter *adapter) 5199 { 5200 if (adapter->has_manage) { 5201 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 5202 5203 /* re-enable hardware interception of ARP */ 5204 manc |= E1000_MANC_ARP_EN; 5205 manc &= ~E1000_MANC_EN_MNG2HOST; 5206 5207 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 5208 } 5209 } 5210 5211 /* 5212 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 5213 * For ASF and Pass Through versions of f/w this means that 5214 * the driver is loaded. 5215 * 5216 */ 5217 static void 5218 igb_get_hw_control(struct adapter *adapter) 5219 { 5220 u32 ctrl_ext; 5221 5222 if (adapter->vf_ifp) 5223 return; 5224 5225 /* Let firmware know the driver has taken over */ 5226 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 5227 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 5228 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 5229 } 5230 5231 /* 5232 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 5233 * For ASF and Pass Through versions of f/w this means that the 5234 * driver is no longer loaded. 5235 * 5236 */ 5237 static void 5238 igb_release_hw_control(struct adapter *adapter) 5239 { 5240 u32 ctrl_ext; 5241 5242 if (adapter->vf_ifp) 5243 return; 5244 5245 /* Let firmware taken over control of h/w */ 5246 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 5247 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 5248 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 5249 } 5250 5251 static int 5252 igb_is_valid_ether_addr(uint8_t *addr) 5253 { 5254 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 5255 5256 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 5257 return (FALSE); 5258 } 5259 5260 return (TRUE); 5261 } 5262 5263 5264 /* 5265 * Enable PCI Wake On Lan capability 5266 */ 5267 static void 5268 igb_enable_wakeup(device_t dev) 5269 { 5270 u16 cap, status; 5271 u8 id; 5272 5273 /* First find the capabilities pointer*/ 5274 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 5275 /* Read the PM Capabilities */ 5276 id = pci_read_config(dev, cap, 1); 5277 if (id != PCIY_PMG) /* Something wrong */ 5278 return; 5279 /* OK, we have the power capabilities, so 5280 now get the status register */ 5281 cap += PCIR_POWER_STATUS; 5282 status = pci_read_config(dev, cap, 2); 5283 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 5284 pci_write_config(dev, cap, status, 2); 5285 return; 5286 } 5287 5288 static void 5289 igb_led_func(void *arg, int onoff) 5290 { 5291 struct adapter *adapter = arg; 5292 5293 IGB_CORE_LOCK(adapter); 5294 if (onoff) { 5295 e1000_setup_led(&adapter->hw); 5296 e1000_led_on(&adapter->hw); 5297 } else { 5298 e1000_led_off(&adapter->hw); 5299 e1000_cleanup_led(&adapter->hw); 5300 } 5301 IGB_CORE_UNLOCK(adapter); 5302 } 5303 5304 /********************************************************************** 5305 * 5306 * Update the board statistics counters. 5307 * 5308 **********************************************************************/ 5309 static void 5310 igb_update_stats_counters(struct adapter *adapter) 5311 { 5312 struct ifnet *ifp; 5313 struct e1000_hw *hw = &adapter->hw; 5314 struct e1000_hw_stats *stats; 5315 5316 /* 5317 ** The virtual function adapter has only a 5318 ** small controlled set of stats, do only 5319 ** those and return. 5320 */ 5321 if (adapter->vf_ifp) { 5322 igb_update_vf_stats_counters(adapter); 5323 return; 5324 } 5325 5326 stats = (struct e1000_hw_stats *)adapter->stats; 5327 5328 if (adapter->hw.phy.media_type == e1000_media_type_copper || 5329 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 5330 stats->symerrs += 5331 E1000_READ_REG(hw,E1000_SYMERRS); 5332 stats->sec += E1000_READ_REG(hw, E1000_SEC); 5333 } 5334 5335 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 5336 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 5337 stats->scc += E1000_READ_REG(hw, E1000_SCC); 5338 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 5339 5340 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 5341 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 5342 stats->colc += E1000_READ_REG(hw, E1000_COLC); 5343 stats->dc += E1000_READ_REG(hw, E1000_DC); 5344 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 5345 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 5346 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 5347 /* 5348 ** For watchdog management we need to know if we have been 5349 ** paused during the last interval, so capture that here. 5350 */ 5351 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 5352 stats->xoffrxc += adapter->pause_frames; 5353 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 5354 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 5355 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 5356 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 5357 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 5358 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 5359 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 5360 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 5361 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 5362 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 5363 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 5364 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 5365 5366 /* For the 64-bit byte counters the low dword must be read first. */ 5367 /* Both registers clear on the read of the high dword */ 5368 5369 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) + 5370 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32); 5371 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) + 5372 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32); 5373 5374 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 5375 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 5376 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 5377 stats->roc += E1000_READ_REG(hw, E1000_ROC); 5378 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 5379 5380 stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC); 5381 stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC); 5382 stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC); 5383 5384 stats->tor += E1000_READ_REG(hw, E1000_TORL) + 5385 ((u64)E1000_READ_REG(hw, E1000_TORH) << 32); 5386 stats->tot += E1000_READ_REG(hw, E1000_TOTL) + 5387 ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32); 5388 5389 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 5390 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 5391 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 5392 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 5393 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 5394 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 5395 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 5396 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 5397 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 5398 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 5399 5400 /* Interrupt Counts */ 5401 5402 stats->iac += E1000_READ_REG(hw, E1000_IAC); 5403 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 5404 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 5405 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 5406 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 5407 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 5408 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 5409 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 5410 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 5411 5412 /* Host to Card Statistics */ 5413 5414 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 5415 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 5416 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 5417 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 5418 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 5419 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 5420 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 5421 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) + 5422 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32)); 5423 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) + 5424 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32)); 5425 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 5426 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 5427 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 5428 5429 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 5430 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 5431 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 5432 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 5433 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 5434 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 5435 5436 ifp = adapter->ifp; 5437 ifp->if_collisions = stats->colc; 5438 5439 /* Rx Errors */ 5440 ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc + 5441 stats->crcerrs + stats->algnerrc + 5442 stats->ruc + stats->roc + stats->mpc + stats->cexterr; 5443 5444 /* Tx Errors */ 5445 ifp->if_oerrors = stats->ecol + 5446 stats->latecol + adapter->watchdog_events; 5447 5448 /* Driver specific counters */ 5449 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL); 5450 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL); 5451 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS); 5452 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS); 5453 adapter->packet_buf_alloc_tx = 5454 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16); 5455 adapter->packet_buf_alloc_rx = 5456 (E1000_READ_REG(hw, E1000_PBA) & 0xffff); 5457 } 5458 5459 5460 /********************************************************************** 5461 * 5462 * Initialize the VF board statistics counters. 5463 * 5464 **********************************************************************/ 5465 static void 5466 igb_vf_init_stats(struct adapter *adapter) 5467 { 5468 struct e1000_hw *hw = &adapter->hw; 5469 struct e1000_vf_stats *stats; 5470 5471 stats = (struct e1000_vf_stats *)adapter->stats; 5472 if (stats == NULL) 5473 return; 5474 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC); 5475 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC); 5476 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC); 5477 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC); 5478 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC); 5479 } 5480 5481 /********************************************************************** 5482 * 5483 * Update the VF board statistics counters. 5484 * 5485 **********************************************************************/ 5486 static void 5487 igb_update_vf_stats_counters(struct adapter *adapter) 5488 { 5489 struct e1000_hw *hw = &adapter->hw; 5490 struct e1000_vf_stats *stats; 5491 5492 if (adapter->link_speed == 0) 5493 return; 5494 5495 stats = (struct e1000_vf_stats *)adapter->stats; 5496 5497 UPDATE_VF_REG(E1000_VFGPRC, 5498 stats->last_gprc, stats->gprc); 5499 UPDATE_VF_REG(E1000_VFGORC, 5500 stats->last_gorc, stats->gorc); 5501 UPDATE_VF_REG(E1000_VFGPTC, 5502 stats->last_gptc, stats->gptc); 5503 UPDATE_VF_REG(E1000_VFGOTC, 5504 stats->last_gotc, stats->gotc); 5505 UPDATE_VF_REG(E1000_VFMPRC, 5506 stats->last_mprc, stats->mprc); 5507 } 5508 5509 /* Export a single 32-bit register via a read-only sysctl. */ 5510 static int 5511 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 5512 { 5513 struct adapter *adapter; 5514 u_int val; 5515 5516 adapter = oidp->oid_arg1; 5517 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2); 5518 return (sysctl_handle_int(oidp, &val, 0, req)); 5519 } 5520 5521 /* 5522 ** Tuneable interrupt rate handler 5523 */ 5524 static int 5525 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) 5526 { 5527 struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1); 5528 int error; 5529 u32 reg, usec, rate; 5530 5531 reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix)); 5532 usec = ((reg & 0x7FFC) >> 2); 5533 if (usec > 0) 5534 rate = 1000000 / usec; 5535 else 5536 rate = 0; 5537 error = sysctl_handle_int(oidp, &rate, 0, req); 5538 if (error || !req->newptr) 5539 return error; 5540 return 0; 5541 } 5542 5543 /* 5544 * Add sysctl variables, one per statistic, to the system. 5545 */ 5546 static void 5547 igb_add_hw_stats(struct adapter *adapter) 5548 { 5549 device_t dev = adapter->dev; 5550 5551 struct tx_ring *txr = adapter->tx_rings; 5552 struct rx_ring *rxr = adapter->rx_rings; 5553 5554 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 5555 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 5556 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 5557 struct e1000_hw_stats *stats = adapter->stats; 5558 5559 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node; 5560 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list; 5561 5562 #define QUEUE_NAME_LEN 32 5563 char namebuf[QUEUE_NAME_LEN]; 5564 5565 /* Driver Statistics */ 5566 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5567 CTLFLAG_RD, &adapter->dropped_pkts, 5568 "Driver dropped packets"); 5569 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 5570 CTLFLAG_RD, &adapter->link_irq, 5571 "Link MSIX IRQ Handled"); 5572 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", 5573 CTLFLAG_RD, &adapter->mbuf_defrag_failed, 5574 "Defragmenting mbuf chain failed"); 5575 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", 5576 CTLFLAG_RD, &adapter->no_tx_dma_setup, 5577 "Driver tx dma failure in xmit"); 5578 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 5579 CTLFLAG_RD, &adapter->rx_overruns, 5580 "RX overruns"); 5581 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", 5582 CTLFLAG_RD, &adapter->watchdog_events, 5583 "Watchdog timeouts"); 5584 5585 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", 5586 CTLFLAG_RD, &adapter->device_control, 5587 "Device Control Register"); 5588 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", 5589 CTLFLAG_RD, &adapter->rx_control, 5590 "Receiver Control Register"); 5591 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", 5592 CTLFLAG_RD, &adapter->int_mask, 5593 "Interrupt Mask"); 5594 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", 5595 CTLFLAG_RD, &adapter->eint_mask, 5596 "Extended Interrupt Mask"); 5597 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", 5598 CTLFLAG_RD, &adapter->packet_buf_alloc_tx, 5599 "Transmit Buffer Packet Allocation"); 5600 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", 5601 CTLFLAG_RD, &adapter->packet_buf_alloc_rx, 5602 "Receive Buffer Packet Allocation"); 5603 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 5604 CTLFLAG_RD, &adapter->hw.fc.high_water, 0, 5605 "Flow Control High Watermark"); 5606 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 5607 CTLFLAG_RD, &adapter->hw.fc.low_water, 0, 5608 "Flow Control Low Watermark"); 5609 5610 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5611 struct lro_ctrl *lro = &rxr->lro; 5612 5613 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5614 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5615 CTLFLAG_RD, NULL, "Queue Name"); 5616 queue_list = SYSCTL_CHILDREN(queue_node); 5617 5618 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 5619 CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i], 5620 sizeof(&adapter->queues[i]), 5621 igb_sysctl_interrupt_rate_handler, 5622 "IU", "Interrupt Rate"); 5623 5624 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 5625 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me), 5626 igb_sysctl_reg_handler, "IU", 5627 "Transmit Descriptor Head"); 5628 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 5629 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me), 5630 igb_sysctl_reg_handler, "IU", 5631 "Transmit Descriptor Tail"); 5632 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 5633 CTLFLAG_RD, &txr->no_desc_avail, 5634 "Queue No Descriptor Available"); 5635 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", 5636 CTLFLAG_RD, &txr->total_packets, 5637 "Queue Packets Transmitted"); 5638 5639 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 5640 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me), 5641 igb_sysctl_reg_handler, "IU", 5642 "Receive Descriptor Head"); 5643 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 5644 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me), 5645 igb_sysctl_reg_handler, "IU", 5646 "Receive Descriptor Tail"); 5647 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets", 5648 CTLFLAG_RD, &rxr->rx_packets, 5649 "Queue Packets Received"); 5650 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 5651 CTLFLAG_RD, &rxr->rx_bytes, 5652 "Queue Bytes Received"); 5653 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued", 5654 CTLFLAG_RD, &lro->lro_queued, 0, 5655 "LRO Queued"); 5656 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed", 5657 CTLFLAG_RD, &lro->lro_flushed, 0, 5658 "LRO Flushed"); 5659 } 5660 5661 /* MAC stats get their own sub node */ 5662 5663 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 5664 CTLFLAG_RD, NULL, "MAC Statistics"); 5665 stat_list = SYSCTL_CHILDREN(stat_node); 5666 5667 /* 5668 ** VF adapter has a very limited set of stats 5669 ** since its not managing the metal, so to speak. 5670 */ 5671 if (adapter->vf_ifp) { 5672 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 5673 CTLFLAG_RD, &stats->gprc, 5674 "Good Packets Received"); 5675 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 5676 CTLFLAG_RD, &stats->gptc, 5677 "Good Packets Transmitted"); 5678 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 5679 CTLFLAG_RD, &stats->gorc, 5680 "Good Octets Received"); 5681 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 5682 CTLFLAG_RD, &stats->gotc, 5683 "Good Octets Transmitted"); 5684 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 5685 CTLFLAG_RD, &stats->mprc, 5686 "Multicast Packets Received"); 5687 return; 5688 } 5689 5690 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", 5691 CTLFLAG_RD, &stats->ecol, 5692 "Excessive collisions"); 5693 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", 5694 CTLFLAG_RD, &stats->scc, 5695 "Single collisions"); 5696 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 5697 CTLFLAG_RD, &stats->mcc, 5698 "Multiple collisions"); 5699 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", 5700 CTLFLAG_RD, &stats->latecol, 5701 "Late collisions"); 5702 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", 5703 CTLFLAG_RD, &stats->colc, 5704 "Collision Count"); 5705 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 5706 CTLFLAG_RD, &stats->symerrs, 5707 "Symbol Errors"); 5708 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 5709 CTLFLAG_RD, &stats->sec, 5710 "Sequence Errors"); 5711 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count", 5712 CTLFLAG_RD, &stats->dc, 5713 "Defer Count"); 5714 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets", 5715 CTLFLAG_RD, &stats->mpc, 5716 "Missed Packets"); 5717 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", 5718 CTLFLAG_RD, &stats->rlec, 5719 "Receive Length Errors"); 5720 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 5721 CTLFLAG_RD, &stats->rnbc, 5722 "Receive No Buffers"); 5723 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 5724 CTLFLAG_RD, &stats->ruc, 5725 "Receive Undersize"); 5726 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 5727 CTLFLAG_RD, &stats->rfc, 5728 "Fragmented Packets Received"); 5729 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 5730 CTLFLAG_RD, &stats->roc, 5731 "Oversized Packets Received"); 5732 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 5733 CTLFLAG_RD, &stats->rjc, 5734 "Recevied Jabber"); 5735 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs", 5736 CTLFLAG_RD, &stats->rxerrc, 5737 "Receive Errors"); 5738 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs", 5739 CTLFLAG_RD, &stats->crcerrs, 5740 "CRC errors"); 5741 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 5742 CTLFLAG_RD, &stats->algnerrc, 5743 "Alignment Errors"); 5744 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs", 5745 CTLFLAG_RD, &stats->tncrs, 5746 "Transmit with No CRS"); 5747 /* On 82575 these are collision counts */ 5748 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 5749 CTLFLAG_RD, &stats->cexterr, 5750 "Collision/Carrier extension errors"); 5751 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 5752 CTLFLAG_RD, &stats->xonrxc, 5753 "XON Received"); 5754 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd", 5755 CTLFLAG_RD, &stats->xontxc, 5756 "XON Transmitted"); 5757 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 5758 CTLFLAG_RD, &stats->xoffrxc, 5759 "XOFF Received"); 5760 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 5761 CTLFLAG_RD, &stats->xofftxc, 5762 "XOFF Transmitted"); 5763 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", 5764 CTLFLAG_RD, &stats->fcruc, 5765 "Unsupported Flow Control Received"); 5766 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", 5767 CTLFLAG_RD, &stats->mgprc, 5768 "Management Packets Received"); 5769 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", 5770 CTLFLAG_RD, &stats->mgpdc, 5771 "Management Packets Dropped"); 5772 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", 5773 CTLFLAG_RD, &stats->mgptc, 5774 "Management Packets Transmitted"); 5775 /* Packet Reception Stats */ 5776 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 5777 CTLFLAG_RD, &stats->tpr, 5778 "Total Packets Received"); 5779 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 5780 CTLFLAG_RD, &stats->gprc, 5781 "Good Packets Received"); 5782 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 5783 CTLFLAG_RD, &stats->bprc, 5784 "Broadcast Packets Received"); 5785 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 5786 CTLFLAG_RD, &stats->mprc, 5787 "Multicast Packets Received"); 5788 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 5789 CTLFLAG_RD, &stats->prc64, 5790 "64 byte frames received"); 5791 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 5792 CTLFLAG_RD, &stats->prc127, 5793 "65-127 byte frames received"); 5794 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 5795 CTLFLAG_RD, &stats->prc255, 5796 "128-255 byte frames received"); 5797 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 5798 CTLFLAG_RD, &stats->prc511, 5799 "256-511 byte frames received"); 5800 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 5801 CTLFLAG_RD, &stats->prc1023, 5802 "512-1023 byte frames received"); 5803 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 5804 CTLFLAG_RD, &stats->prc1522, 5805 "1023-1522 byte frames received"); 5806 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 5807 CTLFLAG_RD, &stats->gorc, 5808 "Good Octets Received"); 5809 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd", 5810 CTLFLAG_RD, &stats->tor, 5811 "Total Octets Received"); 5812 5813 /* Packet Transmission Stats */ 5814 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 5815 CTLFLAG_RD, &stats->gotc, 5816 "Good Octets Transmitted"); 5817 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd", 5818 CTLFLAG_RD, &stats->tot, 5819 "Total Octets Transmitted"); 5820 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 5821 CTLFLAG_RD, &stats->tpt, 5822 "Total Packets Transmitted"); 5823 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 5824 CTLFLAG_RD, &stats->gptc, 5825 "Good Packets Transmitted"); 5826 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 5827 CTLFLAG_RD, &stats->bptc, 5828 "Broadcast Packets Transmitted"); 5829 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 5830 CTLFLAG_RD, &stats->mptc, 5831 "Multicast Packets Transmitted"); 5832 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 5833 CTLFLAG_RD, &stats->ptc64, 5834 "64 byte frames transmitted"); 5835 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 5836 CTLFLAG_RD, &stats->ptc127, 5837 "65-127 byte frames transmitted"); 5838 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 5839 CTLFLAG_RD, &stats->ptc255, 5840 "128-255 byte frames transmitted"); 5841 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 5842 CTLFLAG_RD, &stats->ptc511, 5843 "256-511 byte frames transmitted"); 5844 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 5845 CTLFLAG_RD, &stats->ptc1023, 5846 "512-1023 byte frames transmitted"); 5847 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 5848 CTLFLAG_RD, &stats->ptc1522, 5849 "1024-1522 byte frames transmitted"); 5850 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd", 5851 CTLFLAG_RD, &stats->tsctc, 5852 "TSO Contexts Transmitted"); 5853 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 5854 CTLFLAG_RD, &stats->tsctfc, 5855 "TSO Contexts Failed"); 5856 5857 5858 /* Interrupt Stats */ 5859 5860 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 5861 CTLFLAG_RD, NULL, "Interrupt Statistics"); 5862 int_list = SYSCTL_CHILDREN(int_node); 5863 5864 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts", 5865 CTLFLAG_RD, &stats->iac, 5866 "Interrupt Assertion Count"); 5867 5868 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 5869 CTLFLAG_RD, &stats->icrxptc, 5870 "Interrupt Cause Rx Pkt Timer Expire Count"); 5871 5872 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 5873 CTLFLAG_RD, &stats->icrxatc, 5874 "Interrupt Cause Rx Abs Timer Expire Count"); 5875 5876 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 5877 CTLFLAG_RD, &stats->ictxptc, 5878 "Interrupt Cause Tx Pkt Timer Expire Count"); 5879 5880 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 5881 CTLFLAG_RD, &stats->ictxatc, 5882 "Interrupt Cause Tx Abs Timer Expire Count"); 5883 5884 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 5885 CTLFLAG_RD, &stats->ictxqec, 5886 "Interrupt Cause Tx Queue Empty Count"); 5887 5888 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 5889 CTLFLAG_RD, &stats->ictxqmtc, 5890 "Interrupt Cause Tx Queue Min Thresh Count"); 5891 5892 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 5893 CTLFLAG_RD, &stats->icrxdmtc, 5894 "Interrupt Cause Rx Desc Min Thresh Count"); 5895 5896 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun", 5897 CTLFLAG_RD, &stats->icrxoc, 5898 "Interrupt Cause Receiver Overrun Count"); 5899 5900 /* Host to Card Stats */ 5901 5902 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", 5903 CTLFLAG_RD, NULL, 5904 "Host to Card Statistics"); 5905 5906 host_list = SYSCTL_CHILDREN(host_node); 5907 5908 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt", 5909 CTLFLAG_RD, &stats->cbtmpc, 5910 "Circuit Breaker Tx Packet Count"); 5911 5912 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard", 5913 CTLFLAG_RD, &stats->htdpmc, 5914 "Host Transmit Discarded Packets"); 5915 5916 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt", 5917 CTLFLAG_RD, &stats->rpthc, 5918 "Rx Packets To Host"); 5919 5920 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts", 5921 CTLFLAG_RD, &stats->cbrmpc, 5922 "Circuit Breaker Rx Packet Count"); 5923 5924 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop", 5925 CTLFLAG_RD, &stats->cbrdpc, 5926 "Circuit Breaker Rx Dropped Count"); 5927 5928 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt", 5929 CTLFLAG_RD, &stats->hgptc, 5930 "Host Good Packets Tx Count"); 5931 5932 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop", 5933 CTLFLAG_RD, &stats->htcbdpc, 5934 "Host Tx Circuit Breaker Dropped Count"); 5935 5936 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes", 5937 CTLFLAG_RD, &stats->hgorc, 5938 "Host Good Octets Received Count"); 5939 5940 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes", 5941 CTLFLAG_RD, &stats->hgotc, 5942 "Host Good Octets Transmit Count"); 5943 5944 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors", 5945 CTLFLAG_RD, &stats->lenerrs, 5946 "Length Errors"); 5947 5948 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt", 5949 CTLFLAG_RD, &stats->scvpc, 5950 "SerDes/SGMII Code Violation Pkt Count"); 5951 5952 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed", 5953 CTLFLAG_RD, &stats->hrmpc, 5954 "Header Redirection Missed Packet Count"); 5955 } 5956 5957 5958 /********************************************************************** 5959 * 5960 * This routine provides a way to dump out the adapter eeprom, 5961 * often a useful debug/service tool. This only dumps the first 5962 * 32 words, stuff that matters is in that extent. 5963 * 5964 **********************************************************************/ 5965 static int 5966 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 5967 { 5968 struct adapter *adapter; 5969 int error; 5970 int result; 5971 5972 result = -1; 5973 error = sysctl_handle_int(oidp, &result, 0, req); 5974 5975 if (error || !req->newptr) 5976 return (error); 5977 5978 /* 5979 * This value will cause a hex dump of the 5980 * first 32 16-bit words of the EEPROM to 5981 * the screen. 5982 */ 5983 if (result == 1) { 5984 adapter = (struct adapter *)arg1; 5985 igb_print_nvm_info(adapter); 5986 } 5987 5988 return (error); 5989 } 5990 5991 static void 5992 igb_print_nvm_info(struct adapter *adapter) 5993 { 5994 u16 eeprom_data; 5995 int i, j, row = 0; 5996 5997 /* Its a bit crude, but it gets the job done */ 5998 printf("\nInterface EEPROM Dump:\n"); 5999 printf("Offset\n0x0000 "); 6000 for (i = 0, j = 0; i < 32; i++, j++) { 6001 if (j == 8) { /* Make the offset block */ 6002 j = 0; ++row; 6003 printf("\n0x00%x0 ",row); 6004 } 6005 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 6006 printf("%04x ", eeprom_data); 6007 } 6008 printf("\n"); 6009 } 6010 6011 static void 6012 igb_set_sysctl_value(struct adapter *adapter, const char *name, 6013 const char *description, int *limit, int value) 6014 { 6015 *limit = value; 6016 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 6017 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 6018 OID_AUTO, name, CTLFLAG_RW, limit, value, description); 6019 } 6020 6021 /* 6022 ** Set flow control using sysctl: 6023 ** Flow control values: 6024 ** 0 - off 6025 ** 1 - rx pause 6026 ** 2 - tx pause 6027 ** 3 - full 6028 */ 6029 static int 6030 igb_set_flowcntl(SYSCTL_HANDLER_ARGS) 6031 { 6032 int error; 6033 static int input = 3; /* default is full */ 6034 struct adapter *adapter = (struct adapter *) arg1; 6035 6036 error = sysctl_handle_int(oidp, &input, 0, req); 6037 6038 if ((error) || (req->newptr == NULL)) 6039 return (error); 6040 6041 switch (input) { 6042 case e1000_fc_rx_pause: 6043 case e1000_fc_tx_pause: 6044 case e1000_fc_full: 6045 case e1000_fc_none: 6046 adapter->hw.fc.requested_mode = input; 6047 adapter->fc = input; 6048 break; 6049 default: 6050 /* Do nothing */ 6051 return (error); 6052 } 6053 6054 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; 6055 e1000_force_mac_fc(&adapter->hw); 6056 /* XXX TODO: update DROP_EN on each RX queue if appropriate */ 6057 return (error); 6058 } 6059 6060 /* 6061 ** Manage DMA Coalesce: 6062 ** Control values: 6063 ** 0/1 - off/on 6064 ** Legal timer values are: 6065 ** 250,500,1000-10000 in thousands 6066 */ 6067 static int 6068 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) 6069 { 6070 struct adapter *adapter = (struct adapter *) arg1; 6071 int error; 6072 6073 error = sysctl_handle_int(oidp, &adapter->dmac, 0, req); 6074 6075 if ((error) || (req->newptr == NULL)) 6076 return (error); 6077 6078 switch (adapter->dmac) { 6079 case 0: 6080 /* Disabling */ 6081 break; 6082 case 1: /* Just enable and use default */ 6083 adapter->dmac = 1000; 6084 break; 6085 case 250: 6086 case 500: 6087 case 1000: 6088 case 2000: 6089 case 3000: 6090 case 4000: 6091 case 5000: 6092 case 6000: 6093 case 7000: 6094 case 8000: 6095 case 9000: 6096 case 10000: 6097 /* Legal values - allow */ 6098 break; 6099 default: 6100 /* Do nothing, illegal value */ 6101 adapter->dmac = 0; 6102 return (EINVAL); 6103 } 6104 /* Reinit the interface */ 6105 igb_init(adapter); 6106 return (error); 6107 } 6108 6109 /* 6110 ** Manage Energy Efficient Ethernet: 6111 ** Control values: 6112 ** 0/1 - enabled/disabled 6113 */ 6114 static int 6115 igb_sysctl_eee(SYSCTL_HANDLER_ARGS) 6116 { 6117 struct adapter *adapter = (struct adapter *) arg1; 6118 int error, value; 6119 6120 value = adapter->hw.dev_spec._82575.eee_disable; 6121 error = sysctl_handle_int(oidp, &value, 0, req); 6122 if (error || req->newptr == NULL) 6123 return (error); 6124 IGB_CORE_LOCK(adapter); 6125 adapter->hw.dev_spec._82575.eee_disable = (value != 0); 6126 igb_init_locked(adapter); 6127 IGB_CORE_UNLOCK(adapter); 6128 return (0); 6129 }
Cache object: 6b362b02230d42a46d8c4583d3c5976a
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]