Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/dev/netif/ixgbe/ixgbe.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /****************************************************************************** 2 3 Copyright (c) 2001-2012, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/ 34 35 #include "opt_inet.h" 36 #include "opt_inet6.h" 37 38 #include "ixgbe.h" 39 40 /********************************************************************* 41 * Set this to one to display debug statistics 42 *********************************************************************/ 43 int ixgbe_display_debug_stats = 0; 44 45 /********************************************************************* 46 * Driver version 47 *********************************************************************/ 48 char ixgbe_driver_version[] = "2.4.8"; 49 50 /********************************************************************* 51 * PCI Device ID Table 52 * 53 * Used by probe to select devices to load on 54 * Last field stores an index into ixgbe_strings 55 * Last entry must be all 0s 56 * 57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 58 *********************************************************************/ 59 60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 61 { 62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 86 /* required last entry */ 87 {0, 0, 0, 0, 0} 88 }; 89 90 /********************************************************************* 91 * Table of branding strings 92 *********************************************************************/ 93 94 static char *ixgbe_strings[] = { 95 "Intel(R) PRO/10GbE PCI-Express Network Driver" 96 }; 97 98 /********************************************************************* 99 * Function prototypes 100 *********************************************************************/ 101 static int ixgbe_probe(device_t); 102 static int ixgbe_attach(device_t); 103 static int ixgbe_detach(device_t); 104 static int ixgbe_shutdown(device_t); 105 static void ixgbe_start(struct ifnet *, struct ifaltq_subque *); 106 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 107 #if 0 /* __FreeBSD_version >= 800000 */ 108 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); 109 static int ixgbe_mq_start_locked(struct ifnet *, 110 struct tx_ring *, struct mbuf *); 111 static void ixgbe_qflush(struct ifnet *); 112 #endif 113 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 114 static void ixgbe_init(void *); 115 static void ixgbe_init_locked(struct adapter *); 116 static void ixgbe_stop(void *); 117 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 118 static int ixgbe_media_change(struct ifnet *); 119 static void ixgbe_identify_hardware(struct adapter *); 120 static int ixgbe_allocate_pci_resources(struct adapter *); 121 static int ixgbe_allocate_msix(struct adapter *); 122 static int ixgbe_allocate_legacy(struct adapter *); 123 static int ixgbe_allocate_queues(struct adapter *); 124 static int ixgbe_setup_msix(struct adapter *); 125 static void ixgbe_free_pci_resources(struct adapter *); 126 static void ixgbe_local_timer(void *); 127 static int ixgbe_setup_interface(device_t, struct adapter *); 128 static void ixgbe_config_link(struct adapter *); 129 130 static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 131 static int ixgbe_setup_transmit_structures(struct adapter *); 132 static void ixgbe_setup_transmit_ring(struct tx_ring *); 133 static void ixgbe_initialize_transmit_units(struct adapter *); 134 static void ixgbe_free_transmit_structures(struct adapter *); 135 static void ixgbe_free_transmit_buffers(struct tx_ring *); 136 137 static int ixgbe_allocate_receive_buffers(struct rx_ring *); 138 static int ixgbe_setup_receive_structures(struct adapter *); 139 static int ixgbe_setup_receive_ring(struct rx_ring *); 140 static void ixgbe_initialize_receive_units(struct adapter *); 141 static void ixgbe_free_receive_structures(struct adapter *); 142 static void ixgbe_free_receive_buffers(struct rx_ring *); 143 #if 0 /* NET_LRO */ 144 static void ixgbe_setup_hw_rsc(struct rx_ring *); 145 #endif 146 147 static void ixgbe_enable_intr(struct adapter *); 148 static void ixgbe_disable_intr(struct adapter *); 149 static void ixgbe_update_stats_counters(struct adapter *); 150 static void ixgbe_txeof(struct tx_ring *); 151 static void ixgbe_rxeof(struct ix_queue *, int); 152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32); 153 static void ixgbe_set_promisc(struct adapter *); 154 static void ixgbe_set_multi(struct adapter *); 155 static void ixgbe_update_link_status(struct adapter *); 156 static void ixgbe_refresh_mbufs(struct rx_ring *, int); 157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **); 158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS); 159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS); 160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS); 161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 162 struct ixgbe_dma_alloc *, int); 163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 164 static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 165 const char *, int *, int); 166 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 167 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); 168 static int ixgbe_tso_pullup(struct tx_ring *, struct mbuf **); 169 static void ixgbe_add_sysctl(struct adapter *); 170 static void ixgbe_set_eitr(struct adapter *, int, int); 171 static int ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS); 172 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 173 static void ixgbe_configure_ivars(struct adapter *); 174 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 175 176 static void ixgbe_setup_vlan_hw_support(struct adapter *); 177 static void ixgbe_register_vlan(void *, struct ifnet *, u16); 178 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 179 180 static void ixgbe_add_hw_stats(struct adapter *adapter); 181 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int); 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 184 struct mbuf *, u32); 185 186 /* Support for pluggable optic modules */ 187 static bool ixgbe_sfp_probe(struct adapter *); 188 static void ixgbe_setup_optics(struct adapter *); 189 190 /* Legacy (single vector interrupt handler */ 191 static void ixgbe_legacy_irq(void *); 192 193 /* The MSI/X Interrupt handlers */ 194 static void ixgbe_msix_que(void *); 195 static void ixgbe_msix_link(void *); 196 197 /* Deferred interrupt tasklets */ 198 static void ixgbe_handle_msf(void *, int); 199 static void ixgbe_handle_mod(void *, int); 200 201 #ifdef IXGBE_FDIR 202 static void ixgbe_atr(struct tx_ring *, struct mbuf *); 203 static void ixgbe_reinit_fdir(void *, int); 204 #endif 205 206 /********************************************************************* 207 * FreeBSD Device Interface Entry Points 208 *********************************************************************/ 209 210 static device_method_t ixgbe_methods[] = { 211 /* Device interface */ 212 DEVMETHOD(device_probe, ixgbe_probe), 213 DEVMETHOD(device_attach, ixgbe_attach), 214 DEVMETHOD(device_detach, ixgbe_detach), 215 DEVMETHOD(device_shutdown, ixgbe_shutdown), 216 DEVMETHOD_END 217 }; 218 219 static driver_t ixgbe_driver = { 220 "ix", ixgbe_methods, sizeof(struct adapter), 221 }; 222 223 devclass_t ixgbe_devclass; 224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, NULL, NULL); 225 226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 228 229 /* 230 ** TUNEABLE PARAMETERS: 231 */ 232 233 /* How many packets rxeof tries to clean at a time */ 234 static int ixgbe_rx_process_limit = 128; 235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 236 237 /* 238 ** Smart speed setting, default to on 239 ** this only works as a compile option 240 ** right now as its during attach, set 241 ** this to 'ixgbe_smart_speed_off' to 242 ** disable. 243 */ 244 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 245 246 static int ixgbe_msi_enable = 1; 247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable); 248 249 /* 250 * MSIX should be the default for best performance, 251 * but this allows it to be forced off for testing. 252 */ 253 static int ixgbe_enable_msix = 1; 254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 255 256 /* 257 * Header split: this causes the hardware to DMA 258 * the header into a separate mbuf from the payload, 259 * it can be a performance win in some workloads, but 260 * in others it actually hurts, its off by default. 261 */ 262 static int ixgbe_header_split = FALSE; 263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); 264 265 /* 266 * Number of Queues, can be set to 0, 267 * it then autoconfigures based on the 268 * number of cpus with a max of 8. This 269 * can be overriden manually here. 270 */ 271 static int ixgbe_num_queues = 0; 272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); 273 274 /* 275 ** Number of TX descriptors per ring, 276 ** setting higher than RX as this seems 277 ** the better performing choice. 278 */ 279 static int ixgbe_txd = PERFORM_TXD; 280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 281 282 /* Number of RX descriptors per ring */ 283 static int ixgbe_rxd = PERFORM_RXD; 284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 285 286 /* Keep running tab on them for sanity check */ 287 static int ixgbe_total_ports; 288 289 #ifdef IXGBE_FDIR 290 /* 291 ** For Flow Director: this is the 292 ** number of TX packets we sample 293 ** for the filter pool, this means 294 ** every 20th packet will be probed. 295 ** 296 ** This feature can be disabled by 297 ** setting this to 0. 298 */ 299 static int atr_sample_rate = 20; 300 /* 301 ** Flow Director actually 'steals' 302 ** part of the packet buffer as its 303 ** filter pool, this variable controls 304 ** how much it uses: 305 ** 0 = 64K, 1 = 128K, 2 = 256K 306 */ 307 static int fdir_pballoc = 1; 308 #endif 309 310 #ifdef DEV_NETMAP 311 /* 312 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to 313 * be a reference on how to implement netmap support in a driver. 314 * Additional comments are in ixgbe_netmap.h . 315 * 316 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support 317 * that extend the standard driver. 318 */ 319 #include <dev/netmap/ixgbe_netmap.h> 320 #endif /* DEV_NETMAP */ 321 322 /********************************************************************* 323 * Device identification routine 324 * 325 * ixgbe_probe determines if the driver should be loaded on 326 * adapter based on PCI vendor/device id of the adapter. 327 * 328 * return BUS_PROBE_DEFAULT on success, positive on failure 329 *********************************************************************/ 330 331 static int 332 ixgbe_probe(device_t dev) 333 { 334 ixgbe_vendor_info_t *ent; 335 336 u16 pci_vendor_id = 0; 337 u16 pci_device_id = 0; 338 u16 pci_subvendor_id = 0; 339 u16 pci_subdevice_id = 0; 340 char adapter_name[256]; 341 342 INIT_DEBUGOUT("ixgbe_probe: begin"); 343 344 pci_vendor_id = pci_get_vendor(dev); 345 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 346 return (ENXIO); 347 348 pci_device_id = pci_get_device(dev); 349 pci_subvendor_id = pci_get_subvendor(dev); 350 pci_subdevice_id = pci_get_subdevice(dev); 351 352 ent = ixgbe_vendor_info_array; 353 while (ent->vendor_id != 0) { 354 if ((pci_vendor_id == ent->vendor_id) && 355 (pci_device_id == ent->device_id) && 356 357 ((pci_subvendor_id == ent->subvendor_id) || 358 (ent->subvendor_id == 0)) && 359 360 ((pci_subdevice_id == ent->subdevice_id) || 361 (ent->subdevice_id == 0))) { 362 ksprintf(adapter_name, "%s, Version - %s", 363 ixgbe_strings[ent->index], 364 ixgbe_driver_version); 365 device_set_desc_copy(dev, adapter_name); 366 ++ixgbe_total_ports; 367 return (BUS_PROBE_DEFAULT); 368 } 369 ent++; 370 } 371 return (ENXIO); 372 } 373 374 /********************************************************************* 375 * Device initialization routine 376 * 377 * The attach entry point is called when the driver is being loaded. 378 * This routine identifies the type of hardware, allocates all resources 379 * and initializes the hardware. 380 * 381 * return 0 on success, positive on failure 382 *********************************************************************/ 383 384 static int 385 ixgbe_attach(device_t dev) 386 { 387 struct adapter *adapter; 388 struct ixgbe_hw *hw; 389 int error = 0; 390 u16 csum; 391 u32 ctrl_ext; 392 393 INIT_DEBUGOUT("ixgbe_attach: begin"); 394 395 if (resource_disabled("ixgbe", device_get_unit(dev))) { 396 device_printf(dev, "Disabled by device hint\n"); 397 return (ENXIO); 398 } 399 400 /* Allocate, clear, and link in our adapter structure */ 401 adapter = device_get_softc(dev); 402 adapter->dev = adapter->osdep.dev = dev; 403 hw = &adapter->hw; 404 405 /* Core Lock Init*/ 406 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 407 408 /* Set up the timer callout */ 409 callout_init_mp(&adapter->timer); 410 411 /* Determine hardware revision */ 412 ixgbe_identify_hardware(adapter); 413 414 /* Enable bus mastering */ 415 pci_enable_busmaster(dev); 416 417 /* Do base PCI setup - map BAR0 */ 418 if (ixgbe_allocate_pci_resources(adapter)) { 419 device_printf(dev, "Allocation of PCI resources failed\n"); 420 error = ENXIO; 421 goto err_out; 422 } 423 424 /* Do descriptor calc and sanity checks */ 425 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 426 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 427 device_printf(dev, "TXD config issue, using default!\n"); 428 adapter->num_tx_desc = DEFAULT_TXD; 429 } else 430 adapter->num_tx_desc = ixgbe_txd; 431 432 /* 433 ** With many RX rings it is easy to exceed the 434 ** system mbuf allocation. Tuning nmbclusters 435 ** can alleviate this. 436 */ 437 if (nmbclusters > 0 ) { 438 int s; 439 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; 440 if (s > nmbclusters) { 441 device_printf(dev, "RX Descriptors exceed " 442 "system mbuf max, using default instead!\n"); 443 ixgbe_rxd = DEFAULT_RXD; 444 } 445 } 446 447 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 448 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 449 device_printf(dev, "RXD config issue, using default!\n"); 450 adapter->num_rx_desc = DEFAULT_RXD; 451 } else 452 adapter->num_rx_desc = ixgbe_rxd; 453 454 /* Allocate our TX/RX Queues */ 455 if (ixgbe_allocate_queues(adapter)) { 456 error = ENOMEM; 457 goto err_out; 458 } 459 460 /* Allocate multicast array memory. */ 461 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 462 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 463 if (adapter->mta == NULL) { 464 device_printf(dev, "Can not allocate multicast setup array\n"); 465 error = ENOMEM; 466 goto err_late; 467 } 468 469 /* Initialize the shared code */ 470 error = ixgbe_init_shared_code(hw); 471 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 472 /* 473 ** No optics in this port, set up 474 ** so the timer routine will probe 475 ** for later insertion. 476 */ 477 adapter->sfp_probe = TRUE; 478 error = 0; 479 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 480 device_printf(dev,"Unsupported SFP+ module detected!\n"); 481 error = EIO; 482 goto err_late; 483 } else if (error) { 484 device_printf(dev,"Unable to initialize the shared code\n"); 485 error = EIO; 486 goto err_late; 487 } 488 489 /* Make sure we have a good EEPROM before we read from it */ 490 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 491 device_printf(dev,"The EEPROM Checksum Is Not Valid\n"); 492 error = EIO; 493 goto err_late; 494 } 495 496 error = ixgbe_init_hw(hw); 497 switch (error) { 498 case IXGBE_ERR_EEPROM_VERSION: 499 device_printf(dev, "This device is a pre-production adapter/" 500 "LOM. Please be aware there may be issues associated " 501 "with your hardware.\n If you are experiencing problems " 502 "please contact your Intel or hardware representative " 503 "who provided you with this hardware.\n"); 504 break; 505 case IXGBE_ERR_SFP_NOT_SUPPORTED: 506 device_printf(dev,"Unsupported SFP+ Module\n"); 507 error = EIO; 508 device_printf(dev,"Hardware Initialization Failure\n"); 509 goto err_late; 510 case IXGBE_ERR_SFP_NOT_PRESENT: 511 device_printf(dev,"No SFP+ Module found\n"); 512 /* falls thru */ 513 default: 514 break; 515 } 516 517 /* Detect and set physical type */ 518 ixgbe_setup_optics(adapter); 519 520 if ((adapter->msix > 1) && (ixgbe_enable_msix)) { 521 adapter->intr_type = PCI_INTR_TYPE_MSIX; 522 error = ixgbe_allocate_msix(adapter); 523 } else { 524 error = ixgbe_allocate_legacy(adapter); 525 } 526 if (error) 527 goto err_late; 528 529 /* Setup OS specific network interface */ 530 if (ixgbe_setup_interface(dev, adapter) != 0) 531 goto err_late; 532 533 /* Add sysctl tree */ 534 ixgbe_add_sysctl(adapter); 535 536 /* Initialize statistics */ 537 ixgbe_update_stats_counters(adapter); 538 539 /* Register for VLAN events */ 540 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 541 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 542 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 543 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 544 545 /* Print PCIE bus type/speed/width info */ 546 ixgbe_get_bus_info(hw); 547 device_printf(dev,"PCI Express Bus: Speed %s %s\n", 548 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": 549 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), 550 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : 551 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : 552 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : 553 ("Unknown")); 554 555 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 556 (hw->bus.speed == ixgbe_bus_speed_2500)) { 557 device_printf(dev, "PCI-Express bandwidth available" 558 " for this card\n is not sufficient for" 559 " optimal performance.\n"); 560 device_printf(dev, "For optimal performance a x8 " 561 "PCIE, or x4 PCIE 2 slot is required.\n"); 562 } 563 564 /* let hardware know driver is loaded */ 565 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 566 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 567 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 568 569 ixgbe_add_hw_stats(adapter); 570 571 #ifdef DEV_NETMAP 572 ixgbe_netmap_attach(adapter); 573 #endif /* DEV_NETMAP */ 574 INIT_DEBUGOUT("ixgbe_attach: end"); 575 return (0); 576 err_late: 577 ixgbe_free_transmit_structures(adapter); 578 ixgbe_free_receive_structures(adapter); 579 err_out: 580 if (adapter->ifp != NULL) 581 if_free(adapter->ifp); 582 ixgbe_free_pci_resources(adapter); 583 kfree(adapter->mta, M_DEVBUF); 584 return (error); 585 586 } 587 588 /********************************************************************* 589 * Device removal routine 590 * 591 * The detach entry point is called when the driver is being removed. 592 * This routine stops the adapter and deallocates all the resources 593 * that were allocated for driver operation. 594 * 595 * return 0 on success, positive on failure 596 *********************************************************************/ 597 598 static int 599 ixgbe_detach(device_t dev) 600 { 601 struct adapter *adapter = device_get_softc(dev); 602 u32 ctrl_ext; 603 604 INIT_DEBUGOUT("ixgbe_detach: begin"); 605 606 /* Make sure VLANS are not using driver */ 607 if (adapter->ifp->if_vlantrunks != NULL) { 608 device_printf(dev,"Vlan in use, detach first\n"); 609 return (EBUSY); 610 } 611 612 IXGBE_CORE_LOCK(adapter); 613 ixgbe_stop(adapter); 614 IXGBE_CORE_UNLOCK(adapter); 615 616 /* Drain the Link queue */ 617 if (adapter->tq) { 618 taskqueue_drain(adapter->tq, &adapter->mod_task); 619 taskqueue_drain(adapter->tq, &adapter->msf_task); 620 #ifdef IXGBE_FDIR 621 taskqueue_drain(adapter->tq, &adapter->fdir_task); 622 #endif 623 taskqueue_free(adapter->tq); 624 } 625 626 /* let hardware know driver is unloading */ 627 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 628 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 629 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 630 631 /* Unregister VLAN events */ 632 if (adapter->vlan_attach != NULL) 633 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 634 if (adapter->vlan_detach != NULL) 635 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 636 637 ether_ifdetach(adapter->ifp); 638 callout_stop(&adapter->timer); 639 #ifdef DEV_NETMAP 640 netmap_detach(adapter->ifp); 641 #endif /* DEV_NETMAP */ 642 ixgbe_free_pci_resources(adapter); 643 bus_generic_detach(dev); 644 if_free(adapter->ifp); 645 646 ixgbe_free_transmit_structures(adapter); 647 ixgbe_free_receive_structures(adapter); 648 kfree(adapter->mta, M_DEVBUF); 649 sysctl_ctx_free(&adapter->sysctl_ctx); 650 651 IXGBE_CORE_LOCK_DESTROY(adapter); 652 return (0); 653 } 654 655 /********************************************************************* 656 * 657 * Shutdown entry point 658 * 659 **********************************************************************/ 660 661 static int 662 ixgbe_shutdown(device_t dev) 663 { 664 struct adapter *adapter = device_get_softc(dev); 665 IXGBE_CORE_LOCK(adapter); 666 ixgbe_stop(adapter); 667 IXGBE_CORE_UNLOCK(adapter); 668 return (0); 669 } 670 671 672 /********************************************************************* 673 * Transmit entry point 674 * 675 * ixgbe_start is called by the stack to initiate a transmit. 676 * The driver will remain in this routine as long as there are 677 * packets to transmit and transmit resources are available. 678 * In case resources are not available stack is notified and 679 * the packet is requeued. 680 **********************************************************************/ 681 682 static void 683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 684 { 685 struct mbuf *m_head; 686 struct adapter *adapter = txr->adapter; 687 688 IXGBE_TX_LOCK_ASSERT(txr); 689 690 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) 691 return; 692 693 if (!adapter->link_active) { 694 ifq_purge(&ifp->if_snd); 695 return; 696 } 697 698 while (!ifq_is_empty(&ifp->if_snd)) { 699 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) { 700 txr->queue_status |= IXGBE_QUEUE_DEPLETED; 701 break; 702 } 703 704 m_head = ifq_dequeue(&ifp->if_snd); 705 if (m_head == NULL) 706 break; 707 708 if (ixgbe_xmit(txr, &m_head)) { 709 if (m_head != NULL) 710 ifq_prepend(&ifp->if_snd, m_head); 711 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) 712 txr->queue_status |= IXGBE_QUEUE_DEPLETED; 713 break; 714 } 715 /* Send a copy of the frame to the BPF listener */ 716 ETHER_BPF_MTAP(ifp, m_head); 717 718 /* Set watchdog on */ 719 txr->watchdog_time = ticks; 720 txr->queue_status = IXGBE_QUEUE_WORKING; 721 722 } 723 return; 724 } 725 726 /* 727 * Legacy TX start - called by the stack, this 728 * always uses the first tx ring, and should 729 * not be used with multiqueue tx enabled. 730 */ 731 static void 732 ixgbe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 733 { 734 struct adapter *adapter = ifp->if_softc; 735 struct tx_ring *txr = adapter->tx_rings; 736 737 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); 738 739 if (ifp->if_flags & IFF_RUNNING) { 740 IXGBE_TX_LOCK(txr); 741 ixgbe_start_locked(txr, ifp); 742 IXGBE_TX_UNLOCK(txr); 743 } 744 return; 745 } 746 747 #if 0 /* __FreeBSD_version >= 800000 */ 748 /* 749 ** Multiqueue Transmit driver 750 ** 751 */ 752 static int 753 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 754 { 755 struct adapter *adapter = ifp->if_softc; 756 struct ix_queue *que; 757 struct tx_ring *txr; 758 int i = 0, err = 0; 759 760 /* Which queue to use */ 761 if ((m->m_flags & M_FLOWID) != 0) 762 i = m->m_pkthdr.flowid % adapter->num_queues; 763 else 764 i = curcpu % adapter->num_queues; 765 766 txr = &adapter->tx_rings[i]; 767 que = &adapter->queues[i]; 768 769 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) && 770 IXGBE_TX_TRYLOCK(txr)) { 771 err = ixgbe_mq_start_locked(ifp, txr, m); 772 IXGBE_TX_UNLOCK(txr); 773 } else { 774 err = drbr_enqueue(ifp, txr->br, m); 775 taskqueue_enqueue(que->tq, &que->que_task); 776 } 777 778 return (err); 779 } 780 781 static int 782 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 783 { 784 struct adapter *adapter = txr->adapter; 785 struct mbuf *next; 786 int enqueued, err = 0; 787 788 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || 789 (txr->queue_status == IXGBE_QUEUE_DEPLETED) || 790 adapter->link_active == 0) { 791 if (m != NULL) 792 err = drbr_enqueue(ifp, txr->br, m); 793 return (err); 794 } 795 796 enqueued = 0; 797 if (m == NULL) { 798 next = drbr_dequeue(ifp, txr->br); 799 } else if (drbr_needs_enqueue(ifp, txr->br)) { 800 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 801 return (err); 802 next = drbr_dequeue(ifp, txr->br); 803 } else 804 next = m; 805 806 /* Process the queue */ 807 while (next != NULL) { 808 if ((err = ixgbe_xmit(txr, &next)) != 0) { 809 if (next != NULL) 810 err = drbr_enqueue(ifp, txr->br, next); 811 break; 812 } 813 enqueued++; 814 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 815 /* Send a copy of the frame to the BPF listener */ 816 ETHER_BPF_MTAP(ifp, next); 817 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 818 break; 819 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) 820 ixgbe_txeof(txr); 821 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { 822 txr->queue_status |= IXGBE_QUEUE_DEPLETED; 823 break; 824 } 825 next = drbr_dequeue(ifp, txr->br); 826 } 827 828 if (enqueued > 0) { 829 /* Set watchdog on */ 830 txr->queue_status |= IXGBE_QUEUE_WORKING; 831 txr->watchdog_time = ticks; 832 } 833 834 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD) 835 ixgbe_txeof(txr); 836 837 return (err); 838 } 839 840 /* 841 ** Flush all ring buffers 842 */ 843 static void 844 ixgbe_qflush(struct ifnet *ifp) 845 { 846 struct adapter *adapter = ifp->if_softc; 847 struct tx_ring *txr = adapter->tx_rings; 848 struct mbuf *m; 849 850 for (int i = 0; i < adapter->num_queues; i++, txr++) { 851 IXGBE_TX_LOCK(txr); 852 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 853 m_freem(m); 854 IXGBE_TX_UNLOCK(txr); 855 } 856 if_qflush(ifp); 857 } 858 #endif /* __FreeBSD_version >= 800000 */ 859 860 /********************************************************************* 861 * Ioctl entry point 862 * 863 * ixgbe_ioctl is called when the user wants to configure the 864 * interface. 865 * 866 * return 0 on success, positive on failure 867 **********************************************************************/ 868 869 static int 870 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 871 { 872 struct adapter *adapter = ifp->if_softc; 873 struct ifreq *ifr = (struct ifreq *) data; 874 #if defined(INET) || defined(INET6) 875 struct ifaddr *ifa = (struct ifaddr *)data; 876 bool avoid_reset = FALSE; 877 #endif 878 int error = 0; 879 880 switch (command) { 881 882 case SIOCSIFADDR: 883 #ifdef INET 884 if (ifa->ifa_addr->sa_family == AF_INET) 885 avoid_reset = TRUE; 886 #endif 887 #ifdef INET6 888 if (ifa->ifa_addr->sa_family == AF_INET6) 889 avoid_reset = TRUE; 890 #endif 891 #if defined(INET) || defined(INET6) 892 /* 893 ** Calling init results in link renegotiation, 894 ** so we avoid doing it when possible. 895 */ 896 if (avoid_reset) { 897 ifp->if_flags |= IFF_UP; 898 if (!(ifp->if_flags & IFF_RUNNING)) 899 ixgbe_init(adapter); 900 if (!(ifp->if_flags & IFF_NOARP)) 901 arp_ifinit(ifp, ifa); 902 } else 903 error = ether_ioctl(ifp, command, data); 904 #endif 905 break; 906 case SIOCSIFMTU: 907 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 908 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 909 error = EINVAL; 910 } else { 911 IXGBE_CORE_LOCK(adapter); 912 ifp->if_mtu = ifr->ifr_mtu; 913 adapter->max_frame_size = 914 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 915 ixgbe_init_locked(adapter); 916 IXGBE_CORE_UNLOCK(adapter); 917 } 918 break; 919 case SIOCSIFFLAGS: 920 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 921 IXGBE_CORE_LOCK(adapter); 922 if (ifp->if_flags & IFF_UP) { 923 if ((ifp->if_flags & IFF_RUNNING)) { 924 if ((ifp->if_flags ^ adapter->if_flags) & 925 (IFF_PROMISC | IFF_ALLMULTI)) { 926 ixgbe_set_promisc(adapter); 927 } 928 } else 929 ixgbe_init_locked(adapter); 930 } else 931 if (ifp->if_flags & IFF_RUNNING) 932 ixgbe_stop(adapter); 933 adapter->if_flags = ifp->if_flags; 934 IXGBE_CORE_UNLOCK(adapter); 935 break; 936 case SIOCADDMULTI: 937 case SIOCDELMULTI: 938 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 939 if (ifp->if_flags & IFF_RUNNING) { 940 IXGBE_CORE_LOCK(adapter); 941 ixgbe_disable_intr(adapter); 942 ixgbe_set_multi(adapter); 943 ixgbe_enable_intr(adapter); 944 IXGBE_CORE_UNLOCK(adapter); 945 } 946 break; 947 case SIOCSIFMEDIA: 948 case SIOCGIFMEDIA: 949 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 950 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 951 break; 952 case SIOCSIFCAP: 953 { 954 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 955 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 956 if (mask & IFCAP_HWCSUM) 957 ifp->if_capenable ^= IFCAP_HWCSUM; 958 if (mask & IFCAP_TSO4) 959 ifp->if_capenable ^= IFCAP_TSO4; 960 if (mask & IFCAP_TSO6) 961 ifp->if_capenable ^= IFCAP_TSO6; 962 #if 0 /* NET_LRO */ 963 if (mask & IFCAP_LRO) 964 ifp->if_capenable ^= IFCAP_LRO; 965 #endif 966 if (mask & IFCAP_VLAN_HWTAGGING) 967 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 968 if (mask & IFCAP_VLAN_HWFILTER) 969 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 970 #if 0 /* NET_TSO */ 971 if (mask & IFCAP_VLAN_HWTSO) 972 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 973 #endif 974 if (ifp->if_flags & IFF_RUNNING) { 975 IXGBE_CORE_LOCK(adapter); 976 ixgbe_init_locked(adapter); 977 IXGBE_CORE_UNLOCK(adapter); 978 } 979 #if 0 980 VLAN_CAPABILITIES(ifp); 981 #endif 982 break; 983 } 984 985 default: 986 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 987 error = ether_ioctl(ifp, command, data); 988 break; 989 } 990 991 return (error); 992 } 993 994 /********************************************************************* 995 * Init entry point 996 * 997 * This routine is used in two ways. It is used by the stack as 998 * init entry point in network interface structure. It is also used 999 * by the driver as a hw/sw initialization routine to get to a 1000 * consistent state. 1001 * 1002 * return 0 on success, positive on failure 1003 **********************************************************************/ 1004 #define IXGBE_MHADD_MFS_SHIFT 16 1005 1006 static void 1007 ixgbe_init_locked(struct adapter *adapter) 1008 { 1009 struct ifnet *ifp = adapter->ifp; 1010 device_t dev = adapter->dev; 1011 struct ixgbe_hw *hw = &adapter->hw; 1012 u32 k, txdctl, mhadd, gpie; 1013 u32 rxdctl, rxctrl; 1014 1015 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0); 1016 INIT_DEBUGOUT("ixgbe_init: begin"); 1017 hw->adapter_stopped = FALSE; 1018 ixgbe_stop_adapter(hw); 1019 callout_stop(&adapter->timer); 1020 1021 /* reprogram the RAR[0] in case user changed it. */ 1022 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1023 1024 /* Get the latest mac address, User can use a LAA */ 1025 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, 1026 IXGBE_ETH_LENGTH_OF_ADDRESS); 1027 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 1028 hw->addr_ctrl.rar_used_count = 1; 1029 1030 /* Set the various hardware offload abilities */ 1031 ifp->if_hwassist = 0; 1032 if (ifp->if_capenable & IFCAP_TSO) 1033 ifp->if_hwassist |= CSUM_TSO; 1034 if (ifp->if_capenable & IFCAP_TXCSUM) { 1035 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1036 #if 0 1037 if (hw->mac.type != ixgbe_mac_82598EB) 1038 ifp->if_hwassist |= CSUM_SCTP; 1039 #endif 1040 } 1041 1042 /* Prepare transmit descriptors and buffers */ 1043 if (ixgbe_setup_transmit_structures(adapter)) { 1044 device_printf(dev,"Could not setup transmit structures\n"); 1045 ixgbe_stop(adapter); 1046 return; 1047 } 1048 1049 ixgbe_init_hw(hw); 1050 ixgbe_initialize_transmit_units(adapter); 1051 1052 /* Setup Multicast table */ 1053 ixgbe_set_multi(adapter); 1054 1055 /* 1056 ** Determine the correct mbuf pool 1057 ** for doing jumbo/headersplit 1058 */ 1059 #if 0 /* XXX */ 1060 if (adapter->max_frame_size <= 2048) 1061 adapter->rx_mbuf_sz = MCLBYTES; 1062 else if (adapter->max_frame_size <= 4096) 1063 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1064 else if (adapter->max_frame_size <= 9216) 1065 adapter->rx_mbuf_sz = MJUM9BYTES; 1066 else 1067 adapter->rx_mbuf_sz = MJUM16BYTES; 1068 #else 1069 adapter->rx_mbuf_sz = MCLBYTES; 1070 #endif 1071 1072 /* Prepare receive descriptors and buffers */ 1073 if (ixgbe_setup_receive_structures(adapter)) { 1074 device_printf(dev,"Could not setup receive structures\n"); 1075 ixgbe_stop(adapter); 1076 return; 1077 } 1078 1079 /* Configure RX settings */ 1080 ixgbe_initialize_receive_units(adapter); 1081 1082 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 1083 1084 /* Enable Fan Failure Interrupt */ 1085 gpie |= IXGBE_SDP1_GPIEN; 1086 1087 /* Add for Module detection */ 1088 if (hw->mac.type == ixgbe_mac_82599EB) 1089 gpie |= IXGBE_SDP2_GPIEN; 1090 1091 /* Thermal Failure Detection */ 1092 if (hw->mac.type == ixgbe_mac_X540) 1093 gpie |= IXGBE_SDP0_GPIEN; 1094 1095 if (adapter->msix > 1) { 1096 /* Enable Enhanced MSIX mode */ 1097 gpie |= IXGBE_GPIE_MSIX_MODE; 1098 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1099 IXGBE_GPIE_OCD; 1100 } 1101 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 1102 1103 /* Set MTU size */ 1104 if (ifp->if_mtu > ETHERMTU) { 1105 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 1106 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1107 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1108 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 1109 } 1110 1111 /* Now enable all the queues */ 1112 1113 for (int i = 0; i < adapter->num_queues; i++) { 1114 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1115 txdctl |= IXGBE_TXDCTL_ENABLE; 1116 /* Set WTHRESH to 8, burst writeback */ 1117 txdctl |= (8 << 16); 1118 /* 1119 * When the internal queue falls below PTHRESH (32), 1120 * start prefetching as long as there are at least 1121 * HTHRESH (1) buffers ready. The values are taken 1122 * from the Intel linux driver 3.8.21. 1123 * Prefetching enables tx line rate even with 1 queue. 1124 */ 1125 txdctl |= (32 << 0) | (1 << 8); 1126 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1127 } 1128 1129 for (int i = 0; i < adapter->num_queues; i++) { 1130 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1131 if (hw->mac.type == ixgbe_mac_82598EB) { 1132 /* 1133 ** PTHRESH = 21 1134 ** HTHRESH = 4 1135 ** WTHRESH = 8 1136 */ 1137 rxdctl &= ~0x3FFFFF; 1138 rxdctl |= 0x080420; 1139 } 1140 rxdctl |= IXGBE_RXDCTL_ENABLE; 1141 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1142 for (k = 0; k < 10; k++) { 1143 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1144 IXGBE_RXDCTL_ENABLE) 1145 break; 1146 else 1147 msec_delay(1); 1148 } 1149 wmb(); 1150 #ifdef DEV_NETMAP 1151 /* 1152 * In netmap mode, we must preserve the buffers made 1153 * available to userspace before the if_init() 1154 * (this is true by default on the TX side, because 1155 * init makes all buffers available to userspace). 1156 * 1157 * netmap_reset() and the device specific routines 1158 * (e.g. ixgbe_setup_receive_rings()) map these 1159 * buffers at the end of the NIC ring, so here we 1160 * must set the RDT (tail) register to make sure 1161 * they are not overwritten. 1162 * 1163 * In this driver the NIC ring starts at RDH = 0, 1164 * RDT points to the last slot available for reception (?), 1165 * so RDT = num_rx_desc - 1 means the whole ring is available. 1166 */ 1167 if (ifp->if_capenable & IFCAP_NETMAP) { 1168 struct netmap_adapter *na = NA(adapter->ifp); 1169 struct netmap_kring *kring = &na->rx_rings[i]; 1170 int t = na->num_rx_desc - 1 - kring->nr_hwavail; 1171 1172 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t); 1173 } else 1174 #endif /* DEV_NETMAP */ 1175 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); 1176 } 1177 1178 /* Set up VLAN support and filter */ 1179 ixgbe_setup_vlan_hw_support(adapter); 1180 1181 /* Enable Receive engine */ 1182 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1183 if (hw->mac.type == ixgbe_mac_82598EB) 1184 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1185 rxctrl |= IXGBE_RXCTRL_RXEN; 1186 ixgbe_enable_rx_dma(hw, rxctrl); 1187 1188 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1189 1190 /* Set up MSI/X routing */ 1191 if (ixgbe_enable_msix) { 1192 ixgbe_configure_ivars(adapter); 1193 /* Set up auto-mask */ 1194 if (hw->mac.type == ixgbe_mac_82598EB) 1195 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1196 else { 1197 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1198 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1199 } 1200 } else { /* Simple settings for Legacy/MSI */ 1201 ixgbe_set_ivar(adapter, 0, 0, 0); 1202 ixgbe_set_ivar(adapter, 0, 0, 1); 1203 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1204 } 1205 1206 #ifdef IXGBE_FDIR 1207 /* Init Flow director */ 1208 if (hw->mac.type != ixgbe_mac_82598EB) { 1209 u32 hdrm = 32 << fdir_pballoc; 1210 1211 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL); 1212 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); 1213 } 1214 #endif 1215 1216 /* 1217 ** Check on any SFP devices that 1218 ** need to be kick-started 1219 */ 1220 if (hw->phy.type == ixgbe_phy_none) { 1221 int err = hw->phy.ops.identify(hw); 1222 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1223 device_printf(dev, 1224 "Unsupported SFP+ module type was detected.\n"); 1225 return; 1226 } 1227 } 1228 1229 /* Set moderation on the Link interrupt */ 1230 ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR); 1231 1232 /* Config/Enable Link */ 1233 ixgbe_config_link(adapter); 1234 1235 /* Hardware Packet Buffer & Flow Control setup */ 1236 { 1237 u32 rxpb, frame, size, tmp; 1238 1239 frame = adapter->max_frame_size; 1240 1241 /* Calculate High Water */ 1242 if (hw->mac.type == ixgbe_mac_X540) 1243 tmp = IXGBE_DV_X540(frame, frame); 1244 else 1245 tmp = IXGBE_DV(frame, frame); 1246 size = IXGBE_BT2KB(tmp); 1247 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 1248 hw->fc.high_water[0] = rxpb - size; 1249 1250 /* Now calculate Low Water */ 1251 if (hw->mac.type == ixgbe_mac_X540) 1252 tmp = IXGBE_LOW_DV_X540(frame); 1253 else 1254 tmp = IXGBE_LOW_DV(frame); 1255 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 1256 1257 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full; 1258 hw->fc.pause_time = IXGBE_FC_PAUSE; 1259 hw->fc.send_xon = TRUE; 1260 } 1261 /* Initialize the FC settings */ 1262 ixgbe_start_hw(hw); 1263 1264 /* And now turn on interrupts */ 1265 ixgbe_enable_intr(adapter); 1266 1267 /* Now inform the stack we're ready */ 1268 ifp->if_flags |= IFF_RUNNING; 1269 ifq_clr_oactive(&ifp->if_snd); 1270 1271 return; 1272 } 1273 1274 static void 1275 ixgbe_init(void *arg) 1276 { 1277 struct adapter *adapter = arg; 1278 1279 IXGBE_CORE_LOCK(adapter); 1280 ixgbe_init_locked(adapter); 1281 IXGBE_CORE_UNLOCK(adapter); 1282 return; 1283 } 1284 1285 1286 /* 1287 ** 1288 ** MSIX Interrupt Handlers and Tasklets 1289 ** 1290 */ 1291 1292 static inline void 1293 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 1294 { 1295 struct ixgbe_hw *hw = &adapter->hw; 1296 u64 queue = (u64)(1 << vector); 1297 u32 mask; 1298 1299 if (hw->mac.type == ixgbe_mac_82598EB) { 1300 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1301 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 1302 } else { 1303 mask = (queue & 0xFFFFFFFF); 1304 if (mask) 1305 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 1306 mask = (queue >> 32); 1307 if (mask) 1308 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 1309 } 1310 } 1311 1312 static inline void 1313 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 1314 { 1315 struct ixgbe_hw *hw = &adapter->hw; 1316 u64 queue = (u64)(1 << vector); 1317 u32 mask; 1318 1319 if (hw->mac.type == ixgbe_mac_82598EB) { 1320 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1321 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 1322 } else { 1323 mask = (queue & 0xFFFFFFFF); 1324 if (mask) 1325 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 1326 mask = (queue >> 32); 1327 if (mask) 1328 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 1329 } 1330 } 1331 1332 static inline void 1333 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 1334 { 1335 u32 mask; 1336 1337 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 1338 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1339 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 1340 } else { 1341 mask = (queues & 0xFFFFFFFF); 1342 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 1343 mask = (queues >> 32); 1344 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 1345 } 1346 } 1347 1348 /********************************************************************* 1349 * 1350 * Legacy Interrupt Service routine 1351 * 1352 **********************************************************************/ 1353 1354 static void 1355 ixgbe_legacy_irq(void *arg) 1356 { 1357 struct ix_queue *que = arg; 1358 struct adapter *adapter = que->adapter; 1359 struct ixgbe_hw *hw = &adapter->hw; 1360 struct tx_ring *txr = adapter->tx_rings; 1361 u32 reg_eicr; 1362 1363 1364 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1365 1366 ++que->irqs; 1367 if (reg_eicr == 0) { 1368 ixgbe_enable_intr(adapter); 1369 return; 1370 } 1371 1372 ixgbe_rxeof(que, adapter->rx_process_limit); 1373 1374 IXGBE_TX_LOCK(txr); 1375 ixgbe_txeof(txr); 1376 if (!ifq_is_empty(&adapter->ifp->if_snd)) 1377 ixgbe_start_locked(txr, adapter->ifp); 1378 IXGBE_TX_UNLOCK(txr); 1379 1380 /* Check for fan failure */ 1381 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1382 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1383 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1384 "REPLACE IMMEDIATELY!!\n"); 1385 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1386 } 1387 1388 /* Link status change */ 1389 if (reg_eicr & IXGBE_EICR_LSC) { 1390 ixgbe_check_link(&adapter->hw, 1391 &adapter->link_speed, &adapter->link_up, 0); 1392 ixgbe_update_link_status(adapter); 1393 } 1394 1395 ixgbe_enable_intr(adapter); 1396 } 1397 1398 1399 /********************************************************************* 1400 * 1401 * MSIX Queue Interrupt Service routine 1402 * 1403 **********************************************************************/ 1404 void 1405 ixgbe_msix_que(void *arg) 1406 { 1407 struct ix_queue *que = arg; 1408 struct adapter *adapter = que->adapter; 1409 struct tx_ring *txr = que->txr; 1410 1411 ixgbe_disable_queue(adapter, que->msix); 1412 ++que->irqs; 1413 1414 ixgbe_rxeof(que, adapter->rx_process_limit); 1415 1416 IXGBE_TX_LOCK(txr); 1417 ixgbe_txeof(txr); 1418 if (!ifq_is_empty(&adapter->ifp->if_snd)) 1419 ixgbe_start_locked(txr, adapter->ifp); 1420 IXGBE_TX_UNLOCK(txr); 1421 1422 /* Reenable this interrupt */ 1423 ixgbe_enable_queue(adapter, que->msix); 1424 } 1425 1426 1427 static void 1428 ixgbe_msix_link(void *arg) 1429 { 1430 struct adapter *adapter = arg; 1431 struct ixgbe_hw *hw = &adapter->hw; 1432 u32 reg_eicr; 1433 1434 ++adapter->link_irq; 1435 1436 /* First get the cause */ 1437 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1438 /* Clear interrupt with write */ 1439 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1440 1441 /* Link status change */ 1442 if (reg_eicr & IXGBE_EICR_LSC) { 1443 ixgbe_check_link(&adapter->hw, 1444 &adapter->link_speed, &adapter->link_up, 0); 1445 ixgbe_update_link_status(adapter); 1446 } 1447 1448 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 1449 #ifdef IXGBE_FDIR 1450 if (reg_eicr & IXGBE_EICR_FLOW_DIR) { 1451 /* This is probably overkill :) */ 1452 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) 1453 return; 1454 /* Disable the interrupt */ 1455 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR); 1456 taskqueue_enqueue(adapter->tq, &adapter->fdir_task); 1457 } else 1458 #endif 1459 if (reg_eicr & IXGBE_EICR_ECC) { 1460 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " 1461 "Please Reboot!!\n"); 1462 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1463 } else 1464 1465 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1466 /* Clear the interrupt */ 1467 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1468 taskqueue_enqueue(adapter->tq, &adapter->msf_task); 1469 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1470 /* Clear the interrupt */ 1471 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1472 taskqueue_enqueue(adapter->tq, &adapter->mod_task); 1473 } 1474 } 1475 1476 /* Check for fan failure */ 1477 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1478 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1479 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1480 "REPLACE IMMEDIATELY!!\n"); 1481 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1482 } 1483 1484 /* Check for over temp condition */ 1485 if ((hw->mac.type == ixgbe_mac_X540) && 1486 (reg_eicr & IXGBE_EICR_GPI_SDP0)) { 1487 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! " 1488 "PHY IS SHUT DOWN!!\n"); 1489 device_printf(adapter->dev, "System shutdown required\n"); 1490 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0); 1491 } 1492 1493 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1494 return; 1495 } 1496 1497 /********************************************************************* 1498 * 1499 * Media Ioctl callback 1500 * 1501 * This routine is called whenever the user queries the status of 1502 * the interface using ifconfig. 1503 * 1504 **********************************************************************/ 1505 static void 1506 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1507 { 1508 struct adapter *adapter = ifp->if_softc; 1509 1510 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1511 1512 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1513 ixgbe_update_link_status(adapter); 1514 1515 ifmr->ifm_status = IFM_AVALID; 1516 ifmr->ifm_active = IFM_ETHER; 1517 1518 if (!adapter->link_active) 1519 return; 1520 1521 ifmr->ifm_status |= IFM_ACTIVE; 1522 1523 switch (adapter->link_speed) { 1524 case IXGBE_LINK_SPEED_100_FULL: 1525 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1526 break; 1527 case IXGBE_LINK_SPEED_1GB_FULL: 1528 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1529 break; 1530 case IXGBE_LINK_SPEED_10GB_FULL: 1531 ifmr->ifm_active |= adapter->optics | IFM_FDX; 1532 break; 1533 } 1534 1535 return; 1536 } 1537 1538 /********************************************************************* 1539 * 1540 * Media Ioctl callback 1541 * 1542 * This routine is called when the user changes speed/duplex using 1543 * media/mediopt option with ifconfig. 1544 * 1545 **********************************************************************/ 1546 static int 1547 ixgbe_media_change(struct ifnet * ifp) 1548 { 1549 struct adapter *adapter = ifp->if_softc; 1550 struct ifmedia *ifm = &adapter->media; 1551 1552 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1553 1554 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1555 return (EINVAL); 1556 1557 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1558 case IFM_AUTO: 1559 adapter->hw.phy.autoneg_advertised = 1560 IXGBE_LINK_SPEED_100_FULL | 1561 IXGBE_LINK_SPEED_1GB_FULL | 1562 IXGBE_LINK_SPEED_10GB_FULL; 1563 break; 1564 default: 1565 device_printf(adapter->dev, "Only auto media type\n"); 1566 return (EINVAL); 1567 } 1568 1569 return (0); 1570 } 1571 1572 /********************************************************************* 1573 * 1574 * This routine maps the mbufs to tx descriptors, allowing the 1575 * TX engine to transmit the packets. 1576 * - return 0 on success, positive on failure 1577 * 1578 **********************************************************************/ 1579 1580 static int 1581 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1582 { 1583 struct adapter *adapter = txr->adapter; 1584 u32 olinfo_status = 0, cmd_type_len; 1585 u32 paylen = 0; 1586 int i, j, error, nsegs, maxsegs; 1587 int first, last = 0; 1588 struct mbuf *m_head; 1589 bus_dma_segment_t segs[adapter->num_segs]; 1590 bus_dmamap_t map; 1591 struct ixgbe_tx_buf *txbuf; 1592 union ixgbe_adv_tx_desc *txd = NULL; 1593 1594 m_head = *m_headp; 1595 1596 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1597 error = ixgbe_tso_pullup(txr, m_headp); 1598 if (error) 1599 return error; 1600 m_head = *m_headp; 1601 } 1602 1603 /* Basic descriptor defines */ 1604 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1605 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1606 1607 if (m_head->m_flags & M_VLANTAG) 1608 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1609 1610 /* 1611 * Important to capture the first descriptor 1612 * used because it will contain the index of 1613 * the one we tell the hardware to report back 1614 */ 1615 first = txr->next_avail_desc; 1616 txbuf = &txr->tx_buffers[first]; 1617 map = txbuf->map; 1618 1619 /* 1620 * Map the packet for DMA. 1621 */ 1622 maxsegs = txr->tx_avail - IXGBE_TX_RESERVED; 1623 if (maxsegs > adapter->num_segs) 1624 maxsegs = adapter->num_segs; 1625 1626 error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp, 1627 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1628 if (error) { 1629 if (error == ENOBUFS) 1630 adapter->mbuf_defrag_failed++; 1631 else 1632 adapter->no_tx_dma_setup++; 1633 1634 m_freem(*m_headp); 1635 *m_headp = NULL; 1636 return (error); 1637 } 1638 1639 /* Make certain there are enough descriptors */ 1640 if (nsegs > txr->tx_avail - 2) { 1641 txr->no_desc_avail++; 1642 error = ENOBUFS; 1643 goto xmit_fail; 1644 } 1645 m_head = *m_headp; 1646 1647 /* 1648 ** Set up the appropriate offload context 1649 ** this becomes the first descriptor of 1650 ** a packet. 1651 */ 1652 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1653 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) { 1654 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1655 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1656 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1657 ++adapter->tso_tx; 1658 } else 1659 return (ENXIO); 1660 } else if (ixgbe_tx_ctx_setup(txr, m_head)) 1661 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1662 1663 #ifdef IXGBE_IEEE1588 1664 /* This is changing soon to an mtag detection */ 1665 if (we detect this mbuf has a TSTAMP mtag) 1666 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; 1667 #endif 1668 1669 #ifdef IXGBE_FDIR 1670 /* Do the flow director magic */ 1671 if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 1672 ++txr->atr_count; 1673 if (txr->atr_count >= atr_sample_rate) { 1674 ixgbe_atr(txr, m_head); 1675 txr->atr_count = 0; 1676 } 1677 } 1678 #endif 1679 /* Record payload length */ 1680 if (paylen == 0) 1681 olinfo_status |= m_head->m_pkthdr.len << 1682 IXGBE_ADVTXD_PAYLEN_SHIFT; 1683 1684 i = txr->next_avail_desc; 1685 for (j = 0; j < nsegs; j++) { 1686 bus_size_t seglen; 1687 bus_addr_t segaddr; 1688 1689 txbuf = &txr->tx_buffers[i]; 1690 txd = &txr->tx_base[i]; 1691 seglen = segs[j].ds_len; 1692 segaddr = htole64(segs[j].ds_addr); 1693 1694 txd->read.buffer_addr = segaddr; 1695 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1696 cmd_type_len |seglen); 1697 txd->read.olinfo_status = htole32(olinfo_status); 1698 last = i; /* descriptor that will get completion IRQ */ 1699 1700 if (++i == adapter->num_tx_desc) 1701 i = 0; 1702 1703 txbuf->m_head = NULL; 1704 txbuf->eop_index = -1; 1705 } 1706 1707 txd->read.cmd_type_len |= 1708 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1709 txr->tx_avail -= nsegs; 1710 txr->next_avail_desc = i; 1711 1712 txbuf->m_head = m_head; 1713 /* Swap the dma map between the first and last descriptor */ 1714 txr->tx_buffers[first].map = txbuf->map; 1715 txbuf->map = map; 1716 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1717 1718 /* Set the index of the descriptor that will be marked done */ 1719 txbuf = &txr->tx_buffers[first]; 1720 txbuf->eop_index = last; 1721 1722 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1723 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1724 /* 1725 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1726 * hardware that this frame is available to transmit. 1727 */ 1728 ++txr->total_packets; 1729 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); 1730 1731 return (0); 1732 1733 xmit_fail: 1734 bus_dmamap_unload(txr->txtag, txbuf->map); 1735 return (error); 1736 1737 } 1738 1739 static void 1740 ixgbe_set_promisc(struct adapter *adapter) 1741 { 1742 u_int32_t reg_rctl; 1743 struct ifnet *ifp = adapter->ifp; 1744 1745 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1746 reg_rctl &= (~IXGBE_FCTRL_UPE); 1747 reg_rctl &= (~IXGBE_FCTRL_MPE); 1748 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1749 1750 if (ifp->if_flags & IFF_PROMISC) { 1751 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1752 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1753 } else if (ifp->if_flags & IFF_ALLMULTI) { 1754 reg_rctl |= IXGBE_FCTRL_MPE; 1755 reg_rctl &= ~IXGBE_FCTRL_UPE; 1756 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1757 } 1758 return; 1759 } 1760 1761 1762 /********************************************************************* 1763 * Multicast Update 1764 * 1765 * This routine is called whenever multicast address list is updated. 1766 * 1767 **********************************************************************/ 1768 #define IXGBE_RAR_ENTRIES 16 1769 1770 static void 1771 ixgbe_set_multi(struct adapter *adapter) 1772 { 1773 u32 fctrl; 1774 u8 *mta; 1775 u8 *update_ptr; 1776 struct ifmultiaddr *ifma; 1777 int mcnt = 0; 1778 struct ifnet *ifp = adapter->ifp; 1779 1780 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1781 1782 mta = adapter->mta; 1783 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1784 MAX_NUM_MULTICAST_ADDRESSES); 1785 1786 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1787 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1788 if (ifp->if_flags & IFF_PROMISC) 1789 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1790 else if (ifp->if_flags & IFF_ALLMULTI) { 1791 fctrl |= IXGBE_FCTRL_MPE; 1792 fctrl &= ~IXGBE_FCTRL_UPE; 1793 } else 1794 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1795 1796 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1797 1798 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1799 if (ifma->ifma_addr->sa_family != AF_LINK) 1800 continue; 1801 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1802 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1803 IXGBE_ETH_LENGTH_OF_ADDRESS); 1804 mcnt++; 1805 } 1806 1807 update_ptr = mta; 1808 ixgbe_update_mc_addr_list(&adapter->hw, 1809 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE); 1810 1811 return; 1812 } 1813 1814 /* 1815 * This is an iterator function now needed by the multicast 1816 * shared code. It simply feeds the shared code routine the 1817 * addresses in the array of ixgbe_set_multi() one by one. 1818 */ 1819 static u8 * 1820 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1821 { 1822 u8 *addr = *update_ptr; 1823 u8 *newptr; 1824 *vmdq = 0; 1825 1826 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1827 *update_ptr = newptr; 1828 return addr; 1829 } 1830 1831 1832 /********************************************************************* 1833 * Timer routine 1834 * 1835 * This routine checks for link status,updates statistics, 1836 * and runs the watchdog check. 1837 * 1838 **********************************************************************/ 1839 1840 static void 1841 ixgbe_local_timer(void *arg) 1842 { 1843 struct adapter *adapter = arg; 1844 device_t dev = adapter->dev; 1845 struct ifnet *ifp = adapter->ifp; 1846 struct ix_queue *que = adapter->queues; 1847 struct tx_ring *txr = adapter->tx_rings; 1848 int hung, busy, paused; 1849 1850 IXGBE_CORE_LOCK(adapter); 1851 hung = busy = paused = 0; 1852 1853 /* Check for pluggable optics */ 1854 if (adapter->sfp_probe) 1855 if (!ixgbe_sfp_probe(adapter)) 1856 goto out; /* Nothing to do */ 1857 1858 ixgbe_update_link_status(adapter); 1859 ixgbe_update_stats_counters(adapter); 1860 1861 /* 1862 * If the interface has been paused 1863 * then don't do the watchdog check 1864 */ 1865 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1866 paused = 1; 1867 1868 /* 1869 ** Check the TX queues status 1870 ** - central locked handling of OACTIVE 1871 ** - watchdog only if all queues show hung 1872 */ 1873 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 1874 if ((txr->queue_status & IXGBE_QUEUE_HUNG) && 1875 (paused == 0)) 1876 ++hung; 1877 if (txr->queue_status & IXGBE_QUEUE_DEPLETED) 1878 ++busy; 1879 } 1880 /* Only truely watchdog if all queues show hung */ 1881 if (hung == adapter->num_queues) 1882 goto watchdog; 1883 /* Only turn off the stack flow when ALL are depleted */ 1884 if (busy == adapter->num_queues) 1885 ifq_set_oactive(&ifp->if_snd); 1886 else if (ifq_is_oactive(&ifp->if_snd) && (busy < adapter->num_queues)) 1887 ifq_clr_oactive(&ifp->if_snd); 1888 1889 out: 1890 ixgbe_rearm_queues(adapter, adapter->que_mask); 1891 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1892 IXGBE_CORE_UNLOCK(adapter); 1893 return; 1894 1895 watchdog: 1896 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1897 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1898 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), 1899 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); 1900 device_printf(dev,"TX(%d) desc avail = %d," 1901 "Next TX to Clean = %d\n", 1902 txr->me, txr->tx_avail, txr->next_to_clean); 1903 adapter->ifp->if_flags &= ~IFF_RUNNING; 1904 adapter->watchdog_events++; 1905 ixgbe_init_locked(adapter); 1906 1907 IXGBE_CORE_UNLOCK(adapter); 1908 } 1909 1910 /* 1911 ** Note: this routine updates the OS on the link state 1912 ** the real check of the hardware only happens with 1913 ** a link interrupt. 1914 */ 1915 static void 1916 ixgbe_update_link_status(struct adapter *adapter) 1917 { 1918 struct ifnet *ifp = adapter->ifp; 1919 struct tx_ring *txr = adapter->tx_rings; 1920 device_t dev = adapter->dev; 1921 1922 1923 if (adapter->link_up){ 1924 if (adapter->link_active == FALSE) { 1925 if (bootverbose) 1926 device_printf(dev,"Link is up %d Gbps %s \n", 1927 ((adapter->link_speed == 128)? 10:1), 1928 "Full Duplex"); 1929 adapter->link_active = TRUE; 1930 /* Update any Flow Control changes */ 1931 ixgbe_fc_enable(&adapter->hw); 1932 ifp->if_link_state = LINK_STATE_UP; 1933 if_link_state_change(ifp); 1934 } 1935 } else { /* Link down */ 1936 if (adapter->link_active == TRUE) { 1937 if (bootverbose) 1938 device_printf(dev,"Link is Down\n"); 1939 ifp->if_link_state = LINK_STATE_DOWN; 1940 if_link_state_change(ifp); 1941 adapter->link_active = FALSE; 1942 for (int i = 0; i < adapter->num_queues; 1943 i++, txr++) 1944 txr->queue_status = IXGBE_QUEUE_IDLE; 1945 } 1946 } 1947 1948 return; 1949 } 1950 1951 1952 /********************************************************************* 1953 * 1954 * This routine disables all traffic on the adapter by issuing a 1955 * global reset on the MAC and deallocates TX/RX buffers. 1956 * 1957 **********************************************************************/ 1958 1959 static void 1960 ixgbe_stop(void *arg) 1961 { 1962 struct ifnet *ifp; 1963 struct adapter *adapter = arg; 1964 struct ixgbe_hw *hw = &adapter->hw; 1965 ifp = adapter->ifp; 1966 1967 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0); 1968 1969 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1970 ixgbe_disable_intr(adapter); 1971 callout_stop(&adapter->timer); 1972 1973 /* Let the stack know...*/ 1974 ifp->if_flags &= ~IFF_RUNNING; 1975 ifq_clr_oactive(&ifp->if_snd); 1976 1977 ixgbe_reset_hw(hw); 1978 hw->adapter_stopped = FALSE; 1979 ixgbe_stop_adapter(hw); 1980 /* Turn off the laser */ 1981 if (hw->phy.multispeed_fiber) 1982 ixgbe_disable_tx_laser(hw); 1983 1984 /* reprogram the RAR[0] in case user changed it. */ 1985 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1986 1987 return; 1988 } 1989 1990 1991 /********************************************************************* 1992 * 1993 * Determine hardware revision. 1994 * 1995 **********************************************************************/ 1996 static void 1997 ixgbe_identify_hardware(struct adapter *adapter) 1998 { 1999 device_t dev = adapter->dev; 2000 struct ixgbe_hw *hw = &adapter->hw; 2001 2002 /* Save off the information about this board */ 2003 hw->vendor_id = pci_get_vendor(dev); 2004 hw->device_id = pci_get_device(dev); 2005 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); 2006 hw->subsystem_vendor_id = 2007 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2008 hw->subsystem_device_id = 2009 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2010 2011 /* We need this here to set the num_segs below */ 2012 ixgbe_set_mac_type(hw); 2013 2014 /* Pick up the 82599 and VF settings */ 2015 if (hw->mac.type != ixgbe_mac_82598EB) { 2016 hw->phy.smart_speed = ixgbe_smart_speed; 2017 adapter->num_segs = IXGBE_82599_SCATTER; 2018 } else 2019 adapter->num_segs = IXGBE_82598_SCATTER; 2020 2021 return; 2022 } 2023 2024 /********************************************************************* 2025 * 2026 * Determine optic type 2027 * 2028 **********************************************************************/ 2029 static void 2030 ixgbe_setup_optics(struct adapter *adapter) 2031 { 2032 struct ixgbe_hw *hw = &adapter->hw; 2033 int layer; 2034 2035 layer = ixgbe_get_supported_physical_layer(hw); 2036 2037 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 2038 adapter->optics = IFM_10G_T; 2039 return; 2040 } 2041 2042 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 2043 adapter->optics = IFM_1000_T; 2044 return; 2045 } 2046 2047 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR | 2048 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) { 2049 adapter->optics = IFM_10G_LR; 2050 return; 2051 } 2052 2053 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 2054 adapter->optics = IFM_10G_SR; 2055 return; 2056 } 2057 2058 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) { 2059 adapter->optics = IFM_10G_TWINAX; 2060 return; 2061 } 2062 2063 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 | 2064 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) { 2065 adapter->optics = IFM_10G_CX4; 2066 return; 2067 } 2068 2069 /* If we get here just set the default */ 2070 adapter->optics = IFM_ETHER | IFM_AUTO; 2071 return; 2072 } 2073 2074 /********************************************************************* 2075 * 2076 * Setup the Legacy or MSI Interrupt handler 2077 * 2078 **********************************************************************/ 2079 static int 2080 ixgbe_allocate_legacy(struct adapter *adapter) 2081 { 2082 device_t dev = adapter->dev; 2083 struct ix_queue *que = adapter->queues; 2084 int error, rid = 0; 2085 unsigned int intr_flags; 2086 2087 /* MSI RID at 1 */ 2088 if (adapter->msix == 1) 2089 rid = 1; 2090 2091 /* Try allocating a MSI interrupt first */ 2092 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable, 2093 &rid, &intr_flags); 2094 2095 /* We allocate a single interrupt resource */ 2096 adapter->res = bus_alloc_resource_any(dev, 2097 SYS_RES_IRQ, &rid, intr_flags); 2098 if (adapter->res == NULL) { 2099 device_printf(dev, "Unable to allocate bus resource: " 2100 "interrupt\n"); 2101 return (ENXIO); 2102 } 2103 2104 /* Tasklets for Link, SFP and Multispeed Fiber */ 2105 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter); 2106 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter); 2107 #ifdef IXGBE_FDIR 2108 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter); 2109 #endif 2110 2111 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE, 2112 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) { 2113 device_printf(dev, "Failed to register fast interrupt " 2114 "handler: %d\n", error); 2115 taskqueue_free(adapter->tq); 2116 adapter->tq = NULL; 2117 return (error); 2118 } 2119 /* For simplicity in the handlers */ 2120 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK; 2121 2122 return (0); 2123 } 2124 2125 2126 /********************************************************************* 2127 * 2128 * Setup MSIX Interrupt resources and handlers 2129 * 2130 **********************************************************************/ 2131 static int 2132 ixgbe_allocate_msix(struct adapter *adapter) 2133 { 2134 device_t dev = adapter->dev; 2135 struct ix_queue *que = adapter->queues; 2136 int error, rid, vector = 0; 2137 char desc[16]; 2138 2139 error = pci_setup_msix(dev); 2140 if (error) { 2141 device_printf(dev, "MSI-X setup failed\n"); 2142 return (error); 2143 } 2144 2145 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2146 rid = vector + 1; 2147 2148 /* 2149 ** Bind the msix vector, and thus the 2150 ** ring to the corresponding cpu. 2151 */ 2152 error = pci_alloc_msix_vector(dev, vector, &rid, i); 2153 if (error) { 2154 device_printf(dev, "pci_alloc_msix_vector failed\n"); 2155 return (error); 2156 } 2157 2158 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2159 RF_SHAREABLE | RF_ACTIVE); 2160 if (que->res == NULL) { 2161 device_printf(dev,"Unable to allocate" 2162 " bus resource: que interrupt [%d]\n", vector); 2163 return (ENXIO); 2164 } 2165 /* Set the handler function */ 2166 ksnprintf(desc, sizeof(desc), "%s que %d", 2167 device_get_nameunit(dev), i); 2168 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE, 2169 ixgbe_msix_que, que, &que->tag, &que->serializer, desc); 2170 if (error) { 2171 que->res = NULL; 2172 device_printf(dev, "Failed to register QUE handler"); 2173 return (error); 2174 } 2175 que->msix = vector; 2176 adapter->que_mask |= (u64)(1 << que->msix); 2177 } 2178 2179 /* and Link, bind vector to cpu #0 */ 2180 rid = vector + 1; 2181 error = pci_alloc_msix_vector(dev, vector, &rid, 0); 2182 if (error) { 2183 device_printf(dev, "pci_alloc_msix_vector failed\n"); 2184 return (error); 2185 } 2186 adapter->res = bus_alloc_resource_any(dev, 2187 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2188 if (!adapter->res) { 2189 device_printf(dev,"Unable to allocate" 2190 " bus resource: Link interrupt [%d]\n", rid); 2191 return (ENXIO); 2192 } 2193 /* Set the link handler function */ 2194 error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE, 2195 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer, 2196 "link"); 2197 if (error) { 2198 adapter->res = NULL; 2199 device_printf(dev, "Failed to register LINK handler"); 2200 return (error); 2201 } 2202 pci_enable_msix(dev); 2203 2204 adapter->linkvec = vector; 2205 /* Tasklets for Link, SFP and Multispeed Fiber */ 2206 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter); 2207 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter); 2208 #ifdef IXGBE_FDIR 2209 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter); 2210 #endif 2211 2212 return (0); 2213 } 2214 2215 /* 2216 * Setup Either MSI/X or MSI 2217 */ 2218 static int 2219 ixgbe_setup_msix(struct adapter *adapter) 2220 { 2221 device_t dev = adapter->dev; 2222 int rid, want, queues, msgs; 2223 2224 /* Override by tuneable */ 2225 if (ixgbe_enable_msix == 0) 2226 goto msi; 2227 2228 /* First try MSI/X */ 2229 rid = PCIR_BAR(MSIX_82598_BAR); 2230 adapter->msix_mem = bus_alloc_resource_any(dev, 2231 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2232 if (!adapter->msix_mem) { 2233 rid += 4; /* 82599 maps in higher BAR */ 2234 adapter->msix_mem = bus_alloc_resource_any(dev, 2235 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2236 } 2237 if (!adapter->msix_mem) { 2238 /* May not be enabled */ 2239 device_printf(adapter->dev, 2240 "Unable to map MSIX table \n"); 2241 goto msi; 2242 } 2243 2244 msgs = pci_msix_count(dev); 2245 if (msgs == 0) { /* system has msix disabled */ 2246 bus_release_resource(dev, SYS_RES_MEMORY, 2247 rid, adapter->msix_mem); 2248 adapter->msix_mem = NULL; 2249 goto msi; 2250 } 2251 2252 /* Figure out a reasonable auto config value */ 2253 queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus; 2254 2255 if (ixgbe_num_queues != 0) 2256 queues = ixgbe_num_queues; 2257 /* Set max queues to 8 when autoconfiguring */ 2258 else if ((ixgbe_num_queues == 0) && (queues > 8)) 2259 queues = 8; 2260 2261 /* 2262 ** Want one vector (RX/TX pair) per queue 2263 ** plus an additional for Link. 2264 */ 2265 want = queues + 1; 2266 if (msgs >= want) 2267 msgs = want; 2268 else { 2269 device_printf(adapter->dev, 2270 "MSIX Configuration Problem, " 2271 "%d vectors but %d queues wanted!\n", 2272 msgs, want); 2273 return (0); /* Will go to Legacy setup */ 2274 } 2275 if (msgs) { 2276 device_printf(adapter->dev, 2277 "Using MSIX interrupts with %d vectors\n", msgs); 2278 adapter->num_queues = queues; 2279 return (msgs); 2280 } 2281 msi: 2282 msgs = pci_msi_count(dev); 2283 return (msgs); 2284 } 2285 2286 2287 static int 2288 ixgbe_allocate_pci_resources(struct adapter *adapter) 2289 { 2290 int rid; 2291 device_t dev = adapter->dev; 2292 2293 rid = PCIR_BAR(0); 2294 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2295 &rid, RF_ACTIVE); 2296 2297 if (!(adapter->pci_mem)) { 2298 device_printf(dev,"Unable to allocate bus resource: memory\n"); 2299 return (ENXIO); 2300 } 2301 2302 adapter->osdep.mem_bus_space_tag = 2303 rman_get_bustag(adapter->pci_mem); 2304 adapter->osdep.mem_bus_space_handle = 2305 rman_get_bushandle(adapter->pci_mem); 2306 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 2307 2308 /* Legacy defaults */ 2309 adapter->num_queues = 1; 2310 adapter->hw.back = &adapter->osdep; 2311 2312 /* 2313 ** Now setup MSI or MSI/X, should 2314 ** return us the number of supported 2315 ** vectors. (Will be 1 for MSI) 2316 */ 2317 adapter->msix = ixgbe_setup_msix(adapter); 2318 return (0); 2319 } 2320 2321 static void 2322 ixgbe_free_pci_resources(struct adapter * adapter) 2323 { 2324 struct ix_queue *que = adapter->queues; 2325 device_t dev = adapter->dev; 2326 int rid, memrid; 2327 2328 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 2329 memrid = PCIR_BAR(MSIX_82598_BAR); 2330 else 2331 memrid = PCIR_BAR(MSIX_82599_BAR); 2332 2333 /* 2334 ** There is a slight possibility of a failure mode 2335 ** in attach that will result in entering this function 2336 ** before interrupt resources have been initialized, and 2337 ** in that case we do not want to execute the loops below 2338 ** We can detect this reliably by the state of the adapter 2339 ** res pointer. 2340 */ 2341 if (adapter->res == NULL) 2342 goto mem; 2343 2344 /* 2345 ** Release all msix queue resources: 2346 */ 2347 for (int i = 0; i < adapter->num_queues; i++, que++) { 2348 rid = que->msix + 1; 2349 if (que->tag != NULL) { 2350 bus_teardown_intr(dev, que->res, que->tag); 2351 que->tag = NULL; 2352 } 2353 if (que->res != NULL) 2354 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2355 } 2356 2357 2358 /* Clean the Legacy or Link interrupt last */ 2359 if (adapter->linkvec) /* we are doing MSIX */ 2360 rid = adapter->linkvec + 1; 2361 else 2362 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2363 2364 if (adapter->tag != NULL) { 2365 bus_teardown_intr(dev, adapter->res, adapter->tag); 2366 adapter->tag = NULL; 2367 } 2368 if (adapter->res != NULL) 2369 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2370 if (adapter->intr_type == PCI_INTR_TYPE_MSI) 2371 pci_release_msi(adapter->dev); 2372 2373 mem: 2374 if (adapter->msix) 2375 pci_release_msi(dev); 2376 2377 if (adapter->msix_mem != NULL) 2378 bus_release_resource(dev, SYS_RES_MEMORY, 2379 memrid, adapter->msix_mem); 2380 2381 if (adapter->pci_mem != NULL) 2382 bus_release_resource(dev, SYS_RES_MEMORY, 2383 PCIR_BAR(0), adapter->pci_mem); 2384 2385 return; 2386 } 2387 2388 /********************************************************************* 2389 * 2390 * Setup networking device structure and register an interface. 2391 * 2392 **********************************************************************/ 2393 static int 2394 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2395 { 2396 struct ixgbe_hw *hw = &adapter->hw; 2397 struct ifnet *ifp; 2398 2399 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2400 2401 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2402 if (ifp == NULL) { 2403 device_printf(dev, "can not allocate ifnet structure\n"); 2404 return (-1); 2405 } 2406 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2407 ifp->if_baudrate = 1000000000; 2408 ifp->if_init = ixgbe_init; 2409 ifp->if_softc = adapter; 2410 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2411 ifp->if_ioctl = ixgbe_ioctl; 2412 ifp->if_start = ixgbe_start; 2413 #if 0 /* __FreeBSD_version >= 800000 */ 2414 ifp->if_transmit = ixgbe_mq_start; 2415 ifp->if_qflush = ixgbe_qflush; 2416 #endif 2417 ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 2); 2418 2419 ether_ifattach(ifp, adapter->hw.mac.addr, NULL); 2420 2421 adapter->max_frame_size = 2422 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2423 2424 /* 2425 * Tell the upper layer(s) we support long frames. 2426 */ 2427 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2428 2429 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM; 2430 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2431 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 2432 #if 0 /* NET_TSO */ 2433 | IFCAP_VLAN_HWTSO 2434 #endif 2435 | IFCAP_VLAN_MTU; 2436 ifp->if_capenable = ifp->if_capabilities; 2437 2438 /* Don't enable LRO by default */ 2439 #if 0 /* NET_LRO */ 2440 ifp->if_capabilities |= IFCAP_LRO; 2441 #endif 2442 2443 /* 2444 ** Don't turn this on by default, if vlans are 2445 ** created on another pseudo device (eg. lagg) 2446 ** then vlan events are not passed thru, breaking 2447 ** operation, but with HW FILTER off it works. If 2448 ** using vlans directly on the ixgbe driver you can 2449 ** enable this and get full hardware tag filtering. 2450 */ 2451 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2452 2453 /* 2454 * Specify the media types supported by this adapter and register 2455 * callbacks to update media and link information 2456 */ 2457 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2458 ixgbe_media_status); 2459 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL); 2460 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics); 2461 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2462 ifmedia_add(&adapter->media, 2463 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2464 ifmedia_add(&adapter->media, 2465 IFM_ETHER | IFM_1000_T, 0, NULL); 2466 } 2467 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2468 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2469 2470 return (0); 2471 } 2472 2473 static void 2474 ixgbe_config_link(struct adapter *adapter) 2475 { 2476 struct ixgbe_hw *hw = &adapter->hw; 2477 u32 autoneg, err = 0; 2478 bool sfp, negotiate; 2479 2480 sfp = ixgbe_is_sfp(hw); 2481 2482 if (sfp) { 2483 if (hw->phy.multispeed_fiber) { 2484 hw->mac.ops.setup_sfp(hw); 2485 ixgbe_enable_tx_laser(hw); 2486 taskqueue_enqueue(adapter->tq, &adapter->msf_task); 2487 } else 2488 taskqueue_enqueue(adapter->tq, &adapter->mod_task); 2489 } else { 2490 if (hw->mac.ops.check_link) 2491 err = ixgbe_check_link(hw, &autoneg, 2492 &adapter->link_up, FALSE); 2493 if (err) 2494 goto out; 2495 autoneg = hw->phy.autoneg_advertised; 2496 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 2497 err = hw->mac.ops.get_link_capabilities(hw, 2498 &autoneg, &negotiate); 2499 if (err) 2500 goto out; 2501 if (hw->mac.ops.setup_link) 2502 err = hw->mac.ops.setup_link(hw, autoneg, 2503 negotiate, adapter->link_up); 2504 } 2505 out: 2506 return; 2507 } 2508 2509 /******************************************************************** 2510 * Manage DMA'able memory. 2511 *******************************************************************/ 2512 static void 2513 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 2514 { 2515 if (error) 2516 return; 2517 *(bus_addr_t *) arg = segs->ds_addr; 2518 return; 2519 } 2520 2521 static int 2522 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size, 2523 struct ixgbe_dma_alloc *dma, int mapflags) 2524 { 2525 device_t dev = adapter->dev; 2526 int r; 2527 2528 r = bus_dma_tag_create(NULL, /* parent */ 2529 DBA_ALIGN, 0, /* alignment, bounds */ 2530 BUS_SPACE_MAXADDR, /* lowaddr */ 2531 BUS_SPACE_MAXADDR, /* highaddr */ 2532 NULL, NULL, /* filter, filterarg */ 2533 size, /* maxsize */ 2534 1, /* nsegments */ 2535 size, /* maxsegsize */ 2536 BUS_DMA_ALLOCNOW, /* flags */ 2537 &dma->dma_tag); 2538 if (r != 0) { 2539 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; " 2540 "error %u\n", r); 2541 goto fail_0; 2542 } 2543 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 2544 BUS_DMA_NOWAIT, &dma->dma_map); 2545 if (r != 0) { 2546 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2547 "error %u\n", r); 2548 goto fail_1; 2549 } 2550 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2551 size, 2552 ixgbe_dmamap_cb, 2553 &dma->dma_paddr, 2554 mapflags | BUS_DMA_NOWAIT); 2555 if (r != 0) { 2556 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; " 2557 "error %u\n", r); 2558 goto fail_2; 2559 } 2560 dma->dma_size = size; 2561 return (0); 2562 fail_2: 2563 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2564 fail_1: 2565 bus_dma_tag_destroy(dma->dma_tag); 2566 fail_0: 2567 dma->dma_map = NULL; 2568 dma->dma_tag = NULL; 2569 return (r); 2570 } 2571 2572 static void 2573 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2574 { 2575 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2576 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2577 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2578 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2579 bus_dma_tag_destroy(dma->dma_tag); 2580 } 2581 2582 2583 /********************************************************************* 2584 * 2585 * Allocate memory for the transmit and receive rings, and then 2586 * the descriptors associated with each, called only once at attach. 2587 * 2588 **********************************************************************/ 2589 static int 2590 ixgbe_allocate_queues(struct adapter *adapter) 2591 { 2592 device_t dev = adapter->dev; 2593 struct ix_queue *que; 2594 struct tx_ring *txr; 2595 struct rx_ring *rxr; 2596 int rsize, tsize, error = IXGBE_SUCCESS; 2597 int txconf = 0, rxconf = 0; 2598 2599 /* First allocate the top level queue structs */ 2600 if (!(adapter->queues = 2601 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) * 2602 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2603 device_printf(dev, "Unable to allocate queue memory\n"); 2604 error = ENOMEM; 2605 goto fail; 2606 } 2607 2608 /* First allocate the TX ring struct memory */ 2609 if (!(adapter->tx_rings = 2610 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) * 2611 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2612 device_printf(dev, "Unable to allocate TX ring memory\n"); 2613 error = ENOMEM; 2614 goto tx_fail; 2615 } 2616 2617 /* Next allocate the RX */ 2618 if (!(adapter->rx_rings = 2619 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) * 2620 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2621 device_printf(dev, "Unable to allocate RX ring memory\n"); 2622 error = ENOMEM; 2623 goto rx_fail; 2624 } 2625 2626 /* For the ring itself */ 2627 tsize = roundup2(adapter->num_tx_desc * 2628 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2629 2630 /* 2631 * Now set up the TX queues, txconf is needed to handle the 2632 * possibility that things fail midcourse and we need to 2633 * undo memory gracefully 2634 */ 2635 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2636 /* Set up some basics */ 2637 txr = &adapter->tx_rings[i]; 2638 txr->adapter = adapter; 2639 txr->me = i; 2640 2641 /* Initialize the TX side lock */ 2642 IXGBE_TX_LOCK_INIT(txr); 2643 2644 if (ixgbe_dma_malloc(adapter, tsize, 2645 &txr->txdma, BUS_DMA_NOWAIT)) { 2646 device_printf(dev, 2647 "Unable to allocate TX Descriptor memory\n"); 2648 error = ENOMEM; 2649 goto err_tx_desc; 2650 } 2651 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2652 bzero((void *)txr->tx_base, tsize); 2653 2654 /* Now allocate transmit buffers for the ring */ 2655 if (ixgbe_allocate_transmit_buffers(txr)) { 2656 device_printf(dev, 2657 "Critical Failure setting up transmit buffers\n"); 2658 error = ENOMEM; 2659 goto err_tx_desc; 2660 } 2661 #if 0 /* __FreeBSD_version >= 800000 */ 2662 /* Allocate a buf ring */ 2663 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 2664 M_WAITOK, &txr->tx_mtx); 2665 if (txr->br == NULL) { 2666 device_printf(dev, 2667 "Critical Failure setting up buf ring\n"); 2668 error = ENOMEM; 2669 goto err_tx_desc; 2670 } 2671 #endif 2672 } 2673 2674 /* 2675 * Next the RX queues... 2676 */ 2677 rsize = roundup2(adapter->num_rx_desc * 2678 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2679 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2680 rxr = &adapter->rx_rings[i]; 2681 /* Set up some basics */ 2682 rxr->adapter = adapter; 2683 rxr->me = i; 2684 2685 /* Initialize the RX side lock */ 2686 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)", 2687 device_get_nameunit(dev), rxr->me); 2688 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE); 2689 2690 if (ixgbe_dma_malloc(adapter, rsize, 2691 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2692 device_printf(dev, 2693 "Unable to allocate RxDescriptor memory\n"); 2694 error = ENOMEM; 2695 goto err_rx_desc; 2696 } 2697 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2698 bzero((void *)rxr->rx_base, rsize); 2699 2700 /* Allocate receive buffers for the ring*/ 2701 if (ixgbe_allocate_receive_buffers(rxr)) { 2702 device_printf(dev, 2703 "Critical Failure setting up receive buffers\n"); 2704 error = ENOMEM; 2705 goto err_rx_desc; 2706 } 2707 } 2708 2709 /* 2710 ** Finally set up the queue holding structs 2711 */ 2712 for (int i = 0; i < adapter->num_queues; i++) { 2713 que = &adapter->queues[i]; 2714 que->adapter = adapter; 2715 que->txr = &adapter->tx_rings[i]; 2716 que->rxr = &adapter->rx_rings[i]; 2717 } 2718 2719 return (0); 2720 2721 err_rx_desc: 2722 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2723 ixgbe_dma_free(adapter, &rxr->rxdma); 2724 err_tx_desc: 2725 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2726 ixgbe_dma_free(adapter, &txr->txdma); 2727 kfree(adapter->rx_rings, M_DEVBUF); 2728 rx_fail: 2729 kfree(adapter->tx_rings, M_DEVBUF); 2730 tx_fail: 2731 kfree(adapter->queues, M_DEVBUF); 2732 fail: 2733 return (error); 2734 } 2735 2736 /********************************************************************* 2737 * 2738 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2739 * the information needed to transmit a packet on the wire. This is 2740 * called only once at attach, setup is done every reset. 2741 * 2742 **********************************************************************/ 2743 static int 2744 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2745 { 2746 struct adapter *adapter = txr->adapter; 2747 device_t dev = adapter->dev; 2748 struct ixgbe_tx_buf *txbuf; 2749 int error, i; 2750 2751 /* 2752 * Setup DMA descriptor areas. 2753 */ 2754 if ((error = bus_dma_tag_create( 2755 NULL, /* parent */ 2756 1, 0, /* alignment, bounds */ 2757 BUS_SPACE_MAXADDR, /* lowaddr */ 2758 BUS_SPACE_MAXADDR, /* highaddr */ 2759 NULL, NULL, /* filter, filterarg */ 2760 IXGBE_TSO_SIZE, /* maxsize */ 2761 adapter->num_segs, /* nsegments */ 2762 PAGE_SIZE, /* maxsegsize */ 2763 0, /* flags */ 2764 &txr->txtag))) { 2765 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2766 goto fail; 2767 } 2768 2769 if (!(txr->tx_buffers = 2770 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) * 2771 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2772 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2773 error = ENOMEM; 2774 goto fail; 2775 } 2776 2777 /* Create the descriptor buffer dma maps */ 2778 txbuf = txr->tx_buffers; 2779 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2780 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2781 if (error != 0) { 2782 device_printf(dev, "Unable to create TX DMA map\n"); 2783 goto fail; 2784 } 2785 } 2786 2787 return 0; 2788 fail: 2789 /* We free all, it handles case where we are in the middle */ 2790 ixgbe_free_transmit_structures(adapter); 2791 return (error); 2792 } 2793 2794 /********************************************************************* 2795 * 2796 * Initialize a transmit ring. 2797 * 2798 **********************************************************************/ 2799 static void 2800 ixgbe_setup_transmit_ring(struct tx_ring *txr) 2801 { 2802 struct adapter *adapter = txr->adapter; 2803 struct ixgbe_tx_buf *txbuf; 2804 int i; 2805 #ifdef DEV_NETMAP 2806 struct netmap_adapter *na = NA(adapter->ifp); 2807 struct netmap_slot *slot; 2808 #endif /* DEV_NETMAP */ 2809 2810 /* Clear the old ring contents */ 2811 IXGBE_TX_LOCK(txr); 2812 #ifdef DEV_NETMAP 2813 /* 2814 * (under lock): if in netmap mode, do some consistency 2815 * checks and set slot to entry 0 of the netmap ring. 2816 */ 2817 slot = netmap_reset(na, NR_TX, txr->me, 0); 2818 #endif /* DEV_NETMAP */ 2819 bzero((void *)txr->tx_base, 2820 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2821 /* Reset indices */ 2822 txr->next_avail_desc = 0; 2823 txr->next_to_clean = 0; 2824 2825 /* Free any existing tx buffers. */ 2826 txbuf = txr->tx_buffers; 2827 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2828 if (txbuf->m_head != NULL) { 2829 bus_dmamap_sync(txr->txtag, txbuf->map, 2830 BUS_DMASYNC_POSTWRITE); 2831 bus_dmamap_unload(txr->txtag, txbuf->map); 2832 m_freem(txbuf->m_head); 2833 txbuf->m_head = NULL; 2834 } 2835 #ifdef DEV_NETMAP 2836 /* 2837 * In netmap mode, set the map for the packet buffer. 2838 * NOTE: Some drivers (not this one) also need to set 2839 * the physical buffer address in the NIC ring. 2840 * Slots in the netmap ring (indexed by "si") are 2841 * kring->nkr_hwofs positions "ahead" wrt the 2842 * corresponding slot in the NIC ring. In some drivers 2843 * (not here) nkr_hwofs can be negative. Function 2844 * netmap_idx_n2k() handles wraparounds properly. 2845 */ 2846 if (slot) { 2847 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); 2848 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si)); 2849 } 2850 #endif /* DEV_NETMAP */ 2851 /* Clear the EOP index */ 2852 txbuf->eop_index = -1; 2853 } 2854 2855 #ifdef IXGBE_FDIR 2856 /* Set the rate at which we sample packets */ 2857 if (adapter->hw.mac.type != ixgbe_mac_82598EB) 2858 txr->atr_sample = atr_sample_rate; 2859 #endif 2860 2861 /* Set number of descriptors available */ 2862 txr->tx_avail = adapter->num_tx_desc; 2863 2864 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2865 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2866 IXGBE_TX_UNLOCK(txr); 2867 } 2868 2869 /********************************************************************* 2870 * 2871 * Initialize all transmit rings. 2872 * 2873 **********************************************************************/ 2874 static int 2875 ixgbe_setup_transmit_structures(struct adapter *adapter) 2876 { 2877 struct tx_ring *txr = adapter->tx_rings; 2878 2879 for (int i = 0; i < adapter->num_queues; i++, txr++) 2880 ixgbe_setup_transmit_ring(txr); 2881 2882 return (0); 2883 } 2884 2885 /********************************************************************* 2886 * 2887 * Enable transmit unit. 2888 * 2889 **********************************************************************/ 2890 static void 2891 ixgbe_initialize_transmit_units(struct adapter *adapter) 2892 { 2893 struct tx_ring *txr = adapter->tx_rings; 2894 struct ixgbe_hw *hw = &adapter->hw; 2895 2896 /* Setup the Base and Length of the Tx Descriptor Ring */ 2897 2898 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2899 u64 tdba = txr->txdma.dma_paddr; 2900 u32 txctrl; 2901 2902 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2903 (tdba & 0x00000000ffffffffULL)); 2904 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2905 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2906 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2907 2908 /* Setup the HW Tx Head and Tail descriptor pointers */ 2909 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2910 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2911 2912 /* Setup Transmit Descriptor Cmd Settings */ 2913 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2914 txr->queue_status = IXGBE_QUEUE_IDLE; 2915 2916 /* Disable Head Writeback */ 2917 switch (hw->mac.type) { 2918 case ixgbe_mac_82598EB: 2919 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2920 break; 2921 case ixgbe_mac_82599EB: 2922 case ixgbe_mac_X540: 2923 default: 2924 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2925 break; 2926 } 2927 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2928 switch (hw->mac.type) { 2929 case ixgbe_mac_82598EB: 2930 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2931 break; 2932 case ixgbe_mac_82599EB: 2933 case ixgbe_mac_X540: 2934 default: 2935 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2936 break; 2937 } 2938 2939 } 2940 2941 if (hw->mac.type != ixgbe_mac_82598EB) { 2942 u32 dmatxctl, rttdcs; 2943 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2944 dmatxctl |= IXGBE_DMATXCTL_TE; 2945 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2946 /* Disable arbiter to set MTQC */ 2947 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2948 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2949 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2950 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2951 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2952 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2953 } 2954 2955 return; 2956 } 2957 2958 /********************************************************************* 2959 * 2960 * Free all transmit rings. 2961 * 2962 **********************************************************************/ 2963 static void 2964 ixgbe_free_transmit_structures(struct adapter *adapter) 2965 { 2966 struct tx_ring *txr = adapter->tx_rings; 2967 2968 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2969 IXGBE_TX_LOCK(txr); 2970 ixgbe_free_transmit_buffers(txr); 2971 ixgbe_dma_free(adapter, &txr->txdma); 2972 IXGBE_TX_UNLOCK(txr); 2973 IXGBE_TX_LOCK_DESTROY(txr); 2974 } 2975 kfree(adapter->tx_rings, M_DEVBUF); 2976 } 2977 2978 /********************************************************************* 2979 * 2980 * Free transmit ring related data structures. 2981 * 2982 **********************************************************************/ 2983 static void 2984 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2985 { 2986 struct adapter *adapter = txr->adapter; 2987 struct ixgbe_tx_buf *tx_buffer; 2988 int i; 2989 2990 INIT_DEBUGOUT("free_transmit_ring: begin"); 2991 2992 if (txr->tx_buffers == NULL) 2993 return; 2994 2995 tx_buffer = txr->tx_buffers; 2996 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2997 if (tx_buffer->m_head != NULL) { 2998 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2999 BUS_DMASYNC_POSTWRITE); 3000 bus_dmamap_unload(txr->txtag, 3001 tx_buffer->map); 3002 m_freem(tx_buffer->m_head); 3003 tx_buffer->m_head = NULL; 3004 if (tx_buffer->map != NULL) { 3005 bus_dmamap_destroy(txr->txtag, 3006 tx_buffer->map); 3007 tx_buffer->map = NULL; 3008 } 3009 } else if (tx_buffer->map != NULL) { 3010 bus_dmamap_unload(txr->txtag, 3011 tx_buffer->map); 3012 bus_dmamap_destroy(txr->txtag, 3013 tx_buffer->map); 3014 tx_buffer->map = NULL; 3015 } 3016 } 3017 #if 0 /* __FreeBSD_version >= 800000 */ 3018 if (txr->br != NULL) 3019 buf_ring_free(txr->br, M_DEVBUF); 3020 #endif 3021 if (txr->tx_buffers != NULL) { 3022 kfree(txr->tx_buffers, M_DEVBUF); 3023 txr->tx_buffers = NULL; 3024 } 3025 if (txr->txtag != NULL) { 3026 bus_dma_tag_destroy(txr->txtag); 3027 txr->txtag = NULL; 3028 } 3029 return; 3030 } 3031 3032 /********************************************************************* 3033 * 3034 * Advanced Context Descriptor setup for VLAN or CSUM 3035 * 3036 **********************************************************************/ 3037 3038 static bool 3039 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3040 { 3041 struct adapter *adapter = txr->adapter; 3042 struct ixgbe_adv_tx_context_desc *TXD; 3043 struct ixgbe_tx_buf *tx_buffer; 3044 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3045 struct ether_vlan_header *eh; 3046 struct ip *ip; 3047 struct ip6_hdr *ip6; 3048 int ehdrlen, ip_hlen = 0; 3049 u16 etype; 3050 u8 ipproto = 0; 3051 bool offload = TRUE; 3052 int ctxd = txr->next_avail_desc; 3053 u16 vtag = 0; 3054 3055 3056 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 3057 offload = FALSE; 3058 3059 tx_buffer = &txr->tx_buffers[ctxd]; 3060 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3061 3062 /* 3063 ** In advanced descriptors the vlan tag must 3064 ** be placed into the descriptor itself. 3065 */ 3066 if (mp->m_flags & M_VLANTAG) { 3067 vtag = htole16(mp->m_pkthdr.ether_vlantag); 3068 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3069 } else if (offload == FALSE) 3070 return FALSE; 3071 3072 /* 3073 * Determine where frame payload starts. 3074 * Jump over vlan headers if already present, 3075 * helpful for QinQ too. 3076 */ 3077 eh = mtod(mp, struct ether_vlan_header *); 3078 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3079 etype = ntohs(eh->evl_proto); 3080 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3081 } else { 3082 etype = ntohs(eh->evl_encap_proto); 3083 ehdrlen = ETHER_HDR_LEN; 3084 } 3085 3086 /* Set the ether header length */ 3087 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3088 3089 switch (etype) { 3090 case ETHERTYPE_IP: 3091 ip = (struct ip *)(mp->m_data + ehdrlen); 3092 ip_hlen = ip->ip_hl << 2; 3093 ipproto = ip->ip_p; 3094 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3095 break; 3096 case ETHERTYPE_IPV6: 3097 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3098 ip_hlen = sizeof(struct ip6_hdr); 3099 /* XXX-BZ this will go badly in case of ext hdrs. */ 3100 ipproto = ip6->ip6_nxt; 3101 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3102 break; 3103 default: 3104 offload = FALSE; 3105 break; 3106 } 3107 3108 vlan_macip_lens |= ip_hlen; 3109 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3110 3111 switch (ipproto) { 3112 case IPPROTO_TCP: 3113 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 3114 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3115 break; 3116 3117 case IPPROTO_UDP: 3118 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 3119 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 3120 break; 3121 3122 #if 0 3123 case IPPROTO_SCTP: 3124 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 3125 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3126 break; 3127 #endif 3128 default: 3129 offload = FALSE; 3130 break; 3131 } 3132 3133 /* Now copy bits into descriptor */ 3134 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3135 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3136 TXD->seqnum_seed = htole32(0); 3137 TXD->mss_l4len_idx = htole32(0); 3138 3139 tx_buffer->m_head = NULL; 3140 tx_buffer->eop_index = -1; 3141 3142 /* We've consumed the first desc, adjust counters */ 3143 if (++ctxd == adapter->num_tx_desc) 3144 ctxd = 0; 3145 txr->next_avail_desc = ctxd; 3146 --txr->tx_avail; 3147 3148 return (offload); 3149 } 3150 3151 /********************************************************************** 3152 * 3153 * Setup work for hardware segmentation offload (TSO) on 3154 * adapters using advanced tx descriptors 3155 * 3156 **********************************************************************/ 3157 static bool 3158 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, 3159 u32 *olinfo_status) 3160 { 3161 struct adapter *adapter = txr->adapter; 3162 struct ixgbe_adv_tx_context_desc *TXD; 3163 struct ixgbe_tx_buf *tx_buffer; 3164 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3165 u16 vtag = 0, eh_type; 3166 u32 mss_l4len_idx = 0, len; 3167 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3168 struct ether_vlan_header *eh; 3169 #if 0 /* IPv6 TSO */ 3170 #ifdef INET6 3171 struct ip6_hdr *ip6; 3172 #endif 3173 #endif 3174 #ifdef INET 3175 struct ip *ip; 3176 #endif 3177 struct tcphdr *th; 3178 3179 3180 /* 3181 * Determine where frame payload starts. 3182 * Jump over vlan headers if already present 3183 */ 3184 eh = mtod(mp, struct ether_vlan_header *); 3185 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3186 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3187 eh_type = eh->evl_proto; 3188 } else { 3189 ehdrlen = ETHER_HDR_LEN; 3190 eh_type = eh->evl_encap_proto; 3191 } 3192 3193 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3194 len = ehdrlen + sizeof(struct tcphdr); 3195 switch (ntohs(eh_type)) { 3196 #if 0 /* IPv6 TSO */ 3197 #ifdef INET6 3198 case ETHERTYPE_IPV6: 3199 if (mp->m_len < len + sizeof(struct ip6_hdr)) 3200 return FALSE; 3201 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3202 /* XXX-BZ For now we do not pretend to support ext. hdrs. */ 3203 if (ip6->ip6_nxt != IPPROTO_TCP) 3204 return FALSE; 3205 ip_hlen = sizeof(struct ip6_hdr); 3206 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3207 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 3208 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3209 break; 3210 #endif 3211 #endif 3212 #ifdef INET 3213 case ETHERTYPE_IP: 3214 if (mp->m_len < len + sizeof(struct ip)) 3215 return FALSE; 3216 ip = (struct ip *)(mp->m_data + ehdrlen); 3217 if (ip->ip_p != IPPROTO_TCP) 3218 return FALSE; 3219 ip->ip_sum = 0; 3220 ip_hlen = ip->ip_hl << 2; 3221 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3222 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3223 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3224 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3225 /* Tell transmit desc to also do IPv4 checksum. */ 3226 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 3227 break; 3228 #endif 3229 default: 3230 panic("%s: CSUM_TSO but no supported IP version (0x%04x)", 3231 __func__, ntohs(eh_type)); 3232 break; 3233 } 3234 3235 ctxd = txr->next_avail_desc; 3236 tx_buffer = &txr->tx_buffers[ctxd]; 3237 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3238 3239 tcp_hlen = th->th_off << 2; 3240 3241 /* This is used in the transmit desc in encap */ 3242 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 3243 3244 /* VLAN MACLEN IPLEN */ 3245 if (mp->m_flags & M_VLANTAG) { 3246 vtag = htole16(mp->m_pkthdr.ether_vlantag); 3247 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3248 } 3249 3250 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3251 vlan_macip_lens |= ip_hlen; 3252 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3253 3254 /* ADV DTYPE TUCMD */ 3255 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3256 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3257 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3258 3259 /* MSS L4LEN IDX */ 3260 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 3261 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 3262 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3263 3264 TXD->seqnum_seed = htole32(0); 3265 tx_buffer->m_head = NULL; 3266 tx_buffer->eop_index = -1; 3267 3268 if (++ctxd == adapter->num_tx_desc) 3269 ctxd = 0; 3270 3271 txr->tx_avail--; 3272 txr->next_avail_desc = ctxd; 3273 return TRUE; 3274 } 3275 3276 #ifdef IXGBE_FDIR 3277 /* 3278 ** This routine parses packet headers so that Flow 3279 ** Director can make a hashed filter table entry 3280 ** allowing traffic flows to be identified and kept 3281 ** on the same cpu. This would be a performance 3282 ** hit, but we only do it at IXGBE_FDIR_RATE of 3283 ** packets. 3284 */ 3285 static void 3286 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 3287 { 3288 struct adapter *adapter = txr->adapter; 3289 struct ix_queue *que; 3290 struct ip *ip; 3291 struct tcphdr *th; 3292 struct udphdr *uh; 3293 struct ether_vlan_header *eh; 3294 union ixgbe_atr_hash_dword input = {.dword = 0}; 3295 union ixgbe_atr_hash_dword common = {.dword = 0}; 3296 int ehdrlen, ip_hlen; 3297 u16 etype; 3298 3299 eh = mtod(mp, struct ether_vlan_header *); 3300 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3301 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3302 etype = eh->evl_proto; 3303 } else { 3304 ehdrlen = ETHER_HDR_LEN; 3305 etype = eh->evl_encap_proto; 3306 } 3307 3308 /* Only handling IPv4 */ 3309 if (etype != htons(ETHERTYPE_IP)) 3310 return; 3311 3312 ip = (struct ip *)(mp->m_data + ehdrlen); 3313 ip_hlen = ip->ip_hl << 2; 3314 3315 /* check if we're UDP or TCP */ 3316 switch (ip->ip_p) { 3317 case IPPROTO_TCP: 3318 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3319 /* src and dst are inverted */ 3320 common.port.dst ^= th->th_sport; 3321 common.port.src ^= th->th_dport; 3322 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 3323 break; 3324 case IPPROTO_UDP: 3325 uh = (struct udphdr *)((caddr_t)ip + ip_hlen); 3326 /* src and dst are inverted */ 3327 common.port.dst ^= uh->uh_sport; 3328 common.port.src ^= uh->uh_dport; 3329 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 3330 break; 3331 default: 3332 return; 3333 } 3334 3335 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 3336 if (mp->m_pkthdr.ether_vtag) 3337 common.flex_bytes ^= htons(ETHERTYPE_VLAN); 3338 else 3339 common.flex_bytes ^= etype; 3340 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 3341 3342 que = &adapter->queues[txr->me]; 3343 /* 3344 ** This assumes the Rx queue and Tx 3345 ** queue are bound to the same CPU 3346 */ 3347 ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 3348 input, common, que->msix); 3349 } 3350 #endif /* IXGBE_FDIR */ 3351 3352 /********************************************************************** 3353 * 3354 * Examine each tx_buffer in the used queue. If the hardware is done 3355 * processing the packet then free associated resources. The 3356 * tx_buffer is put back on the free queue. 3357 * 3358 **********************************************************************/ 3359 static void 3360 ixgbe_txeof(struct tx_ring *txr) 3361 { 3362 struct adapter *adapter = txr->adapter; 3363 struct ifnet *ifp = adapter->ifp; 3364 u32 first, last, done, processed; 3365 struct ixgbe_tx_buf *tx_buffer; 3366 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 3367 3368 IXGBE_TX_LOCK_ASSERT(txr); 3369 3370 #ifdef DEV_NETMAP 3371 if (ifp->if_capenable & IFCAP_NETMAP) { 3372 struct netmap_adapter *na = NA(ifp); 3373 struct netmap_kring *kring = &na->tx_rings[txr->me]; 3374 3375 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base; 3376 3377 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3378 BUS_DMASYNC_POSTREAD); 3379 /* 3380 * In netmap mode, all the work is done in the context 3381 * of the client thread. Interrupt handlers only wake up 3382 * clients, which may be sleeping on individual rings 3383 * or on a global resource for all rings. 3384 * To implement tx interrupt mitigation, we wake up the client 3385 * thread roughly every half ring, even if the NIC interrupts 3386 * more frequently. This is implemented as follows: 3387 * - ixgbe_txsync() sets kring->nr_kflags with the index of 3388 * the slot that should wake up the thread (nkr_num_slots 3389 * means the user thread should not be woken up); 3390 * - the driver ignores tx interrupts unless netmap_mitigate=0 3391 * or the slot has the DD bit set. 3392 * 3393 * When the driver has separate locks, we need to 3394 * release and re-acquire txlock to avoid deadlocks. 3395 * XXX see if we can find a better way. 3396 */ 3397 if (!netmap_mitigate || 3398 (kring->nr_kflags < kring->nkr_num_slots && 3399 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) { 3400 kring->nr_kflags = kring->nkr_num_slots; 3401 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET); 3402 IXGBE_TX_UNLOCK(txr); 3403 IXGBE_CORE_LOCK(adapter); 3404 selwakeuppri(&na->tx_si, PI_NET); 3405 IXGBE_CORE_UNLOCK(adapter); 3406 IXGBE_TX_LOCK(txr); 3407 } 3408 return FALSE; 3409 } 3410 #endif /* DEV_NETMAP */ 3411 3412 if (txr->tx_avail == adapter->num_tx_desc) { 3413 txr->queue_status = IXGBE_QUEUE_IDLE; 3414 return; 3415 } 3416 3417 processed = 0; 3418 first = txr->next_to_clean; 3419 tx_buffer = &txr->tx_buffers[first]; 3420 /* For cleanup we just use legacy struct */ 3421 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3422 last = tx_buffer->eop_index; 3423 if (last == -1) 3424 return; 3425 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3426 3427 /* 3428 ** Get the index of the first descriptor 3429 ** BEYOND the EOP and call that 'done'. 3430 ** I do this so the comparison in the 3431 ** inner while loop below can be simple 3432 */ 3433 if (++last == adapter->num_tx_desc) last = 0; 3434 done = last; 3435 3436 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3437 BUS_DMASYNC_POSTREAD); 3438 /* 3439 ** Only the EOP descriptor of a packet now has the DD 3440 ** bit set, this is what we look for... 3441 */ 3442 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 3443 /* We clean the range of the packet */ 3444 while (first != done) { 3445 tx_desc->upper.data = 0; 3446 tx_desc->lower.data = 0; 3447 tx_desc->buffer_addr = 0; 3448 ++txr->tx_avail; 3449 ++processed; 3450 3451 if (tx_buffer->m_head) { 3452 txr->bytes += 3453 tx_buffer->m_head->m_pkthdr.len; 3454 bus_dmamap_sync(txr->txtag, 3455 tx_buffer->map, 3456 BUS_DMASYNC_POSTWRITE); 3457 bus_dmamap_unload(txr->txtag, 3458 tx_buffer->map); 3459 m_freem(tx_buffer->m_head); 3460 tx_buffer->m_head = NULL; 3461 tx_buffer->map = NULL; 3462 } 3463 tx_buffer->eop_index = -1; 3464 txr->watchdog_time = ticks; 3465 3466 if (++first == adapter->num_tx_desc) 3467 first = 0; 3468 3469 tx_buffer = &txr->tx_buffers[first]; 3470 tx_desc = 3471 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3472 } 3473 ++txr->packets; 3474 IFNET_STAT_INC(ifp, opackets, 1); 3475 /* See if there is more work now */ 3476 last = tx_buffer->eop_index; 3477 if (last != -1) { 3478 eop_desc = 3479 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3480 /* Get next done point */ 3481 if (++last == adapter->num_tx_desc) last = 0; 3482 done = last; 3483 } else 3484 break; 3485 } 3486 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3487 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3488 3489 txr->next_to_clean = first; 3490 3491 /* 3492 ** Watchdog calculation, we know there's 3493 ** work outstanding or the first return 3494 ** would have been taken, so none processed 3495 ** for too long indicates a hang. 3496 */ 3497 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG)) 3498 txr->queue_status = IXGBE_QUEUE_HUNG; 3499 3500 /* With a minimum free clear the depleted state bit. */ 3501 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) 3502 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED; 3503 3504 if (txr->tx_avail == adapter->num_tx_desc) { 3505 txr->queue_status = IXGBE_QUEUE_IDLE; 3506 } 3507 } 3508 3509 /********************************************************************* 3510 * 3511 * Refresh mbuf buffers for RX descriptor rings 3512 * - now keeps its own state so discards due to resource 3513 * exhaustion are unnecessary, if an mbuf cannot be obtained 3514 * it just returns, keeping its placeholder, thus it can simply 3515 * be recalled to try again. 3516 * 3517 **********************************************************************/ 3518 static void 3519 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 3520 { 3521 struct adapter *adapter = rxr->adapter; 3522 bus_dma_segment_t hseg[1]; 3523 bus_dma_segment_t pseg[1]; 3524 struct ixgbe_rx_buf *rxbuf; 3525 struct mbuf *mh, *mp; 3526 int i, j, nsegs, error; 3527 bool refreshed = FALSE; 3528 3529 i = j = rxr->next_to_refresh; 3530 /* Control the loop with one beyond */ 3531 if (++j == adapter->num_rx_desc) 3532 j = 0; 3533 3534 while (j != limit) { 3535 rxbuf = &rxr->rx_buffers[i]; 3536 if (rxr->hdr_split == FALSE) 3537 goto no_split; 3538 3539 if (rxbuf->m_head == NULL) { 3540 mh = m_gethdr(MB_DONTWAIT, MT_DATA); 3541 if (mh == NULL) 3542 goto update; 3543 } else 3544 mh = rxbuf->m_head; 3545 3546 mh->m_pkthdr.len = mh->m_len = MHLEN; 3547 mh->m_len = MHLEN; 3548 mh->m_flags |= M_PKTHDR; 3549 /* Get the memory mapping */ 3550 error = bus_dmamap_load_mbuf_segment(rxr->htag, 3551 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT); 3552 if (error != 0) { 3553 kprintf("Refresh mbufs: hdr dmamap load" 3554 " failure - %d\n", error); 3555 m_free(mh); 3556 rxbuf->m_head = NULL; 3557 goto update; 3558 } 3559 rxbuf->m_head = mh; 3560 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3561 BUS_DMASYNC_PREREAD); 3562 rxr->rx_base[i].read.hdr_addr = 3563 htole64(hseg[0].ds_addr); 3564 3565 no_split: 3566 if (rxbuf->m_pack == NULL) { 3567 mp = m_getjcl(MB_DONTWAIT, MT_DATA, 3568 M_PKTHDR, adapter->rx_mbuf_sz); 3569 if (mp == NULL) 3570 goto update; 3571 } else 3572 mp = rxbuf->m_pack; 3573 3574 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3575 /* Get the memory mapping */ 3576 error = bus_dmamap_load_mbuf_segment(rxr->ptag, 3577 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT); 3578 if (error != 0) { 3579 kprintf("Refresh mbufs: payload dmamap load" 3580 " failure - %d\n", error); 3581 m_free(mp); 3582 rxbuf->m_pack = NULL; 3583 goto update; 3584 } 3585 rxbuf->m_pack = mp; 3586 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3587 BUS_DMASYNC_PREREAD); 3588 rxr->rx_base[i].read.pkt_addr = 3589 htole64(pseg[0].ds_addr); 3590 3591 refreshed = TRUE; 3592 /* Next is precalculated */ 3593 i = j; 3594 rxr->next_to_refresh = i; 3595 if (++j == adapter->num_rx_desc) 3596 j = 0; 3597 } 3598 update: 3599 if (refreshed) /* Update hardware tail index */ 3600 IXGBE_WRITE_REG(&adapter->hw, 3601 IXGBE_RDT(rxr->me), rxr->next_to_refresh); 3602 return; 3603 } 3604 3605 /********************************************************************* 3606 * 3607 * Allocate memory for rx_buffer structures. Since we use one 3608 * rx_buffer per received packet, the maximum number of rx_buffer's 3609 * that we'll need is equal to the number of receive descriptors 3610 * that we've allocated. 3611 * 3612 **********************************************************************/ 3613 static int 3614 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 3615 { 3616 struct adapter *adapter = rxr->adapter; 3617 device_t dev = adapter->dev; 3618 struct ixgbe_rx_buf *rxbuf; 3619 int i, bsize, error; 3620 3621 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 3622 if (!(rxr->rx_buffers = 3623 (struct ixgbe_rx_buf *) kmalloc(bsize, 3624 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3625 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3626 error = ENOMEM; 3627 goto fail; 3628 } 3629 3630 if ((error = bus_dma_tag_create(NULL, /* parent */ 3631 1, 0, /* alignment, bounds */ 3632 BUS_SPACE_MAXADDR, /* lowaddr */ 3633 BUS_SPACE_MAXADDR, /* highaddr */ 3634 NULL, NULL, /* filter, filterarg */ 3635 MSIZE, /* maxsize */ 3636 1, /* nsegments */ 3637 MSIZE, /* maxsegsize */ 3638 0, /* flags */ 3639 &rxr->htag))) { 3640 device_printf(dev, "Unable to create RX DMA tag\n"); 3641 goto fail; 3642 } 3643 3644 if ((error = bus_dma_tag_create(NULL, /* parent */ 3645 1, 0, /* alignment, bounds */ 3646 BUS_SPACE_MAXADDR, /* lowaddr */ 3647 BUS_SPACE_MAXADDR, /* highaddr */ 3648 NULL, NULL, /* filter, filterarg */ 3649 MJUM16BYTES, /* maxsize */ 3650 1, /* nsegments */ 3651 MJUM16BYTES, /* maxsegsize */ 3652 0, /* flags */ 3653 &rxr->ptag))) { 3654 device_printf(dev, "Unable to create RX DMA tag\n"); 3655 goto fail; 3656 } 3657 3658 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3659 rxbuf = &rxr->rx_buffers[i]; 3660 error = bus_dmamap_create(rxr->htag, 3661 BUS_DMA_NOWAIT, &rxbuf->hmap); 3662 if (error) { 3663 device_printf(dev, "Unable to create RX head map\n"); 3664 goto fail; 3665 } 3666 error = bus_dmamap_create(rxr->ptag, 3667 BUS_DMA_NOWAIT, &rxbuf->pmap); 3668 if (error) { 3669 device_printf(dev, "Unable to create RX pkt map\n"); 3670 goto fail; 3671 } 3672 } 3673 3674 return (0); 3675 3676 fail: 3677 /* Frees all, but can handle partial completion */ 3678 ixgbe_free_receive_structures(adapter); 3679 return (error); 3680 } 3681 3682 /* 3683 ** Used to detect a descriptor that has 3684 ** been merged by Hardware RSC. 3685 */ 3686 static inline u32 3687 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 3688 { 3689 return (le32toh(rx->wb.lower.lo_dword.data) & 3690 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 3691 } 3692 3693 /********************************************************************* 3694 * 3695 * Initialize Hardware RSC (LRO) feature on 82599 3696 * for an RX ring, this is toggled by the LRO capability 3697 * even though it is transparent to the stack. 3698 * 3699 **********************************************************************/ 3700 #if 0 /* NET_LRO */ 3701 static void 3702 ixgbe_setup_hw_rsc(struct rx_ring *rxr) 3703 { 3704 struct adapter *adapter = rxr->adapter; 3705 struct ixgbe_hw *hw = &adapter->hw; 3706 u32 rscctrl, rdrxctl; 3707 3708 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3709 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3710 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */ 3711 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip) 3712 #endif /* DEV_NETMAP */ 3713 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3714 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 3715 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3716 3717 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 3718 rscctrl |= IXGBE_RSCCTL_RSCEN; 3719 /* 3720 ** Limit the total number of descriptors that 3721 ** can be combined, so it does not exceed 64K 3722 */ 3723 if (adapter->rx_mbuf_sz == MCLBYTES) 3724 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3725 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 3726 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 3727 else if (adapter->rx_mbuf_sz == MJUM9BYTES) 3728 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 3729 else /* Using 16K cluster */ 3730 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 3731 3732 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 3733 3734 /* Enable TCP header recognition */ 3735 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 3736 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 3737 IXGBE_PSRTYPE_TCPHDR)); 3738 3739 /* Disable RSC for ACK packets */ 3740 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3741 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3742 3743 rxr->hw_rsc = TRUE; 3744 } 3745 #endif 3746 3747 static void 3748 ixgbe_free_receive_ring(struct rx_ring *rxr) 3749 { 3750 struct adapter *adapter; 3751 struct ixgbe_rx_buf *rxbuf; 3752 int i; 3753 3754 adapter = rxr->adapter; 3755 for (i = 0; i < adapter->num_rx_desc; i++) { 3756 rxbuf = &rxr->rx_buffers[i]; 3757 if (rxbuf->m_head != NULL) { 3758 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3759 BUS_DMASYNC_POSTREAD); 3760 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3761 rxbuf->m_head->m_flags |= M_PKTHDR; 3762 m_freem(rxbuf->m_head); 3763 } 3764 if (rxbuf->m_pack != NULL) { 3765 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3766 BUS_DMASYNC_POSTREAD); 3767 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3768 rxbuf->m_pack->m_flags |= M_PKTHDR; 3769 m_freem(rxbuf->m_pack); 3770 } 3771 rxbuf->m_head = NULL; 3772 rxbuf->m_pack = NULL; 3773 } 3774 } 3775 3776 3777 /********************************************************************* 3778 * 3779 * Initialize a receive ring and its buffers. 3780 * 3781 **********************************************************************/ 3782 static int 3783 ixgbe_setup_receive_ring(struct rx_ring *rxr) 3784 { 3785 struct adapter *adapter; 3786 #if 0 /* NET_LRO */ 3787 struct ifnet *ifp; 3788 device_t dev; 3789 #endif 3790 struct ixgbe_rx_buf *rxbuf; 3791 bus_dma_segment_t pseg[1], hseg[1]; 3792 #if 0 /* NET_LRO */ 3793 struct lro_ctrl *lro = &rxr->lro; 3794 #endif 3795 int rsize, nsegs, error = 0; 3796 #ifdef DEV_NETMAP 3797 struct netmap_adapter *na = NA(rxr->adapter->ifp); 3798 struct netmap_slot *slot; 3799 #endif /* DEV_NETMAP */ 3800 3801 adapter = rxr->adapter; 3802 #if 0 /* NET_LRO */ 3803 ifp = adapter->ifp; 3804 dev = adapter->dev; 3805 #endif 3806 3807 /* Clear the ring contents */ 3808 IXGBE_RX_LOCK(rxr); 3809 #ifdef DEV_NETMAP 3810 /* same as in ixgbe_setup_transmit_ring() */ 3811 slot = netmap_reset(na, NR_RX, rxr->me, 0); 3812 #endif /* DEV_NETMAP */ 3813 rsize = roundup2(adapter->num_rx_desc * 3814 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3815 bzero((void *)rxr->rx_base, rsize); 3816 3817 /* Free current RX buffer structs and their mbufs */ 3818 ixgbe_free_receive_ring(rxr); 3819 3820 /* Configure header split? */ 3821 if (ixgbe_header_split) 3822 rxr->hdr_split = TRUE; 3823 3824 /* Now replenish the mbufs */ 3825 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3826 struct mbuf *mh, *mp; 3827 3828 rxbuf = &rxr->rx_buffers[j]; 3829 #ifdef DEV_NETMAP 3830 /* 3831 * In netmap mode, fill the map and set the buffer 3832 * address in the NIC ring, considering the offset 3833 * between the netmap and NIC rings (see comment in 3834 * ixgbe_setup_transmit_ring() ). No need to allocate 3835 * an mbuf, so end the block with a continue; 3836 */ 3837 if (slot) { 3838 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); 3839 uint64_t paddr; 3840 void *addr; 3841 3842 addr = PNMB(slot + sj, &paddr); 3843 netmap_load_map(rxr->ptag, rxbuf->pmap, addr); 3844 /* Update descriptor */ 3845 rxr->rx_base[j].read.pkt_addr = htole64(paddr); 3846 continue; 3847 } 3848 #endif /* DEV_NETMAP */ 3849 /* 3850 ** Don't allocate mbufs if not 3851 ** doing header split, its wasteful 3852 */ 3853 if (rxr->hdr_split == FALSE) 3854 goto skip_head; 3855 3856 /* First the header */ 3857 rxbuf->m_head = m_gethdr(MB_DONTWAIT, MT_DATA); 3858 if (rxbuf->m_head == NULL) { 3859 error = ENOBUFS; 3860 goto fail; 3861 } 3862 m_adj(rxbuf->m_head, ETHER_ALIGN); 3863 mh = rxbuf->m_head; 3864 mh->m_len = mh->m_pkthdr.len = MHLEN; 3865 mh->m_flags |= M_PKTHDR; 3866 /* Get the memory mapping */ 3867 error = bus_dmamap_load_mbuf_segment(rxr->htag, 3868 rxbuf->hmap, rxbuf->m_head, hseg, 1, 3869 &nsegs, BUS_DMA_NOWAIT); 3870 3871 if (error != 0) /* Nothing elegant to do here */ 3872 goto fail; 3873 bus_dmamap_sync(rxr->htag, 3874 rxbuf->hmap, BUS_DMASYNC_PREREAD); 3875 /* Update descriptor */ 3876 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 3877 3878 skip_head: 3879 /* Now the payload cluster */ 3880 rxbuf->m_pack = m_getjcl(MB_DONTWAIT, MT_DATA, 3881 M_PKTHDR, adapter->rx_mbuf_sz); 3882 if (rxbuf->m_pack == NULL) { 3883 error = ENOBUFS; 3884 goto fail; 3885 } 3886 mp = rxbuf->m_pack; 3887 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3888 /* Get the memory mapping */ 3889 error = bus_dmamap_load_mbuf_segment(rxr->ptag, 3890 rxbuf->pmap, mp, hseg, 1, 3891 &nsegs, BUS_DMA_NOWAIT); 3892 if (error != 0) 3893 goto fail; 3894 bus_dmamap_sync(rxr->ptag, 3895 rxbuf->pmap, BUS_DMASYNC_PREREAD); 3896 /* Update descriptor */ 3897 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 3898 } 3899 3900 3901 /* Setup our descriptor indices */ 3902 rxr->next_to_check = 0; 3903 rxr->next_to_refresh = 0; 3904 rxr->lro_enabled = FALSE; 3905 rxr->rx_split_packets = 0; 3906 rxr->rx_bytes = 0; 3907 rxr->discard = FALSE; 3908 rxr->vtag_strip = FALSE; 3909 3910 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3911 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3912 3913 /* 3914 ** Now set up the LRO interface: 3915 ** 82598 uses software LRO, the 3916 ** 82599 and X540 use a hardware assist. 3917 */ 3918 #if 0 /* NET_LRO */ 3919 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && 3920 (ifp->if_capenable & IFCAP_RXCSUM) && 3921 (ifp->if_capenable & IFCAP_LRO)) 3922 ixgbe_setup_hw_rsc(rxr); 3923 else if (ifp->if_capenable & IFCAP_LRO) { 3924 int err = tcp_lro_init(lro); 3925 if (err) { 3926 device_printf(dev, "LRO Initialization failed!\n"); 3927 goto fail; 3928 } 3929 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 3930 rxr->lro_enabled = TRUE; 3931 lro->ifp = adapter->ifp; 3932 } 3933 #endif 3934 3935 IXGBE_RX_UNLOCK(rxr); 3936 return (0); 3937 3938 fail: 3939 ixgbe_free_receive_ring(rxr); 3940 IXGBE_RX_UNLOCK(rxr); 3941 return (error); 3942 } 3943 3944 /********************************************************************* 3945 * 3946 * Initialize all receive rings. 3947 * 3948 **********************************************************************/ 3949 static int 3950 ixgbe_setup_receive_structures(struct adapter *adapter) 3951 { 3952 struct rx_ring *rxr = adapter->rx_rings; 3953 int j; 3954 3955 for (j = 0; j < adapter->num_queues; j++, rxr++) 3956 if (ixgbe_setup_receive_ring(rxr)) 3957 goto fail; 3958 3959 return (0); 3960 fail: 3961 /* 3962 * Free RX buffers allocated so far, we will only handle 3963 * the rings that completed, the failing case will have 3964 * cleaned up for itself. 'j' failed, so its the terminus. 3965 */ 3966 for (int i = 0; i < j; ++i) { 3967 rxr = &adapter->rx_rings[i]; 3968 ixgbe_free_receive_ring(rxr); 3969 } 3970 3971 return (ENOBUFS); 3972 } 3973 3974 /********************************************************************* 3975 * 3976 * Setup receive registers and features. 3977 * 3978 **********************************************************************/ 3979 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3980 3981 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 3982 3983 static void 3984 ixgbe_initialize_receive_units(struct adapter *adapter) 3985 { 3986 struct rx_ring *rxr = adapter->rx_rings; 3987 struct ixgbe_hw *hw = &adapter->hw; 3988 struct ifnet *ifp = adapter->ifp; 3989 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; 3990 u32 reta, mrqc = 0, hlreg, random[10]; 3991 3992 3993 /* 3994 * Make sure receives are disabled while 3995 * setting up the descriptor ring 3996 */ 3997 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 3998 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, 3999 rxctrl & ~IXGBE_RXCTRL_RXEN); 4000 4001 /* Enable broadcasts */ 4002 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4003 fctrl |= IXGBE_FCTRL_BAM; 4004 fctrl |= IXGBE_FCTRL_DPF; 4005 fctrl |= IXGBE_FCTRL_PMCF; 4006 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4007 4008 /* Set for Jumbo Frames? */ 4009 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4010 if (ifp->if_mtu > ETHERMTU) 4011 hlreg |= IXGBE_HLREG0_JUMBOEN; 4012 else 4013 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 4014 #ifdef DEV_NETMAP 4015 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */ 4016 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip) 4017 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 4018 else 4019 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 4020 #endif /* DEV_NETMAP */ 4021 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 4022 4023 bufsz = (adapter->rx_mbuf_sz + 4024 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 4025 4026 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4027 u64 rdba = rxr->rxdma.dma_paddr; 4028 4029 /* Setup the Base and Length of the Rx Descriptor Ring */ 4030 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 4031 (rdba & 0x00000000ffffffffULL)); 4032 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 4033 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 4034 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 4035 4036 /* Set up the SRRCTL register */ 4037 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 4038 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4039 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 4040 srrctl |= bufsz; 4041 if (rxr->hdr_split) { 4042 /* Use a standard mbuf for the header */ 4043 srrctl |= ((IXGBE_RX_HDR << 4044 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 4045 & IXGBE_SRRCTL_BSIZEHDR_MASK); 4046 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4047 } else 4048 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 4049 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 4050 4051 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 4052 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 4053 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 4054 } 4055 4056 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 4057 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4058 IXGBE_PSRTYPE_UDPHDR | 4059 IXGBE_PSRTYPE_IPV4HDR | 4060 IXGBE_PSRTYPE_IPV6HDR; 4061 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 4062 } 4063 4064 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 4065 4066 /* Setup RSS */ 4067 if (adapter->num_queues > 1) { 4068 int i, j; 4069 reta = 0; 4070 4071 /* set up random bits */ 4072 karc4rand(&random, sizeof(random)); 4073 4074 /* Set up the redirection table */ 4075 for (i = 0, j = 0; i < 128; i++, j++) { 4076 if (j == adapter->num_queues) j = 0; 4077 reta = (reta << 8) | (j * 0x11); 4078 if ((i & 3) == 3) 4079 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 4080 } 4081 4082 /* Now fill our hash function seeds */ 4083 for (int i = 0; i < 10; i++) 4084 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]); 4085 4086 /* Perform hash on these packet types */ 4087 mrqc = IXGBE_MRQC_RSSEN 4088 | IXGBE_MRQC_RSS_FIELD_IPV4 4089 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 4090 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 4091 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 4092 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 4093 | IXGBE_MRQC_RSS_FIELD_IPV6 4094 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 4095 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 4096 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 4097 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4098 4099 /* RSS and RX IPP Checksum are mutually exclusive */ 4100 rxcsum |= IXGBE_RXCSUM_PCSD; 4101 } 4102 4103 if (ifp->if_capenable & IFCAP_RXCSUM) 4104 rxcsum |= IXGBE_RXCSUM_PCSD; 4105 4106 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 4107 rxcsum |= IXGBE_RXCSUM_IPPCSE; 4108 4109 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 4110 4111 return; 4112 } 4113 4114 /********************************************************************* 4115 * 4116 * Free all receive rings. 4117 * 4118 **********************************************************************/ 4119 static void 4120 ixgbe_free_receive_structures(struct adapter *adapter) 4121 { 4122 struct rx_ring *rxr = adapter->rx_rings; 4123 4124 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4125 #if 0 /* NET_LRO */ 4126 struct lro_ctrl *lro = &rxr->lro; 4127 #endif 4128 ixgbe_free_receive_buffers(rxr); 4129 /* Free LRO memory */ 4130 #if 0 /* NET_LRO */ 4131 tcp_lro_free(lro); 4132 #endif 4133 /* Free the ring memory as well */ 4134 ixgbe_dma_free(adapter, &rxr->rxdma); 4135 } 4136 4137 kfree(adapter->rx_rings, M_DEVBUF); 4138 } 4139 4140 4141 /********************************************************************* 4142 * 4143 * Free receive ring data structures 4144 * 4145 **********************************************************************/ 4146 static void 4147 ixgbe_free_receive_buffers(struct rx_ring *rxr) 4148 { 4149 struct adapter *adapter = rxr->adapter; 4150 struct ixgbe_rx_buf *rxbuf; 4151 4152 INIT_DEBUGOUT("free_receive_structures: begin"); 4153 4154 /* Cleanup any existing buffers */ 4155 if (rxr->rx_buffers != NULL) { 4156 for (int i = 0; i < adapter->num_rx_desc; i++) { 4157 rxbuf = &rxr->rx_buffers[i]; 4158 if (rxbuf->m_head != NULL) { 4159 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 4160 BUS_DMASYNC_POSTREAD); 4161 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 4162 rxbuf->m_head->m_flags |= M_PKTHDR; 4163 m_freem(rxbuf->m_head); 4164 } 4165 if (rxbuf->m_pack != NULL) { 4166 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 4167 BUS_DMASYNC_POSTREAD); 4168 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 4169 rxbuf->m_pack->m_flags |= M_PKTHDR; 4170 m_freem(rxbuf->m_pack); 4171 } 4172 rxbuf->m_head = NULL; 4173 rxbuf->m_pack = NULL; 4174 if (rxbuf->hmap != NULL) { 4175 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 4176 rxbuf->hmap = NULL; 4177 } 4178 if (rxbuf->pmap != NULL) { 4179 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4180 rxbuf->pmap = NULL; 4181 } 4182 } 4183 if (rxr->rx_buffers != NULL) { 4184 kfree(rxr->rx_buffers, M_DEVBUF); 4185 rxr->rx_buffers = NULL; 4186 } 4187 } 4188 4189 if (rxr->htag != NULL) { 4190 bus_dma_tag_destroy(rxr->htag); 4191 rxr->htag = NULL; 4192 } 4193 if (rxr->ptag != NULL) { 4194 bus_dma_tag_destroy(rxr->ptag); 4195 rxr->ptag = NULL; 4196 } 4197 4198 return; 4199 } 4200 4201 static __inline void 4202 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4203 { 4204 4205 /* 4206 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet 4207 * should be computed by hardware. Also it should not have VLAN tag in 4208 * ethernet header. In case of IPv6 we do not yet support ext. hdrs. 4209 */ 4210 #if 0 /* NET_LRO */ 4211 if (rxr->lro_enabled && 4212 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4213 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4214 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4215 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) || 4216 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4217 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) && 4218 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4219 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4220 /* 4221 * Send to the stack if: 4222 ** - LRO not enabled, or 4223 ** - no LRO resources, or 4224 ** - lro enqueue fails 4225 */ 4226 if (rxr->lro.lro_cnt != 0) 4227 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4228 return; 4229 } 4230 #endif 4231 IXGBE_RX_UNLOCK(rxr); 4232 (*ifp->if_input)(ifp, m); 4233 IXGBE_RX_LOCK(rxr); 4234 } 4235 4236 static __inline void 4237 ixgbe_rx_discard(struct rx_ring *rxr, int i) 4238 { 4239 struct ixgbe_rx_buf *rbuf; 4240 4241 rbuf = &rxr->rx_buffers[i]; 4242 4243 if (rbuf->fmp != NULL) {/* Partial chain ? */ 4244 rbuf->fmp->m_flags |= M_PKTHDR; 4245 m_freem(rbuf->fmp); 4246 rbuf->fmp = NULL; 4247 } 4248 4249 /* 4250 ** With advanced descriptors the writeback 4251 ** clobbers the buffer addrs, so its easier 4252 ** to just free the existing mbufs and take 4253 ** the normal refresh path to get new buffers 4254 ** and mapping. 4255 */ 4256 if (rbuf->m_head) { 4257 m_free(rbuf->m_head); 4258 rbuf->m_head = NULL; 4259 } 4260 4261 if (rbuf->m_pack) { 4262 m_free(rbuf->m_pack); 4263 rbuf->m_pack = NULL; 4264 } 4265 4266 return; 4267 } 4268 4269 static void 4270 ixgbe_add_sysctl(struct adapter *adapter) 4271 { 4272 sysctl_ctx_init(&adapter->sysctl_ctx); 4273 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx, 4274 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, 4275 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, ""); 4276 if (adapter->sysctl_tree == NULL) { 4277 device_printf(adapter->dev, "can't add sysctl node\n"); 4278 return; 4279 } 4280 SYSCTL_ADD_PROC(&adapter->sysctl_ctx, 4281 SYSCTL_CHILDREN(adapter->sysctl_tree), 4282 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, 4283 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control"); 4284 4285 SYSCTL_ADD_PROC(&adapter->sysctl_ctx, 4286 SYSCTL_CHILDREN(adapter->sysctl_tree), 4287 OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW, 4288 adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate"); 4289 4290 /* 4291 ** Allow a kind of speed control by forcing the autoneg 4292 ** advertised speed list to only a certain value, this 4293 ** supports 1G on 82599 devices, and 100Mb on x540. 4294 */ 4295 SYSCTL_ADD_PROC(&adapter->sysctl_ctx, 4296 SYSCTL_CHILDREN(adapter->sysctl_tree), 4297 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, 4298 adapter, 0, ixgbe_set_advertise, "I", "Link Speed"); 4299 4300 SYSCTL_ADD_PROC(&adapter->sysctl_ctx, 4301 SYSCTL_CHILDREN(adapter->sysctl_tree), 4302 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter, 4303 0, ixgbe_set_thermal_test, "I", "Thermal Test"); 4304 4305 /* Sysctl for limiting the amount of work done in the taskqueue */ 4306 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 4307 "max number of rx packets to process", &adapter->rx_process_limit, 4308 ixgbe_rx_process_limit); 4309 } 4310 4311 /********************************************************************* 4312 * 4313 * This routine executes in interrupt context. It replenishes 4314 * the mbufs in the descriptor and sends data which has been 4315 * dma'ed into host memory to upper layer. 4316 * 4317 * We loop at most count times if count is > 0, or until done if 4318 * count < 0. 4319 * 4320 * Return TRUE for more work, FALSE for all clean. 4321 *********************************************************************/ 4322 static void 4323 ixgbe_rxeof(struct ix_queue *que, int count) 4324 { 4325 struct adapter *adapter = que->adapter; 4326 struct rx_ring *rxr = que->rxr; 4327 struct ifnet *ifp = adapter->ifp; 4328 #if 0 /* NET_LRO */ 4329 struct lro_ctrl *lro = &rxr->lro; 4330 struct lro_entry *queued; 4331 #endif 4332 int i, nextp, processed = 0; 4333 u32 staterr = 0; 4334 union ixgbe_adv_rx_desc *cur; 4335 struct ixgbe_rx_buf *rbuf, *nbuf; 4336 4337 IXGBE_RX_LOCK(rxr); 4338 4339 #ifdef DEV_NETMAP 4340 if (ifp->if_capenable & IFCAP_NETMAP) { 4341 /* 4342 * Same as the txeof routine: only wakeup clients on intr. 4343 * NKR_PENDINTR in nr_kflags is used to implement interrupt 4344 * mitigation (ixgbe_rxsync() will not look for new packets 4345 * unless NKR_PENDINTR is set). 4346 */ 4347 struct netmap_adapter *na = NA(ifp); 4348 4349 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR; 4350 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET); 4351 IXGBE_RX_UNLOCK(rxr); 4352 IXGBE_CORE_LOCK(adapter); 4353 selwakeuppri(&na->rx_si, PI_NET); 4354 IXGBE_CORE_UNLOCK(adapter); 4355 return (FALSE); 4356 } 4357 #endif /* DEV_NETMAP */ 4358 for (i = rxr->next_to_check; count != 0;) { 4359 struct mbuf *sendmp, *mh, *mp; 4360 u32 rsc, ptype; 4361 u16 hlen, plen, hdr; 4362 u16 vtag = 0; 4363 bool eop; 4364 4365 /* Sync the ring. */ 4366 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4367 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4368 4369 cur = &rxr->rx_base[i]; 4370 staterr = le32toh(cur->wb.upper.status_error); 4371 4372 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 4373 break; 4374 if ((ifp->if_flags & IFF_RUNNING) == 0) 4375 break; 4376 4377 count--; 4378 sendmp = NULL; 4379 nbuf = NULL; 4380 rsc = 0; 4381 cur->wb.upper.status_error = 0; 4382 rbuf = &rxr->rx_buffers[i]; 4383 mh = rbuf->m_head; 4384 mp = rbuf->m_pack; 4385 4386 plen = le16toh(cur->wb.upper.length); 4387 ptype = le32toh(cur->wb.lower.lo_dword.data) & 4388 IXGBE_RXDADV_PKTTYPE_MASK; 4389 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4390 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 4391 4392 /* Process vlan info */ 4393 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) 4394 vtag = le16toh(cur->wb.upper.vlan); 4395 4396 /* Make sure bad packets are discarded */ 4397 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 4398 (rxr->discard)) { 4399 IFNET_STAT_INC(ifp, ierrors, 1); 4400 rxr->rx_discarded++; 4401 if (eop) 4402 rxr->discard = FALSE; 4403 else 4404 rxr->discard = TRUE; 4405 ixgbe_rx_discard(rxr, i); 4406 goto next_desc; 4407 } 4408 4409 /* 4410 ** On 82599 which supports a hardware 4411 ** LRO (called HW RSC), packets need 4412 ** not be fragmented across sequential 4413 ** descriptors, rather the next descriptor 4414 ** is indicated in bits of the descriptor. 4415 ** This also means that we might proceses 4416 ** more than one packet at a time, something 4417 ** that has never been true before, it 4418 ** required eliminating global chain pointers 4419 ** in favor of what we are doing here. -jfv 4420 */ 4421 if (!eop) { 4422 /* 4423 ** Figure out the next descriptor 4424 ** of this frame. 4425 */ 4426 if (rxr->hw_rsc == TRUE) { 4427 rsc = ixgbe_rsc_count(cur); 4428 rxr->rsc_num += (rsc - 1); 4429 } 4430 if (rsc) { /* Get hardware index */ 4431 nextp = ((staterr & 4432 IXGBE_RXDADV_NEXTP_MASK) >> 4433 IXGBE_RXDADV_NEXTP_SHIFT); 4434 } else { /* Just sequential */ 4435 nextp = i + 1; 4436 if (nextp == adapter->num_rx_desc) 4437 nextp = 0; 4438 } 4439 nbuf = &rxr->rx_buffers[nextp]; 4440 prefetch(nbuf); 4441 } 4442 /* 4443 ** The header mbuf is ONLY used when header 4444 ** split is enabled, otherwise we get normal 4445 ** behavior, ie, both header and payload 4446 ** are DMA'd into the payload buffer. 4447 ** 4448 ** Rather than using the fmp/lmp global pointers 4449 ** we now keep the head of a packet chain in the 4450 ** buffer struct and pass this along from one 4451 ** descriptor to the next, until we get EOP. 4452 */ 4453 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 4454 /* This must be an initial descriptor */ 4455 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 4456 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 4457 if (hlen > IXGBE_RX_HDR) 4458 hlen = IXGBE_RX_HDR; 4459 mh->m_len = hlen; 4460 mh->m_flags |= M_PKTHDR; 4461 mh->m_next = NULL; 4462 mh->m_pkthdr.len = mh->m_len; 4463 /* Null buf pointer so it is refreshed */ 4464 rbuf->m_head = NULL; 4465 /* 4466 ** Check the payload length, this 4467 ** could be zero if its a small 4468 ** packet. 4469 */ 4470 if (plen > 0) { 4471 mp->m_len = plen; 4472 mp->m_next = NULL; 4473 mp->m_flags &= ~M_PKTHDR; 4474 mh->m_next = mp; 4475 mh->m_pkthdr.len += mp->m_len; 4476 /* Null buf pointer so it is refreshed */ 4477 rbuf->m_pack = NULL; 4478 rxr->rx_split_packets++; 4479 } 4480 /* 4481 ** Now create the forward 4482 ** chain so when complete 4483 ** we wont have to. 4484 */ 4485 if (eop == 0) { 4486 /* stash the chain head */ 4487 nbuf->fmp = mh; 4488 /* Make forward chain */ 4489 if (plen) 4490 mp->m_next = nbuf->m_pack; 4491 else 4492 mh->m_next = nbuf->m_pack; 4493 } else { 4494 /* Singlet, prepare to send */ 4495 sendmp = mh; 4496 /* If hardware handled vtag */ 4497 if (vtag) { 4498 sendmp->m_pkthdr.ether_vlantag = vtag; 4499 sendmp->m_flags |= M_VLANTAG; 4500 } 4501 } 4502 } else { 4503 /* 4504 ** Either no header split, or a 4505 ** secondary piece of a fragmented 4506 ** split packet. 4507 */ 4508 mp->m_len = plen; 4509 /* 4510 ** See if there is a stored head 4511 ** that determines what we are 4512 */ 4513 sendmp = rbuf->fmp; 4514 rbuf->m_pack = rbuf->fmp = NULL; 4515 4516 if (sendmp != NULL) { /* secondary frag */ 4517 mp->m_flags &= ~M_PKTHDR; 4518 sendmp->m_pkthdr.len += mp->m_len; 4519 } else { 4520 /* first desc of a non-ps chain */ 4521 sendmp = mp; 4522 sendmp->m_flags |= M_PKTHDR; 4523 sendmp->m_pkthdr.len = mp->m_len; 4524 if (staterr & IXGBE_RXD_STAT_VP) { 4525 sendmp->m_pkthdr.ether_vlantag = vtag; 4526 sendmp->m_flags |= M_VLANTAG; 4527 } 4528 } 4529 /* Pass the head pointer on */ 4530 if (eop == 0) { 4531 nbuf->fmp = sendmp; 4532 sendmp = NULL; 4533 mp->m_next = nbuf->m_pack; 4534 } 4535 } 4536 ++processed; 4537 /* Sending this frame? */ 4538 if (eop) { 4539 sendmp->m_pkthdr.rcvif = ifp; 4540 IFNET_STAT_INC(ifp, ipackets, 1); 4541 rxr->rx_packets++; 4542 /* capture data for AIM */ 4543 rxr->bytes += sendmp->m_pkthdr.len; 4544 rxr->rx_bytes += sendmp->m_pkthdr.len; 4545 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 4546 ixgbe_rx_checksum(staterr, sendmp, ptype); 4547 #if 0 /* __FreeBSD_version >= 800000 */ 4548 sendmp->m_pkthdr.flowid = que->msix; 4549 sendmp->m_flags |= M_FLOWID; 4550 #endif 4551 } 4552 next_desc: 4553 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4554 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4555 4556 /* Advance our pointers to the next descriptor. */ 4557 if (++i == adapter->num_rx_desc) 4558 i = 0; 4559 4560 /* Now send to the stack or do LRO */ 4561 if (sendmp != NULL) { 4562 rxr->next_to_check = i; 4563 ixgbe_rx_input(rxr, ifp, sendmp, ptype); 4564 i = rxr->next_to_check; 4565 } 4566 4567 /* Every 8 descriptors we go to refresh mbufs */ 4568 if (processed == 8) { 4569 ixgbe_refresh_mbufs(rxr, i); 4570 processed = 0; 4571 } 4572 } 4573 4574 /* Refresh any remaining buf structs */ 4575 if (ixgbe_rx_unrefreshed(rxr)) 4576 ixgbe_refresh_mbufs(rxr, i); 4577 4578 rxr->next_to_check = i; 4579 4580 /* 4581 * Flush any outstanding LRO work 4582 */ 4583 #if 0 /* NET_LRO */ 4584 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4585 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4586 tcp_lro_flush(lro, queued); 4587 } 4588 #endif 4589 4590 IXGBE_RX_UNLOCK(rxr); 4591 4592 /* 4593 ** We still have cleaning to do? 4594 ** Schedule another interrupt if so. 4595 */ 4596 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 4597 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix)); 4598 } 4599 } 4600 4601 4602 /********************************************************************* 4603 * 4604 * Verify that the hardware indicated that the checksum is valid. 4605 * Inform the stack about the status of checksum so that stack 4606 * doesn't spend time verifying the checksum. 4607 * 4608 *********************************************************************/ 4609 static void 4610 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 4611 { 4612 u16 status = (u16) staterr; 4613 u8 errors = (u8) (staterr >> 24); 4614 bool sctp = FALSE; 4615 4616 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4617 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 4618 sctp = TRUE; 4619 4620 if (status & IXGBE_RXD_STAT_IPCS) { 4621 if (!(errors & IXGBE_RXD_ERR_IPE)) { 4622 /* IP Checksum Good */ 4623 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4624 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4625 4626 } else 4627 mp->m_pkthdr.csum_flags = 0; 4628 } 4629 if (status & IXGBE_RXD_STAT_L4CS) { 4630 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4631 #if 0 4632 if (sctp) 4633 type = CSUM_SCTP_VALID; 4634 #endif 4635 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 4636 mp->m_pkthdr.csum_flags |= type; 4637 if (!sctp) 4638 mp->m_pkthdr.csum_data = htons(0xffff); 4639 } 4640 } 4641 return; 4642 } 4643 4644 4645 /* 4646 ** This routine is run via an vlan config EVENT, 4647 ** it enables us to use the HW Filter table since 4648 ** we can get the vlan id. This just creates the 4649 ** entry in the soft version of the VFTA, init will 4650 ** repopulate the real table. 4651 */ 4652 static void 4653 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4654 { 4655 struct adapter *adapter = ifp->if_softc; 4656 u16 index, bit; 4657 4658 if (ifp->if_softc != arg) /* Not our event */ 4659 return; 4660 4661 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4662 return; 4663 4664 IXGBE_CORE_LOCK(adapter); 4665 index = (vtag >> 5) & 0x7F; 4666 bit = vtag & 0x1F; 4667 adapter->shadow_vfta[index] |= (1 << bit); 4668 ++adapter->num_vlans; 4669 ixgbe_init_locked(adapter); 4670 IXGBE_CORE_UNLOCK(adapter); 4671 } 4672 4673 /* 4674 ** This routine is run via an vlan 4675 ** unconfig EVENT, remove our entry 4676 ** in the soft vfta. 4677 */ 4678 static void 4679 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4680 { 4681 struct adapter *adapter = ifp->if_softc; 4682 u16 index, bit; 4683 4684 if (ifp->if_softc != arg) 4685 return; 4686 4687 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4688 return; 4689 4690 IXGBE_CORE_LOCK(adapter); 4691 index = (vtag >> 5) & 0x7F; 4692 bit = vtag & 0x1F; 4693 adapter->shadow_vfta[index] &= ~(1 << bit); 4694 --adapter->num_vlans; 4695 /* Re-init to load the changes */ 4696 ixgbe_init_locked(adapter); 4697 IXGBE_CORE_UNLOCK(adapter); 4698 } 4699 4700 static void 4701 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 4702 { 4703 struct ifnet *ifp = adapter->ifp; 4704 struct ixgbe_hw *hw = &adapter->hw; 4705 struct rx_ring *rxr; 4706 u32 ctrl; 4707 4708 /* 4709 ** We get here thru init_locked, meaning 4710 ** a soft reset, this has already cleared 4711 ** the VFTA and other state, so if there 4712 ** have been no vlan's registered do nothing. 4713 */ 4714 if (adapter->num_vlans == 0) 4715 return; 4716 4717 /* 4718 ** A soft reset zero's out the VFTA, so 4719 ** we need to repopulate it now. 4720 */ 4721 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 4722 if (adapter->shadow_vfta[i] != 0) 4723 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 4724 adapter->shadow_vfta[i]); 4725 4726 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4727 /* Enable the Filter Table if enabled */ 4728 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 4729 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 4730 ctrl |= IXGBE_VLNCTRL_VFE; 4731 } 4732 if (hw->mac.type == ixgbe_mac_82598EB) 4733 ctrl |= IXGBE_VLNCTRL_VME; 4734 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 4735 4736 /* Setup the queues for vlans */ 4737 for (int i = 0; i < adapter->num_queues; i++) { 4738 rxr = &adapter->rx_rings[i]; 4739 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 4740 if (hw->mac.type != ixgbe_mac_82598EB) { 4741 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 4742 ctrl |= IXGBE_RXDCTL_VME; 4743 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 4744 } 4745 rxr->vtag_strip = TRUE; 4746 } 4747 } 4748 4749 static void 4750 ixgbe_enable_intr(struct adapter *adapter) 4751 { 4752 struct ixgbe_hw *hw = &adapter->hw; 4753 struct ix_queue *que = adapter->queues; 4754 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 4755 4756 4757 /* Enable Fan Failure detection */ 4758 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4759 mask |= IXGBE_EIMS_GPI_SDP1; 4760 else { 4761 mask |= IXGBE_EIMS_ECC; 4762 mask |= IXGBE_EIMS_GPI_SDP0; 4763 mask |= IXGBE_EIMS_GPI_SDP1; 4764 mask |= IXGBE_EIMS_GPI_SDP2; 4765 #ifdef IXGBE_FDIR 4766 mask |= IXGBE_EIMS_FLOW_DIR; 4767 #endif 4768 } 4769 4770 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 4771 4772 /* With RSS we use auto clear */ 4773 if (adapter->msix_mem) { 4774 mask = IXGBE_EIMS_ENABLE_MASK; 4775 /* Don't autoclear Link */ 4776 mask &= ~IXGBE_EIMS_OTHER; 4777 mask &= ~IXGBE_EIMS_LSC; 4778 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 4779 } 4780 4781 /* 4782 ** Now enable all queues, this is done separately to 4783 ** allow for handling the extended (beyond 32) MSIX 4784 ** vectors that can be used by 82599 4785 */ 4786 for (int i = 0; i < adapter->num_queues; i++, que++) 4787 ixgbe_enable_queue(adapter, que->msix); 4788 4789 IXGBE_WRITE_FLUSH(hw); 4790 4791 return; 4792 } 4793 4794 static void 4795 ixgbe_disable_intr(struct adapter *adapter) 4796 { 4797 if (adapter->msix_mem) 4798 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 4799 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 4800 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 4801 } else { 4802 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 4803 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 4804 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 4805 } 4806 IXGBE_WRITE_FLUSH(&adapter->hw); 4807 return; 4808 } 4809 4810 u16 4811 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 4812 { 4813 u16 value; 4814 4815 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, 4816 reg, 2); 4817 4818 return (value); 4819 } 4820 4821 void 4822 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 4823 { 4824 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev, 4825 reg, value, 2); 4826 4827 return; 4828 } 4829 4830 /* 4831 ** Setup the correct IVAR register for a particular MSIX interrupt 4832 ** (yes this is all very magic and confusing :) 4833 ** - entry is the register array entry 4834 ** - vector is the MSIX vector for this queue 4835 ** - type is RX/TX/MISC 4836 */ 4837 static void 4838 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4839 { 4840 struct ixgbe_hw *hw = &adapter->hw; 4841 u32 ivar, index; 4842 4843 vector |= IXGBE_IVAR_ALLOC_VAL; 4844 4845 switch (hw->mac.type) { 4846 4847 case ixgbe_mac_82598EB: 4848 if (type == -1) 4849 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4850 else 4851 entry += (type * 64); 4852 index = (entry >> 2) & 0x1F; 4853 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4854 ivar &= ~(0xFF << (8 * (entry & 0x3))); 4855 ivar |= (vector << (8 * (entry & 0x3))); 4856 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4857 break; 4858 4859 case ixgbe_mac_82599EB: 4860 case ixgbe_mac_X540: 4861 if (type == -1) { /* MISC IVAR */ 4862 index = (entry & 1) * 8; 4863 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4864 ivar &= ~(0xFF << index); 4865 ivar |= (vector << index); 4866 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4867 } else { /* RX/TX IVARS */ 4868 index = (16 * (entry & 1)) + (8 * type); 4869 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4870 ivar &= ~(0xFF << index); 4871 ivar |= (vector << index); 4872 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4873 } 4874 4875 default: 4876 break; 4877 } 4878 } 4879 4880 static void 4881 ixgbe_configure_ivars(struct adapter *adapter) 4882 { 4883 struct ix_queue *que = adapter->queues; 4884 4885 for (int i = 0; i < adapter->num_queues; i++, que++) { 4886 /* First the RX queue entry */ 4887 ixgbe_set_ivar(adapter, i, que->msix, 0); 4888 /* ... and the TX */ 4889 ixgbe_set_ivar(adapter, i, que->msix, 1); 4890 /* Set an Initial EITR value */ 4891 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE); 4892 } 4893 4894 /* For the Link interrupt */ 4895 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); 4896 } 4897 4898 /* 4899 ** ixgbe_sfp_probe - called in the local timer to 4900 ** determine if a port had optics inserted. 4901 */ 4902 static bool ixgbe_sfp_probe(struct adapter *adapter) 4903 { 4904 struct ixgbe_hw *hw = &adapter->hw; 4905 device_t dev = adapter->dev; 4906 bool result = FALSE; 4907 4908 if ((hw->phy.type == ixgbe_phy_nl) && 4909 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 4910 s32 ret = hw->phy.ops.identify_sfp(hw); 4911 if (ret) 4912 goto out; 4913 ret = hw->phy.ops.reset(hw); 4914 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4915 device_printf(dev,"Unsupported SFP+ module detected!"); 4916 kprintf(" Reload driver with supported module.\n"); 4917 adapter->sfp_probe = FALSE; 4918 goto out; 4919 } else 4920 device_printf(dev,"SFP+ module detected!\n"); 4921 /* We now have supported optics */ 4922 adapter->sfp_probe = FALSE; 4923 /* Set the optics type so system reports correctly */ 4924 ixgbe_setup_optics(adapter); 4925 result = TRUE; 4926 } 4927 out: 4928 return (result); 4929 } 4930 4931 /* 4932 ** Tasklet for handling SFP module interrupts 4933 */ 4934 static void 4935 ixgbe_handle_mod(void *context, int pending) 4936 { 4937 struct adapter *adapter = context; 4938 struct ixgbe_hw *hw = &adapter->hw; 4939 device_t dev = adapter->dev; 4940 u32 err; 4941 4942 err = hw->phy.ops.identify_sfp(hw); 4943 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4944 device_printf(dev, 4945 "Unsupported SFP+ module type was detected.\n"); 4946 return; 4947 } 4948 err = hw->mac.ops.setup_sfp(hw); 4949 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4950 device_printf(dev, 4951 "Setup failure - unsupported SFP+ module type.\n"); 4952 return; 4953 } 4954 taskqueue_enqueue(adapter->tq, &adapter->msf_task); 4955 return; 4956 } 4957 4958 4959 /* 4960 ** Tasklet for handling MSF (multispeed fiber) interrupts 4961 */ 4962 static void 4963 ixgbe_handle_msf(void *context, int pending) 4964 { 4965 struct adapter *adapter = context; 4966 struct ixgbe_hw *hw = &adapter->hw; 4967 u32 autoneg; 4968 bool negotiate; 4969 4970 autoneg = hw->phy.autoneg_advertised; 4971 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4972 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4973 if (hw->mac.ops.setup_link) 4974 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); 4975 return; 4976 } 4977 4978 #ifdef IXGBE_FDIR 4979 /* 4980 ** Tasklet for reinitializing the Flow Director filter table 4981 */ 4982 static void 4983 ixgbe_reinit_fdir(void *context, int pending) 4984 { 4985 struct adapter *adapter = context; 4986 struct ifnet *ifp = adapter->ifp; 4987 4988 if (adapter->fdir_reinit != 1) /* Shouldn't happen */ 4989 return; 4990 ixgbe_reinit_fdir_tables_82599(&adapter->hw); 4991 adapter->fdir_reinit = 0; 4992 /* re-enable flow director interrupts */ 4993 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); 4994 /* Restart the interface */ 4995 ifp->if_drv_flags |= IFF_DRV_RUNNING; 4996 return; 4997 } 4998 #endif 4999 5000 /********************************************************************** 5001 * 5002 * Update the board statistics counters. 5003 * 5004 **********************************************************************/ 5005 static void 5006 ixgbe_update_stats_counters(struct adapter *adapter) 5007 { 5008 struct ifnet *ifp = adapter->ifp; 5009 struct ixgbe_hw *hw = &adapter->hw; 5010 u32 missed_rx = 0, bprc, lxon, lxoff, total; 5011 u64 total_missed_rx = 0; 5012 5013 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5014 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 5015 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 5016 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 5017 5018 for (int i = 0; i < 8; i++) { 5019 u32 mp; 5020 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5021 /* missed_rx tallies misses for the gprc workaround */ 5022 missed_rx += mp; 5023 /* global total per queue */ 5024 adapter->stats.mpc[i] += mp; 5025 /* Running comprehensive total for stats display */ 5026 total_missed_rx += adapter->stats.mpc[i]; 5027 if (hw->mac.type == ixgbe_mac_82598EB) 5028 adapter->stats.rnbc[i] += 5029 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5030 adapter->stats.pxontxc[i] += 5031 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5032 adapter->stats.pxonrxc[i] += 5033 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5034 adapter->stats.pxofftxc[i] += 5035 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5036 adapter->stats.pxoffrxc[i] += 5037 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 5038 adapter->stats.pxon2offc[i] += 5039 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 5040 } 5041 for (int i = 0; i < 16; i++) { 5042 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5043 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5044 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5045 adapter->stats.qbrc[i] += 5046 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); 5047 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5048 adapter->stats.qbtc[i] += 5049 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); 5050 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5051 } 5052 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 5053 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 5054 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 5055 5056 /* Hardware workaround, gprc counts missed packets */ 5057 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 5058 adapter->stats.gprc -= missed_rx; 5059 5060 if (hw->mac.type != ixgbe_mac_82598EB) { 5061 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) + 5062 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 5063 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 5064 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 5065 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) + 5066 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 5067 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5068 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 5069 } else { 5070 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5071 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 5072 /* 82598 only has a counter in the high register */ 5073 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 5074 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 5075 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 5076 } 5077 5078 /* 5079 * Workaround: mprc hardware is incorrectly counting 5080 * broadcasts, so for now we subtract those. 5081 */ 5082 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5083 adapter->stats.bprc += bprc; 5084 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 5085 if (hw->mac.type == ixgbe_mac_82598EB) 5086 adapter->stats.mprc -= bprc; 5087 5088 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 5089 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 5090 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 5091 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 5092 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5093 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5094 5095 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5096 adapter->stats.lxontxc += lxon; 5097 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5098 adapter->stats.lxofftxc += lxoff; 5099 total = lxon + lxoff; 5100 5101 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 5102 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 5103 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 5104 adapter->stats.gptc -= total; 5105 adapter->stats.mptc -= total; 5106 adapter->stats.ptc64 -= total; 5107 adapter->stats.gotc -= total * ETHER_MIN_LEN; 5108 5109 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 5110 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 5111 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 5112 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 5113 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 5114 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 5115 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 5116 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 5117 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 5118 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 5119 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 5120 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 5121 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5122 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5123 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 5124 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC); 5125 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5126 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 5127 /* Only read FCOE on 82599 */ 5128 if (hw->mac.type != ixgbe_mac_82598EB) { 5129 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5130 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5131 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5132 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5133 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5134 } 5135 5136 /* Fill out the OS statistics structure */ 5137 IFNET_STAT_SET(ifp, ipackets, adapter->stats.gprc); 5138 IFNET_STAT_SET(ifp, opackets, adapter->stats.gptc); 5139 IFNET_STAT_SET(ifp, ibytes, adapter->stats.gorc); 5140 IFNET_STAT_SET(ifp, obytes, adapter->stats.gotc); 5141 IFNET_STAT_SET(ifp, imcasts, adapter->stats.mprc); 5142 5143 /* Rx Errors */ 5144 IFNET_STAT_SET(ifp, ierrors, total_missed_rx + adapter->stats.crcerrs + 5145 adapter->stats.rlec); 5146 } 5147 5148 /** ixgbe_sysctl_tdh_handler - Handler function 5149 * Retrieves the TDH value from the hardware 5150 */ 5151 static int 5152 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS) 5153 { 5154 int error; 5155 5156 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1); 5157 if (!txr) return 0; 5158 5159 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me)); 5160 error = sysctl_handle_int(oidp, &val, 0, req); 5161 if (error || !req->newptr) 5162 return error; 5163 return 0; 5164 } 5165 5166 /** ixgbe_sysctl_tdt_handler - Handler function 5167 * Retrieves the TDT value from the hardware 5168 */ 5169 static int 5170 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS) 5171 { 5172 int error; 5173 5174 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1); 5175 if (!txr) return 0; 5176 5177 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me)); 5178 error = sysctl_handle_int(oidp, &val, 0, req); 5179 if (error || !req->newptr) 5180 return error; 5181 return 0; 5182 } 5183 5184 /** ixgbe_sysctl_rdh_handler - Handler function 5185 * Retrieves the RDH value from the hardware 5186 */ 5187 static int 5188 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS) 5189 { 5190 int error; 5191 5192 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1); 5193 if (!rxr) return 0; 5194 5195 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me)); 5196 error = sysctl_handle_int(oidp, &val, 0, req); 5197 if (error || !req->newptr) 5198 return error; 5199 return 0; 5200 } 5201 5202 /** ixgbe_sysctl_rdt_handler - Handler function 5203 * Retrieves the RDT value from the hardware 5204 */ 5205 static int 5206 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS) 5207 { 5208 int error; 5209 5210 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1); 5211 if (!rxr) return 0; 5212 5213 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me)); 5214 error = sysctl_handle_int(oidp, &val, 0, req); 5215 if (error || !req->newptr) 5216 return error; 5217 return 0; 5218 } 5219 5220 /* 5221 * Add sysctl variables, one per statistic, to the system. 5222 */ 5223 static void 5224 ixgbe_add_hw_stats(struct adapter *adapter) 5225 { 5226 struct tx_ring *txr = adapter->tx_rings; 5227 struct rx_ring *rxr = adapter->rx_rings; 5228 5229 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx; 5230 struct sysctl_oid *tree = adapter->sysctl_tree; 5231 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 5232 struct ixgbe_hw_stats *stats = &adapter->stats; 5233 5234 struct sysctl_oid *stat_node, *queue_node; 5235 struct sysctl_oid_list *stat_list, *queue_list; 5236 5237 #define QUEUE_NAME_LEN 32 5238 char namebuf[QUEUE_NAME_LEN]; 5239 5240 /* Driver Statistics */ 5241 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5242 CTLFLAG_RD, &adapter->dropped_pkts, 5243 "Driver dropped packets"); 5244 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed", 5245 CTLFLAG_RD, &adapter->mbuf_defrag_failed, 5246 "m_defrag() failed"); 5247 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup", 5248 CTLFLAG_RD, &adapter->no_tx_dma_setup, 5249 "Driver tx dma failure in xmit"); 5250 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", 5251 CTLFLAG_RD, &adapter->watchdog_events, 5252 "Watchdog timeouts"); 5253 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx", 5254 CTLFLAG_RD, &adapter->tso_tx, 5255 "TSO"); 5256 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 5257 CTLFLAG_RD, &adapter->link_irq, 5258 "Link MSIX IRQ Handled"); 5259 5260 for (int i = 0; i < adapter->num_queues; i++, txr++) { 5261 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5262 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5263 CTLFLAG_RD, NULL, "Queue Name"); 5264 queue_list = SYSCTL_CHILDREN(queue_node); 5265 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", 5266 CTLFLAG_RD, &(adapter->queues[i].irqs), 0, 5267 "irqs on this queue"); 5268 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 5269 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr), 5270 ixgbe_sysctl_tdh_handler, "IU", 5271 "Transmit Descriptor Head"); 5272 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 5273 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr), 5274 ixgbe_sysctl_tdt_handler, "IU", 5275 "Transmit Descriptor Tail"); 5276 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 5277 CTLFLAG_RD, &txr->no_desc_avail, 0, 5278 "Queue No Descriptor Available"); 5279 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", 5280 CTLFLAG_RD, &txr->total_packets, 0, 5281 "Queue Packets Transmitted"); 5282 } 5283 5284 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 5285 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5286 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5287 CTLFLAG_RD, NULL, "Queue Name"); 5288 queue_list = SYSCTL_CHILDREN(queue_node); 5289 5290 #if 0 /* NET_LRO */ 5291 struct lro_ctrl *lro = &rxr->lro; 5292 #endif 5293 5294 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5295 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5296 CTLFLAG_RD, NULL, "Queue Name"); 5297 queue_list = SYSCTL_CHILDREN(queue_node); 5298 5299 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 5300 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr), 5301 ixgbe_sysctl_rdh_handler, "IU", 5302 "Receive Descriptor Head"); 5303 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 5304 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr), 5305 ixgbe_sysctl_rdt_handler, "IU", 5306 "Receive Descriptor Tail"); 5307 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", 5308 CTLFLAG_RD, &rxr->rx_packets, 0, 5309 "Queue Packets Received"); 5310 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 5311 CTLFLAG_RD, &rxr->rx_bytes, 0, 5312 "Queue Bytes Received"); 5313 #if 0 /* NET_LRO */ 5314 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 5315 CTLFLAG_RD, &lro->lro_queued, 0, 5316 "LRO Queued"); 5317 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 5318 CTLFLAG_RD, &lro->lro_flushed, 0, 5319 "LRO Flushed"); 5320 #endif 5321 } 5322 5323 /* MAC stats get the own sub node */ 5324 5325 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 5326 CTLFLAG_RD, NULL, "MAC Statistics"); 5327 stat_list = SYSCTL_CHILDREN(stat_node); 5328 5329 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 5330 CTLFLAG_RD, &stats->crcerrs, 0, 5331 "CRC Errors"); 5332 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs", 5333 CTLFLAG_RD, &stats->illerrc, 0, 5334 "Illegal Byte Errors"); 5335 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs", 5336 CTLFLAG_RD, &stats->errbc, 0, 5337 "Byte Errors"); 5338 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards", 5339 CTLFLAG_RD, &stats->mspdc, 0, 5340 "MAC Short Packets Discarded"); 5341 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults", 5342 CTLFLAG_RD, &stats->mlfc, 0, 5343 "MAC Local Faults"); 5344 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults", 5345 CTLFLAG_RD, &stats->mrfc, 0, 5346 "MAC Remote Faults"); 5347 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs", 5348 CTLFLAG_RD, &stats->rlec, 0, 5349 "Receive Length Errors"); 5350 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd", 5351 CTLFLAG_RD, &stats->lxontxc, 0, 5352 "Link XON Transmitted"); 5353 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd", 5354 CTLFLAG_RD, &stats->lxonrxc, 0, 5355 "Link XON Received"); 5356 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd", 5357 CTLFLAG_RD, &stats->lxofftxc, 0, 5358 "Link XOFF Transmitted"); 5359 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd", 5360 CTLFLAG_RD, &stats->lxoffrxc, 0, 5361 "Link XOFF Received"); 5362 5363 /* Packet Reception Stats */ 5364 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd", 5365 CTLFLAG_RD, &stats->tor, 0, 5366 "Total Octets Received"); 5367 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd", 5368 CTLFLAG_RD, &stats->gorc, 0, 5369 "Good Octets Received"); 5370 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd", 5371 CTLFLAG_RD, &stats->tpr, 0, 5372 "Total Packets Received"); 5373 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd", 5374 CTLFLAG_RD, &stats->gprc, 0, 5375 "Good Packets Received"); 5376 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd", 5377 CTLFLAG_RD, &stats->mprc, 0, 5378 "Multicast Packets Received"); 5379 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd", 5380 CTLFLAG_RD, &stats->bprc, 0, 5381 "Broadcast Packets Received"); 5382 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 5383 CTLFLAG_RD, &stats->prc64, 0, 5384 "64 byte frames received "); 5385 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 5386 CTLFLAG_RD, &stats->prc127, 0, 5387 "65-127 byte frames received"); 5388 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 5389 CTLFLAG_RD, &stats->prc255, 0, 5390 "128-255 byte frames received"); 5391 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 5392 CTLFLAG_RD, &stats->prc511, 0, 5393 "256-511 byte frames received"); 5394 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 5395 CTLFLAG_RD, &stats->prc1023, 0, 5396 "512-1023 byte frames received"); 5397 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 5398 CTLFLAG_RD, &stats->prc1522, 0, 5399 "1023-1522 byte frames received"); 5400 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized", 5401 CTLFLAG_RD, &stats->ruc, 0, 5402 "Receive Undersized"); 5403 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 5404 CTLFLAG_RD, &stats->rfc, 0, 5405 "Fragmented Packets Received "); 5406 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized", 5407 CTLFLAG_RD, &stats->roc, 0, 5408 "Oversized Packets Received"); 5409 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd", 5410 CTLFLAG_RD, &stats->rjc, 0, 5411 "Received Jabber"); 5412 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd", 5413 CTLFLAG_RD, &stats->mngprc, 0, 5414 "Management Packets Received"); 5415 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd", 5416 CTLFLAG_RD, &stats->mngptc, 0, 5417 "Management Packets Dropped"); 5418 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs", 5419 CTLFLAG_RD, &stats->xec, 0, 5420 "Checksum Errors"); 5421 5422 /* Packet Transmission Stats */ 5423 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 5424 CTLFLAG_RD, &stats->gotc, 0, 5425 "Good Octets Transmitted"); 5426 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 5427 CTLFLAG_RD, &stats->tpt, 0, 5428 "Total Packets Transmitted"); 5429 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 5430 CTLFLAG_RD, &stats->gptc, 0, 5431 "Good Packets Transmitted"); 5432 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 5433 CTLFLAG_RD, &stats->bptc, 0, 5434 "Broadcast Packets Transmitted"); 5435 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 5436 CTLFLAG_RD, &stats->mptc, 0, 5437 "Multicast Packets Transmitted"); 5438 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd", 5439 CTLFLAG_RD, &stats->mngptc, 0, 5440 "Management Packets Transmitted"); 5441 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 5442 CTLFLAG_RD, &stats->ptc64, 0, 5443 "64 byte frames transmitted "); 5444 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 5445 CTLFLAG_RD, &stats->ptc127, 0, 5446 "65-127 byte frames transmitted"); 5447 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 5448 CTLFLAG_RD, &stats->ptc255, 0, 5449 "128-255 byte frames transmitted"); 5450 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 5451 CTLFLAG_RD, &stats->ptc511, 0, 5452 "256-511 byte frames transmitted"); 5453 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 5454 CTLFLAG_RD, &stats->ptc1023, 0, 5455 "512-1023 byte frames transmitted"); 5456 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 5457 CTLFLAG_RD, &stats->ptc1522, 0, 5458 "1024-1522 byte frames transmitted"); 5459 5460 /* FC Stats */ 5461 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc", 5462 CTLFLAG_RD, &stats->fccrc, 0, 5463 "FC CRC Errors"); 5464 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last", 5465 CTLFLAG_RD, &stats->fclast, 0, 5466 "FC Last Error"); 5467 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd", 5468 CTLFLAG_RD, &stats->fcoerpdc, 0, 5469 "FCoE Packets Dropped"); 5470 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd", 5471 CTLFLAG_RD, &stats->fcoeprc, 0, 5472 "FCoE Packets Received"); 5473 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd", 5474 CTLFLAG_RD, &stats->fcoeptc, 0, 5475 "FCoE Packets Transmitted"); 5476 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd", 5477 CTLFLAG_RD, &stats->fcoedwrc, 0, 5478 "FCoE DWords Received"); 5479 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd", 5480 CTLFLAG_RD, &stats->fcoedwtc, 0, 5481 "FCoE DWords Transmitted"); 5482 } 5483 5484 /* 5485 ** Set flow control using sysctl: 5486 ** Flow control values: 5487 ** 0 - off 5488 ** 1 - rx pause 5489 ** 2 - tx pause 5490 ** 3 - full 5491 */ 5492 static int 5493 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) 5494 { 5495 int error, last; 5496 struct adapter *adapter = (struct adapter *) arg1; 5497 5498 last = adapter->fc; 5499 error = sysctl_handle_int(oidp, &adapter->fc, 0, req); 5500 if ((error) || (req->newptr == NULL)) 5501 return (error); 5502 5503 /* Don't bother if it's not changed */ 5504 if (adapter->fc == last) 5505 return (0); 5506 5507 switch (adapter->fc) { 5508 case ixgbe_fc_rx_pause: 5509 case ixgbe_fc_tx_pause: 5510 case ixgbe_fc_full: 5511 adapter->hw.fc.requested_mode = adapter->fc; 5512 break; 5513 case ixgbe_fc_none: 5514 default: 5515 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5516 } 5517 /* Don't autoneg if forcing a value */ 5518 adapter->hw.fc.disable_fc_autoneg = TRUE; 5519 ixgbe_fc_enable(&adapter->hw); 5520 return error; 5521 } 5522 5523 static void 5524 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 5525 const char *description, int *limit, int value) 5526 { 5527 *limit = value; 5528 SYSCTL_ADD_INT(&adapter->sysctl_ctx, 5529 SYSCTL_CHILDREN(adapter->sysctl_tree), 5530 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5531 } 5532 5533 /* 5534 ** Control link advertise speed: 5535 ** 1 - advertise only 1G 5536 ** 2 - advertise 100Mb 5537 ** 3 - advertise normal 5538 */ 5539 static int 5540 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS) 5541 { 5542 int error = 0; 5543 struct adapter *adapter; 5544 device_t dev; 5545 struct ixgbe_hw *hw; 5546 ixgbe_link_speed speed, last; 5547 5548 adapter = (struct adapter *) arg1; 5549 dev = adapter->dev; 5550 hw = &adapter->hw; 5551 last = adapter->advertise; 5552 5553 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req); 5554 if ((error) || (adapter->advertise == -1)) 5555 return (error); 5556 5557 if (adapter->advertise == last) /* no change */ 5558 return (0); 5559 5560 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5561 (hw->phy.multispeed_fiber))) 5562 return (error); 5563 5564 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) { 5565 device_printf(dev, "Set Advertise: 100Mb on X540 only\n"); 5566 return (error); 5567 } 5568 5569 if (adapter->advertise == 1) 5570 speed = IXGBE_LINK_SPEED_1GB_FULL; 5571 else if (adapter->advertise == 2) 5572 speed = IXGBE_LINK_SPEED_100_FULL; 5573 else if (adapter->advertise == 3) 5574 speed = IXGBE_LINK_SPEED_1GB_FULL | 5575 IXGBE_LINK_SPEED_10GB_FULL; 5576 else /* bogus value */ 5577 return (error); 5578 5579 hw->mac.autotry_restart = TRUE; 5580 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); 5581 5582 return (error); 5583 } 5584 5585 /* 5586 ** Thermal Shutdown Trigger 5587 ** - cause a Thermal Overtemp IRQ 5588 */ 5589 static int 5590 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS) 5591 { 5592 int error, fire = 0; 5593 struct adapter *adapter = (struct adapter *) arg1; 5594 struct ixgbe_hw *hw = &adapter->hw; 5595 5596 5597 if (hw->mac.type != ixgbe_mac_X540) 5598 return (0); 5599 5600 error = sysctl_handle_int(oidp, &fire, 0, req); 5601 if ((error) || (req->newptr == NULL)) 5602 return (error); 5603 5604 if (fire) { 5605 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS); 5606 reg |= IXGBE_EICR_TS; 5607 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg); 5608 } 5609 5610 return (0); 5611 } 5612 5613 static void 5614 ixgbe_set_eitr(struct adapter *sc, int idx, int rate) 5615 { 5616 uint32_t eitr = 0; 5617 5618 /* convert rate in intr/s to hw representation */ 5619 if (rate > 0) { 5620 eitr = 1000000 / rate; 5621 eitr <<= IXGBE_EITR_INTVL_SHIFT; 5622 5623 if (eitr == 0) { 5624 /* Don't disable it */ 5625 eitr = 1 << IXGBE_EITR_INTVL_SHIFT; 5626 } else if (eitr > IXGBE_EITR_INTVL_MASK) { 5627 /* Don't allow it to be too large */ 5628 eitr = IXGBE_EITR_INTVL_MASK; 5629 } 5630 } 5631 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr); 5632 } 5633 5634 static int 5635 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS) 5636 { 5637 struct adapter *sc = (void *)arg1; 5638 struct ifnet *ifp = sc->ifp; 5639 int error, intr_rate, running; 5640 struct ix_queue *que = sc->queues; 5641 5642 intr_rate = sc->intr_rate; 5643 error = sysctl_handle_int(oidp, &intr_rate, 0, req); 5644 if (error || req->newptr == NULL) 5645 return error; 5646 if (intr_rate < 0) 5647 return EINVAL; 5648 5649 ifnet_serialize_all(ifp); 5650 5651 sc->intr_rate = intr_rate; 5652 running = ifp->if_flags & IFF_RUNNING; 5653 if (running) 5654 ixgbe_set_eitr(sc, 0, sc->intr_rate); 5655 5656 if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) { 5657 for (int i = 0; i < sc->num_queues; i++, que++) 5658 ixgbe_set_eitr(sc, que->msix, sc->intr_rate); 5659 } 5660 5661 if (bootverbose) 5662 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate); 5663 5664 ifnet_deserialize_all(ifp); 5665 5666 return 0; 5667 } 5668 5669 /* rearrange mbuf chain to get contiguous bytes */ 5670 static int 5671 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp) 5672 { 5673 int hoff, iphlen, thoff; 5674 struct mbuf *m; 5675 5676 m = *mp; 5677 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 5678 5679 iphlen = m->m_pkthdr.csum_iphlen; 5680 thoff = m->m_pkthdr.csum_thlen; 5681 hoff = m->m_pkthdr.csum_lhlen; 5682 5683 KASSERT(iphlen > 0, ("invalid ip hlen")); 5684 KASSERT(thoff > 0, ("invalid tcp hlen")); 5685 KASSERT(hoff > 0, ("invalid ether hlen")); 5686 5687 if (__predict_false(m->m_len < hoff + iphlen + thoff)) { 5688 m = m_pullup(m, hoff + iphlen + thoff); 5689 if (m == NULL) { 5690 *mp = NULL; 5691 return ENOBUFS; 5692 } 5693 *mp = m; 5694 } 5695 5696 return 0; 5697 }
Cache object: c26ac9148de371fe75794e325814260a
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]