FreeBSD/Linux Kernel Cross Reference
sys/dev/e1000/if_igb.c
1 /******************************************************************************
2
3 Copyright (c) 2001-2011, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33 /*$FreeBSD: releng/9.0/sys/dev/e1000/if_igb.c 223831 2011-07-07 00:46:50Z jfv $*/
34
35
36 #ifdef HAVE_KERNEL_OPTION_HEADERS
37 #include "opt_device_polling.h"
38 #include "opt_inet.h"
39 #include "opt_inet6.h"
40 #include "opt_altq.h"
41 #endif
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #if __FreeBSD_version >= 800000
46 #include <sys/buf_ring.h>
47 #endif
48 #include <sys/bus.h>
49 #include <sys/endian.h>
50 #include <sys/kernel.h>
51 #include <sys/kthread.h>
52 #include <sys/malloc.h>
53 #include <sys/mbuf.h>
54 #include <sys/module.h>
55 #include <sys/rman.h>
56 #include <sys/socket.h>
57 #include <sys/sockio.h>
58 #include <sys/sysctl.h>
59 #include <sys/taskqueue.h>
60 #include <sys/eventhandler.h>
61 #include <sys/pcpu.h>
62 #include <sys/smp.h>
63 #include <machine/smp.h>
64 #include <machine/bus.h>
65 #include <machine/resource.h>
66
67 #include <net/bpf.h>
68 #include <net/ethernet.h>
69 #include <net/if.h>
70 #include <net/if_arp.h>
71 #include <net/if_dl.h>
72 #include <net/if_media.h>
73
74 #include <net/if_types.h>
75 #include <net/if_vlan_var.h>
76
77 #include <netinet/in_systm.h>
78 #include <netinet/in.h>
79 #include <netinet/if_ether.h>
80 #include <netinet/ip.h>
81 #include <netinet/ip6.h>
82 #include <netinet/tcp.h>
83 #include <netinet/tcp_lro.h>
84 #include <netinet/udp.h>
85
86 #include <machine/in_cksum.h>
87 #include <dev/led/led.h>
88 #include <dev/pci/pcivar.h>
89 #include <dev/pci/pcireg.h>
90
91 #include "e1000_api.h"
92 #include "e1000_82575.h"
93 #include "if_igb.h"
94
95 /*********************************************************************
96 * Set this to one to display debug statistics
97 *********************************************************************/
98 int igb_display_debug_stats = 0;
99
100 /*********************************************************************
101 * Driver version:
102 *********************************************************************/
103 char igb_driver_version[] = "version - 2.2.5";
104
105
106 /*********************************************************************
107 * PCI Device ID Table
108 *
109 * Used by probe to select devices to load on
110 * Last field stores an index into e1000_strings
111 * Last entry must be all 0s
112 *
113 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
114 *********************************************************************/
115
116 static igb_vendor_info_t igb_vendor_info_array[] =
117 {
118 { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
119 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
120 PCI_ANY_ID, PCI_ANY_ID, 0},
121 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
122 PCI_ANY_ID, PCI_ANY_ID, 0},
123 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0},
124 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
126 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
128 { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
129 PCI_ANY_ID, PCI_ANY_ID, 0},
130 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
131 PCI_ANY_ID, PCI_ANY_ID, 0},
132 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2,
133 PCI_ANY_ID, PCI_ANY_ID, 0},
134 { 0x8086, E1000_DEV_ID_82576_VF, PCI_ANY_ID, PCI_ANY_ID, 0},
135 { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
136 { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
137 { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
138 { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
139 { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL,
140 PCI_ANY_ID, PCI_ANY_ID, 0},
141 { 0x8086, E1000_DEV_ID_82580_QUAD_FIBER,
142 PCI_ANY_ID, PCI_ANY_ID, 0},
143 { 0x8086, E1000_DEV_ID_DH89XXCC_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
144 { 0x8086, E1000_DEV_ID_DH89XXCC_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
145 { 0x8086, E1000_DEV_ID_DH89XXCC_SFP, PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE,
147 PCI_ANY_ID, PCI_ANY_ID, 0},
148 { 0x8086, E1000_DEV_ID_I350_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_I350_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_I350_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
151 { 0x8086, E1000_DEV_ID_I350_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
152 { 0x8086, E1000_DEV_ID_I350_VF, PCI_ANY_ID, PCI_ANY_ID, 0},
153 /* required last entry */
154 { 0, 0, 0, 0, 0}
155 };
156
157 /*********************************************************************
158 * Table of branding strings for all supported NICs.
159 *********************************************************************/
160
161 static char *igb_strings[] = {
162 "Intel(R) PRO/1000 Network Connection"
163 };
164
165 /*********************************************************************
166 * Function prototypes
167 *********************************************************************/
168 static int igb_probe(device_t);
169 static int igb_attach(device_t);
170 static int igb_detach(device_t);
171 static int igb_shutdown(device_t);
172 static int igb_suspend(device_t);
173 static int igb_resume(device_t);
174 #if __FreeBSD_version >= 800000
175 static int igb_mq_start(struct ifnet *, struct mbuf *);
176 static int igb_mq_start_locked(struct ifnet *,
177 struct tx_ring *, struct mbuf *);
178 static void igb_qflush(struct ifnet *);
179 static void igb_deferred_mq_start(void *, int);
180 #else
181 static void igb_start(struct ifnet *);
182 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
183 #endif
184 static int igb_ioctl(struct ifnet *, u_long, caddr_t);
185 static void igb_init(void *);
186 static void igb_init_locked(struct adapter *);
187 static void igb_stop(void *);
188 static void igb_media_status(struct ifnet *, struct ifmediareq *);
189 static int igb_media_change(struct ifnet *);
190 static void igb_identify_hardware(struct adapter *);
191 static int igb_allocate_pci_resources(struct adapter *);
192 static int igb_allocate_msix(struct adapter *);
193 static int igb_allocate_legacy(struct adapter *);
194 static int igb_setup_msix(struct adapter *);
195 static void igb_free_pci_resources(struct adapter *);
196 static void igb_local_timer(void *);
197 static void igb_reset(struct adapter *);
198 static int igb_setup_interface(device_t, struct adapter *);
199 static int igb_allocate_queues(struct adapter *);
200 static void igb_configure_queues(struct adapter *);
201
202 static int igb_allocate_transmit_buffers(struct tx_ring *);
203 static void igb_setup_transmit_structures(struct adapter *);
204 static void igb_setup_transmit_ring(struct tx_ring *);
205 static void igb_initialize_transmit_units(struct adapter *);
206 static void igb_free_transmit_structures(struct adapter *);
207 static void igb_free_transmit_buffers(struct tx_ring *);
208
209 static int igb_allocate_receive_buffers(struct rx_ring *);
210 static int igb_setup_receive_structures(struct adapter *);
211 static int igb_setup_receive_ring(struct rx_ring *);
212 static void igb_initialize_receive_units(struct adapter *);
213 static void igb_free_receive_structures(struct adapter *);
214 static void igb_free_receive_buffers(struct rx_ring *);
215 static void igb_free_receive_ring(struct rx_ring *);
216
217 static void igb_enable_intr(struct adapter *);
218 static void igb_disable_intr(struct adapter *);
219 static void igb_update_stats_counters(struct adapter *);
220 static bool igb_txeof(struct tx_ring *);
221
222 static __inline void igb_rx_discard(struct rx_ring *, int);
223 static __inline void igb_rx_input(struct rx_ring *,
224 struct ifnet *, struct mbuf *, u32);
225
226 static bool igb_rxeof(struct igb_queue *, int, int *);
227 static void igb_rx_checksum(u32, struct mbuf *, u32);
228 static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
229 static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
230 static void igb_set_promisc(struct adapter *);
231 static void igb_disable_promisc(struct adapter *);
232 static void igb_set_multi(struct adapter *);
233 static void igb_update_link_status(struct adapter *);
234 static void igb_refresh_mbufs(struct rx_ring *, int);
235
236 static void igb_register_vlan(void *, struct ifnet *, u16);
237 static void igb_unregister_vlan(void *, struct ifnet *, u16);
238 static void igb_setup_vlan_hw_support(struct adapter *);
239
240 static int igb_xmit(struct tx_ring *, struct mbuf **);
241 static int igb_dma_malloc(struct adapter *, bus_size_t,
242 struct igb_dma_alloc *, int);
243 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
244 static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
245 static void igb_print_nvm_info(struct adapter *);
246 static int igb_is_valid_ether_addr(u8 *);
247 static void igb_add_hw_stats(struct adapter *);
248
249 static void igb_vf_init_stats(struct adapter *);
250 static void igb_update_vf_stats_counters(struct adapter *);
251
252 /* Management and WOL Support */
253 static void igb_init_manageability(struct adapter *);
254 static void igb_release_manageability(struct adapter *);
255 static void igb_get_hw_control(struct adapter *);
256 static void igb_release_hw_control(struct adapter *);
257 static void igb_enable_wakeup(device_t);
258 static void igb_led_func(void *, int);
259
260 static int igb_irq_fast(void *);
261 static void igb_msix_que(void *);
262 static void igb_msix_link(void *);
263 static void igb_handle_que(void *context, int pending);
264 static void igb_handle_link(void *context, int pending);
265
266 static void igb_set_sysctl_value(struct adapter *, const char *,
267 const char *, int *, int);
268 static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
269 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
270
271 #ifdef DEVICE_POLLING
272 static poll_handler_t igb_poll;
273 #endif /* POLLING */
274
275 /*********************************************************************
276 * FreeBSD Device Interface Entry Points
277 *********************************************************************/
278
279 static device_method_t igb_methods[] = {
280 /* Device interface */
281 DEVMETHOD(device_probe, igb_probe),
282 DEVMETHOD(device_attach, igb_attach),
283 DEVMETHOD(device_detach, igb_detach),
284 DEVMETHOD(device_shutdown, igb_shutdown),
285 DEVMETHOD(device_suspend, igb_suspend),
286 DEVMETHOD(device_resume, igb_resume),
287 {0, 0}
288 };
289
290 static driver_t igb_driver = {
291 "igb", igb_methods, sizeof(struct adapter),
292 };
293
294 static devclass_t igb_devclass;
295 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
296 MODULE_DEPEND(igb, pci, 1, 1, 1);
297 MODULE_DEPEND(igb, ether, 1, 1, 1);
298
299 /*********************************************************************
300 * Tunable default values.
301 *********************************************************************/
302
303 SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
304
305 /* Descriptor defaults */
306 static int igb_rxd = IGB_DEFAULT_RXD;
307 static int igb_txd = IGB_DEFAULT_TXD;
308 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
309 TUNABLE_INT("hw.igb.txd", &igb_txd);
310 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
311 "Number of receive descriptors per queue");
312 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
313 "Number of transmit descriptors per queue");
314
315 /*
316 ** AIM: Adaptive Interrupt Moderation
317 ** which means that the interrupt rate
318 ** is varied over time based on the
319 ** traffic for that interrupt vector
320 */
321 static int igb_enable_aim = TRUE;
322 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
323 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0,
324 "Enable adaptive interrupt moderation");
325
326 /*
327 * MSIX should be the default for best performance,
328 * but this allows it to be forced off for testing.
329 */
330 static int igb_enable_msix = 1;
331 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
332 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
333 "Enable MSI-X interrupts");
334
335 /*
336 ** Tuneable Interrupt rate
337 */
338 static int igb_max_interrupt_rate = 8000;
339 TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate);
340 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
341 &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
342
343 /*
344 ** Header split causes the packet header to
345 ** be dma'd to a seperate mbuf from the payload.
346 ** this can have memory alignment benefits. But
347 ** another plus is that small packets often fit
348 ** into the header and thus use no cluster. Its
349 ** a very workload dependent type feature.
350 */
351 static bool igb_header_split = FALSE;
352 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
353 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
354 "Enable receive mbuf header split");
355
356 /*
357 ** This will autoconfigure based on
358 ** the number of CPUs if left at 0.
359 */
360 static int igb_num_queues = 0;
361 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
362 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
363 "Number of queues to configure, 0 indicates autoconfigure");
364
365 /* How many packets rxeof tries to clean at a time */
366 static int igb_rx_process_limit = 100;
367 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
368 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
369 &igb_rx_process_limit, 0,
370 "Maximum number of received packets to process at a time, -1 means unlimited");
371
372 /*********************************************************************
373 * Device identification routine
374 *
375 * igb_probe determines if the driver should be loaded on
376 * adapter based on PCI vendor/device id of the adapter.
377 *
378 * return BUS_PROBE_DEFAULT on success, positive on failure
379 *********************************************************************/
380
381 static int
382 igb_probe(device_t dev)
383 {
384 char adapter_name[60];
385 uint16_t pci_vendor_id = 0;
386 uint16_t pci_device_id = 0;
387 uint16_t pci_subvendor_id = 0;
388 uint16_t pci_subdevice_id = 0;
389 igb_vendor_info_t *ent;
390
391 INIT_DEBUGOUT("igb_probe: begin");
392
393 pci_vendor_id = pci_get_vendor(dev);
394 if (pci_vendor_id != IGB_VENDOR_ID)
395 return (ENXIO);
396
397 pci_device_id = pci_get_device(dev);
398 pci_subvendor_id = pci_get_subvendor(dev);
399 pci_subdevice_id = pci_get_subdevice(dev);
400
401 ent = igb_vendor_info_array;
402 while (ent->vendor_id != 0) {
403 if ((pci_vendor_id == ent->vendor_id) &&
404 (pci_device_id == ent->device_id) &&
405
406 ((pci_subvendor_id == ent->subvendor_id) ||
407 (ent->subvendor_id == PCI_ANY_ID)) &&
408
409 ((pci_subdevice_id == ent->subdevice_id) ||
410 (ent->subdevice_id == PCI_ANY_ID))) {
411 sprintf(adapter_name, "%s %s",
412 igb_strings[ent->index],
413 igb_driver_version);
414 device_set_desc_copy(dev, adapter_name);
415 return (BUS_PROBE_DEFAULT);
416 }
417 ent++;
418 }
419
420 return (ENXIO);
421 }
422
423 /*********************************************************************
424 * Device initialization routine
425 *
426 * The attach entry point is called when the driver is being loaded.
427 * This routine identifies the type of hardware, allocates all resources
428 * and initializes the hardware.
429 *
430 * return 0 on success, positive on failure
431 *********************************************************************/
432
433 static int
434 igb_attach(device_t dev)
435 {
436 struct adapter *adapter;
437 int error = 0;
438 u16 eeprom_data;
439
440 INIT_DEBUGOUT("igb_attach: begin");
441
442 if (resource_disabled("igb", device_get_unit(dev))) {
443 device_printf(dev, "Disabled by device hint\n");
444 return (ENXIO);
445 }
446
447 adapter = device_get_softc(dev);
448 adapter->dev = adapter->osdep.dev = dev;
449 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
450
451 /* SYSCTL stuff */
452 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
453 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
454 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
455 igb_sysctl_nvm_info, "I", "NVM Information");
456
457 igb_set_sysctl_value(adapter, "enable_aim",
458 "Interrupt Moderation", &adapter->enable_aim,
459 igb_enable_aim);
460
461 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
462 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
463 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
464 adapter, 0, igb_set_flowcntl, "I", "Flow Control");
465
466 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
467
468 /* Determine hardware and mac info */
469 igb_identify_hardware(adapter);
470
471 /* Setup PCI resources */
472 if (igb_allocate_pci_resources(adapter)) {
473 device_printf(dev, "Allocation of PCI resources failed\n");
474 error = ENXIO;
475 goto err_pci;
476 }
477
478 /* Do Shared Code initialization */
479 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
480 device_printf(dev, "Setup of Shared code failed\n");
481 error = ENXIO;
482 goto err_pci;
483 }
484
485 e1000_get_bus_info(&adapter->hw);
486
487 /* Sysctl for limiting the amount of work done in the taskqueue */
488 igb_set_sysctl_value(adapter, "rx_processing_limit",
489 "max number of rx packets to process",
490 &adapter->rx_process_limit, igb_rx_process_limit);
491
492 /*
493 * Validate number of transmit and receive descriptors. It
494 * must not exceed hardware maximum, and must be multiple
495 * of E1000_DBA_ALIGN.
496 */
497 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
498 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
499 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
500 IGB_DEFAULT_TXD, igb_txd);
501 adapter->num_tx_desc = IGB_DEFAULT_TXD;
502 } else
503 adapter->num_tx_desc = igb_txd;
504 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
505 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
506 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
507 IGB_DEFAULT_RXD, igb_rxd);
508 adapter->num_rx_desc = IGB_DEFAULT_RXD;
509 } else
510 adapter->num_rx_desc = igb_rxd;
511
512 adapter->hw.mac.autoneg = DO_AUTO_NEG;
513 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
514 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
515
516 /* Copper options */
517 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
518 adapter->hw.phy.mdix = AUTO_ALL_MODES;
519 adapter->hw.phy.disable_polarity_correction = FALSE;
520 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
521 }
522
523 /*
524 * Set the frame limits assuming
525 * standard ethernet sized frames.
526 */
527 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
528 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
529
530 /*
531 ** Allocate and Setup Queues
532 */
533 if (igb_allocate_queues(adapter)) {
534 error = ENOMEM;
535 goto err_pci;
536 }
537
538 /* Allocate the appropriate stats memory */
539 if (adapter->vf_ifp) {
540 adapter->stats =
541 (struct e1000_vf_stats *)malloc(sizeof \
542 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
543 igb_vf_init_stats(adapter);
544 } else
545 adapter->stats =
546 (struct e1000_hw_stats *)malloc(sizeof \
547 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
548 if (adapter->stats == NULL) {
549 device_printf(dev, "Can not allocate stats memory\n");
550 error = ENOMEM;
551 goto err_late;
552 }
553
554 /* Allocate multicast array memory. */
555 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
556 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
557 if (adapter->mta == NULL) {
558 device_printf(dev, "Can not allocate multicast setup array\n");
559 error = ENOMEM;
560 goto err_late;
561 }
562
563 /* Some adapter-specific advanced features */
564 if (adapter->hw.mac.type >= e1000_i350) {
565 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
566 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
567 OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
568 adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
569 igb_set_sysctl_value(adapter, "eee_disabled",
570 "enable Energy Efficient Ethernet",
571 &adapter->hw.dev_spec._82575.eee_disable,
572 TRUE);
573 e1000_set_eee_i350(&adapter->hw);
574 }
575
576 /*
577 ** Start from a known state, this is
578 ** important in reading the nvm and
579 ** mac from that.
580 */
581 e1000_reset_hw(&adapter->hw);
582
583 /* Make sure we have a good EEPROM before we read from it */
584 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
585 /*
586 ** Some PCI-E parts fail the first check due to
587 ** the link being in sleep state, call it again,
588 ** if it fails a second time its a real issue.
589 */
590 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
591 device_printf(dev,
592 "The EEPROM Checksum Is Not Valid\n");
593 error = EIO;
594 goto err_late;
595 }
596 }
597
598 /*
599 ** Copy the permanent MAC address out of the EEPROM
600 */
601 if (e1000_read_mac_addr(&adapter->hw) < 0) {
602 device_printf(dev, "EEPROM read error while reading MAC"
603 " address\n");
604 error = EIO;
605 goto err_late;
606 }
607 /* Check its sanity */
608 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
609 device_printf(dev, "Invalid MAC address\n");
610 error = EIO;
611 goto err_late;
612 }
613
614 /*
615 ** Configure Interrupts
616 */
617 if ((adapter->msix > 1) && (igb_enable_msix))
618 error = igb_allocate_msix(adapter);
619 else /* MSI or Legacy */
620 error = igb_allocate_legacy(adapter);
621 if (error)
622 goto err_late;
623
624 /* Setup OS specific network interface */
625 if (igb_setup_interface(dev, adapter) != 0)
626 goto err_late;
627
628 /* Now get a good starting state */
629 igb_reset(adapter);
630
631 /* Initialize statistics */
632 igb_update_stats_counters(adapter);
633
634 adapter->hw.mac.get_link_status = 1;
635 igb_update_link_status(adapter);
636
637 /* Indicate SOL/IDER usage */
638 if (e1000_check_reset_block(&adapter->hw))
639 device_printf(dev,
640 "PHY reset is blocked due to SOL/IDER session.\n");
641
642 /* Determine if we have to control management hardware */
643 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
644
645 /*
646 * Setup Wake-on-Lan
647 */
648 /* APME bit in EEPROM is mapped to WUC.APME */
649 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
650 if (eeprom_data)
651 adapter->wol = E1000_WUFC_MAG;
652
653 /* Register for VLAN events */
654 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
655 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
656 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
657 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
658
659 igb_add_hw_stats(adapter);
660
661 /* Tell the stack that the interface is not active */
662 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
663
664 adapter->led_dev = led_create(igb_led_func, adapter,
665 device_get_nameunit(dev));
666
667 INIT_DEBUGOUT("igb_attach: end");
668
669 return (0);
670
671 err_late:
672 igb_detach(dev);
673 igb_free_transmit_structures(adapter);
674 igb_free_receive_structures(adapter);
675 igb_release_hw_control(adapter);
676 if (adapter->ifp != NULL)
677 if_free(adapter->ifp);
678 err_pci:
679 igb_free_pci_resources(adapter);
680 free(adapter->mta, M_DEVBUF);
681 IGB_CORE_LOCK_DESTROY(adapter);
682
683 return (error);
684 }
685
686 /*********************************************************************
687 * Device removal routine
688 *
689 * The detach entry point is called when the driver is being removed.
690 * This routine stops the adapter and deallocates all the resources
691 * that were allocated for driver operation.
692 *
693 * return 0 on success, positive on failure
694 *********************************************************************/
695
696 static int
697 igb_detach(device_t dev)
698 {
699 struct adapter *adapter = device_get_softc(dev);
700 struct ifnet *ifp = adapter->ifp;
701
702 INIT_DEBUGOUT("igb_detach: begin");
703
704 /* Make sure VLANS are not using driver */
705 if (adapter->ifp->if_vlantrunk != NULL) {
706 device_printf(dev,"Vlan in use, detach first\n");
707 return (EBUSY);
708 }
709
710 ether_ifdetach(adapter->ifp);
711
712 if (adapter->led_dev != NULL)
713 led_destroy(adapter->led_dev);
714
715 #ifdef DEVICE_POLLING
716 if (ifp->if_capenable & IFCAP_POLLING)
717 ether_poll_deregister(ifp);
718 #endif
719
720 IGB_CORE_LOCK(adapter);
721 adapter->in_detach = 1;
722 igb_stop(adapter);
723 IGB_CORE_UNLOCK(adapter);
724
725 e1000_phy_hw_reset(&adapter->hw);
726
727 /* Give control back to firmware */
728 igb_release_manageability(adapter);
729 igb_release_hw_control(adapter);
730
731 if (adapter->wol) {
732 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
733 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
734 igb_enable_wakeup(dev);
735 }
736
737 /* Unregister VLAN events */
738 if (adapter->vlan_attach != NULL)
739 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
740 if (adapter->vlan_detach != NULL)
741 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
742
743 callout_drain(&adapter->timer);
744
745 igb_free_pci_resources(adapter);
746 bus_generic_detach(dev);
747 if_free(ifp);
748
749 igb_free_transmit_structures(adapter);
750 igb_free_receive_structures(adapter);
751 if (adapter->mta != NULL)
752 free(adapter->mta, M_DEVBUF);
753
754 IGB_CORE_LOCK_DESTROY(adapter);
755
756 return (0);
757 }
758
759 /*********************************************************************
760 *
761 * Shutdown entry point
762 *
763 **********************************************************************/
764
765 static int
766 igb_shutdown(device_t dev)
767 {
768 return igb_suspend(dev);
769 }
770
771 /*
772 * Suspend/resume device methods.
773 */
774 static int
775 igb_suspend(device_t dev)
776 {
777 struct adapter *adapter = device_get_softc(dev);
778
779 IGB_CORE_LOCK(adapter);
780
781 igb_stop(adapter);
782
783 igb_release_manageability(adapter);
784 igb_release_hw_control(adapter);
785
786 if (adapter->wol) {
787 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
788 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
789 igb_enable_wakeup(dev);
790 }
791
792 IGB_CORE_UNLOCK(adapter);
793
794 return bus_generic_suspend(dev);
795 }
796
797 static int
798 igb_resume(device_t dev)
799 {
800 struct adapter *adapter = device_get_softc(dev);
801 struct ifnet *ifp = adapter->ifp;
802 #if __FreeBSD_version >= 800000
803 struct tx_ring *txr = adapter->tx_rings;
804 #endif
805
806 IGB_CORE_LOCK(adapter);
807 igb_init_locked(adapter);
808 igb_init_manageability(adapter);
809
810 if ((ifp->if_flags & IFF_UP) &&
811 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
812 #if __FreeBSD_version < 800000
813 igb_start(ifp);
814 #else
815 for (int i = 0; i < adapter->num_queues; i++, txr++) {
816 IGB_TX_LOCK(txr);
817 if (!drbr_empty(ifp, txr->br))
818 igb_mq_start_locked(ifp, txr, NULL);
819 IGB_TX_UNLOCK(txr);
820 }
821 #endif
822 }
823
824 IGB_CORE_UNLOCK(adapter);
825
826 return bus_generic_resume(dev);
827 }
828
829
830 #if __FreeBSD_version < 800000
831 /*********************************************************************
832 * Transmit entry point
833 *
834 * igb_start is called by the stack to initiate a transmit.
835 * The driver will remain in this routine as long as there are
836 * packets to transmit and transmit resources are available.
837 * In case resources are not available stack is notified and
838 * the packet is requeued.
839 **********************************************************************/
840
841 static void
842 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
843 {
844 struct adapter *adapter = ifp->if_softc;
845 struct mbuf *m_head;
846
847 IGB_TX_LOCK_ASSERT(txr);
848
849 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
850 IFF_DRV_RUNNING)
851 return;
852 if (!adapter->link_active)
853 return;
854
855 /* Call cleanup if number of TX descriptors low */
856 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
857 igb_txeof(txr);
858
859 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
860 if (txr->tx_avail <= IGB_MAX_SCATTER) {
861 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
862 break;
863 }
864 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
865 if (m_head == NULL)
866 break;
867 /*
868 * Encapsulation can modify our pointer, and or make it
869 * NULL on failure. In that event, we can't requeue.
870 */
871 if (igb_xmit(txr, &m_head)) {
872 if (m_head == NULL)
873 break;
874 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
875 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
876 break;
877 }
878
879 /* Send a copy of the frame to the BPF listener */
880 ETHER_BPF_MTAP(ifp, m_head);
881
882 /* Set watchdog on */
883 txr->watchdog_time = ticks;
884 txr->queue_status = IGB_QUEUE_WORKING;
885 }
886 }
887
888 /*
889 * Legacy TX driver routine, called from the
890 * stack, always uses tx[0], and spins for it.
891 * Should not be used with multiqueue tx
892 */
893 static void
894 igb_start(struct ifnet *ifp)
895 {
896 struct adapter *adapter = ifp->if_softc;
897 struct tx_ring *txr = adapter->tx_rings;
898
899 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
900 IGB_TX_LOCK(txr);
901 igb_start_locked(txr, ifp);
902 IGB_TX_UNLOCK(txr);
903 }
904 return;
905 }
906
907 #else /* __FreeBSD_version >= 800000 */
908 /*
909 ** Multiqueue Transmit driver
910 **
911 */
912 static int
913 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
914 {
915 struct adapter *adapter = ifp->if_softc;
916 struct igb_queue *que;
917 struct tx_ring *txr;
918 int i = 0, err = 0;
919
920 /* Which queue to use */
921 if ((m->m_flags & M_FLOWID) != 0)
922 i = m->m_pkthdr.flowid % adapter->num_queues;
923
924 txr = &adapter->tx_rings[i];
925 que = &adapter->queues[i];
926
927 if (IGB_TX_TRYLOCK(txr)) {
928 err = igb_mq_start_locked(ifp, txr, m);
929 IGB_TX_UNLOCK(txr);
930 } else {
931 err = drbr_enqueue(ifp, txr->br, m);
932 taskqueue_enqueue(que->tq, &txr->txq_task);
933 }
934
935 return (err);
936 }
937
938 static int
939 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
940 {
941 struct adapter *adapter = txr->adapter;
942 struct mbuf *next;
943 int err = 0, enq;
944
945 IGB_TX_LOCK_ASSERT(txr);
946
947 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
948 IFF_DRV_RUNNING || adapter->link_active == 0) {
949 if (m != NULL)
950 err = drbr_enqueue(ifp, txr->br, m);
951 return (err);
952 }
953
954 enq = 0;
955 if (m == NULL) {
956 next = drbr_dequeue(ifp, txr->br);
957 } else if (drbr_needs_enqueue(ifp, txr->br)) {
958 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
959 return (err);
960 next = drbr_dequeue(ifp, txr->br);
961 } else
962 next = m;
963
964 /* Process the queue */
965 while (next != NULL) {
966 if ((err = igb_xmit(txr, &next)) != 0) {
967 if (next != NULL)
968 err = drbr_enqueue(ifp, txr->br, next);
969 break;
970 }
971 enq++;
972 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
973 ETHER_BPF_MTAP(ifp, next);
974 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
975 break;
976 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
977 igb_txeof(txr);
978 if (txr->tx_avail <= IGB_MAX_SCATTER) {
979 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
980 break;
981 }
982 next = drbr_dequeue(ifp, txr->br);
983 }
984 if (enq > 0) {
985 /* Set the watchdog */
986 txr->queue_status = IGB_QUEUE_WORKING;
987 txr->watchdog_time = ticks;
988 }
989 return (err);
990 }
991
992 /*
993 * Called from a taskqueue to drain queued transmit packets.
994 */
995 static void
996 igb_deferred_mq_start(void *arg, int pending)
997 {
998 struct tx_ring *txr = arg;
999 struct adapter *adapter = txr->adapter;
1000 struct ifnet *ifp = adapter->ifp;
1001
1002 IGB_TX_LOCK(txr);
1003 if (!drbr_empty(ifp, txr->br))
1004 igb_mq_start_locked(ifp, txr, NULL);
1005 IGB_TX_UNLOCK(txr);
1006 }
1007
1008 /*
1009 ** Flush all ring buffers
1010 */
1011 static void
1012 igb_qflush(struct ifnet *ifp)
1013 {
1014 struct adapter *adapter = ifp->if_softc;
1015 struct tx_ring *txr = adapter->tx_rings;
1016 struct mbuf *m;
1017
1018 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1019 IGB_TX_LOCK(txr);
1020 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1021 m_freem(m);
1022 IGB_TX_UNLOCK(txr);
1023 }
1024 if_qflush(ifp);
1025 }
1026 #endif /* __FreeBSD_version < 800000 */
1027
1028 /*********************************************************************
1029 * Ioctl entry point
1030 *
1031 * igb_ioctl is called when the user wants to configure the
1032 * interface.
1033 *
1034 * return 0 on success, positive on failure
1035 **********************************************************************/
1036
1037 static int
1038 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1039 {
1040 struct adapter *adapter = ifp->if_softc;
1041 struct ifreq *ifr = (struct ifreq *)data;
1042 #if defined(INET) || defined(INET6)
1043 struct ifaddr *ifa = (struct ifaddr *)data;
1044 #endif
1045 bool avoid_reset = FALSE;
1046 int error = 0;
1047
1048 if (adapter->in_detach)
1049 return (error);
1050
1051 switch (command) {
1052 case SIOCSIFADDR:
1053 #ifdef INET
1054 if (ifa->ifa_addr->sa_family == AF_INET)
1055 avoid_reset = TRUE;
1056 #endif
1057 #ifdef INET6
1058 if (ifa->ifa_addr->sa_family == AF_INET6)
1059 avoid_reset = TRUE;
1060 #endif
1061 /*
1062 ** Calling init results in link renegotiation,
1063 ** so we avoid doing it when possible.
1064 */
1065 if (avoid_reset) {
1066 ifp->if_flags |= IFF_UP;
1067 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1068 igb_init(adapter);
1069 #ifdef INET
1070 if (!(ifp->if_flags & IFF_NOARP))
1071 arp_ifinit(ifp, ifa);
1072 #endif
1073 } else
1074 error = ether_ioctl(ifp, command, data);
1075 break;
1076 case SIOCSIFMTU:
1077 {
1078 int max_frame_size;
1079
1080 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1081
1082 IGB_CORE_LOCK(adapter);
1083 max_frame_size = 9234;
1084 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1085 ETHER_CRC_LEN) {
1086 IGB_CORE_UNLOCK(adapter);
1087 error = EINVAL;
1088 break;
1089 }
1090
1091 ifp->if_mtu = ifr->ifr_mtu;
1092 adapter->max_frame_size =
1093 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1094 igb_init_locked(adapter);
1095 IGB_CORE_UNLOCK(adapter);
1096 break;
1097 }
1098 case SIOCSIFFLAGS:
1099 IOCTL_DEBUGOUT("ioctl rcv'd:\
1100 SIOCSIFFLAGS (Set Interface Flags)");
1101 IGB_CORE_LOCK(adapter);
1102 if (ifp->if_flags & IFF_UP) {
1103 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1104 if ((ifp->if_flags ^ adapter->if_flags) &
1105 (IFF_PROMISC | IFF_ALLMULTI)) {
1106 igb_disable_promisc(adapter);
1107 igb_set_promisc(adapter);
1108 }
1109 } else
1110 igb_init_locked(adapter);
1111 } else
1112 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1113 igb_stop(adapter);
1114 adapter->if_flags = ifp->if_flags;
1115 IGB_CORE_UNLOCK(adapter);
1116 break;
1117 case SIOCADDMULTI:
1118 case SIOCDELMULTI:
1119 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1120 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1121 IGB_CORE_LOCK(adapter);
1122 igb_disable_intr(adapter);
1123 igb_set_multi(adapter);
1124 #ifdef DEVICE_POLLING
1125 if (!(ifp->if_capenable & IFCAP_POLLING))
1126 #endif
1127 igb_enable_intr(adapter);
1128 IGB_CORE_UNLOCK(adapter);
1129 }
1130 break;
1131 case SIOCSIFMEDIA:
1132 /*
1133 ** As the speed/duplex settings are being
1134 ** changed, we need toreset the PHY.
1135 */
1136 adapter->hw.phy.reset_disable = FALSE;
1137 /* Check SOL/IDER usage */
1138 IGB_CORE_LOCK(adapter);
1139 if (e1000_check_reset_block(&adapter->hw)) {
1140 IGB_CORE_UNLOCK(adapter);
1141 device_printf(adapter->dev, "Media change is"
1142 " blocked due to SOL/IDER session.\n");
1143 break;
1144 }
1145 IGB_CORE_UNLOCK(adapter);
1146 case SIOCGIFMEDIA:
1147 IOCTL_DEBUGOUT("ioctl rcv'd: \
1148 SIOCxIFMEDIA (Get/Set Interface Media)");
1149 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1150 break;
1151 case SIOCSIFCAP:
1152 {
1153 int mask, reinit;
1154
1155 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1156 reinit = 0;
1157 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1158 #ifdef DEVICE_POLLING
1159 if (mask & IFCAP_POLLING) {
1160 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1161 error = ether_poll_register(igb_poll, ifp);
1162 if (error)
1163 return (error);
1164 IGB_CORE_LOCK(adapter);
1165 igb_disable_intr(adapter);
1166 ifp->if_capenable |= IFCAP_POLLING;
1167 IGB_CORE_UNLOCK(adapter);
1168 } else {
1169 error = ether_poll_deregister(ifp);
1170 /* Enable interrupt even in error case */
1171 IGB_CORE_LOCK(adapter);
1172 igb_enable_intr(adapter);
1173 ifp->if_capenable &= ~IFCAP_POLLING;
1174 IGB_CORE_UNLOCK(adapter);
1175 }
1176 }
1177 #endif
1178 if (mask & IFCAP_HWCSUM) {
1179 ifp->if_capenable ^= IFCAP_HWCSUM;
1180 reinit = 1;
1181 }
1182 if (mask & IFCAP_TSO4) {
1183 ifp->if_capenable ^= IFCAP_TSO4;
1184 reinit = 1;
1185 }
1186 if (mask & IFCAP_VLAN_HWTAGGING) {
1187 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1188 reinit = 1;
1189 }
1190 if (mask & IFCAP_VLAN_HWFILTER) {
1191 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1192 reinit = 1;
1193 }
1194 if (mask & IFCAP_VLAN_HWTSO) {
1195 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1196 reinit = 1;
1197 }
1198 if (mask & IFCAP_LRO) {
1199 ifp->if_capenable ^= IFCAP_LRO;
1200 reinit = 1;
1201 }
1202 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1203 igb_init(adapter);
1204 VLAN_CAPABILITIES(ifp);
1205 break;
1206 }
1207
1208 default:
1209 error = ether_ioctl(ifp, command, data);
1210 break;
1211 }
1212
1213 return (error);
1214 }
1215
1216
1217 /*********************************************************************
1218 * Init entry point
1219 *
1220 * This routine is used in two ways. It is used by the stack as
1221 * init entry point in network interface structure. It is also used
1222 * by the driver as a hw/sw initialization routine to get to a
1223 * consistent state.
1224 *
1225 * return 0 on success, positive on failure
1226 **********************************************************************/
1227
1228 static void
1229 igb_init_locked(struct adapter *adapter)
1230 {
1231 struct ifnet *ifp = adapter->ifp;
1232 device_t dev = adapter->dev;
1233
1234 INIT_DEBUGOUT("igb_init: begin");
1235
1236 IGB_CORE_LOCK_ASSERT(adapter);
1237
1238 igb_disable_intr(adapter);
1239 callout_stop(&adapter->timer);
1240
1241 /* Get the latest mac address, User can use a LAA */
1242 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1243 ETHER_ADDR_LEN);
1244
1245 /* Put the address into the Receive Address Array */
1246 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1247
1248 igb_reset(adapter);
1249 igb_update_link_status(adapter);
1250
1251 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1252
1253 /* Set hardware offload abilities */
1254 ifp->if_hwassist = 0;
1255 if (ifp->if_capenable & IFCAP_TXCSUM) {
1256 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1257 #if __FreeBSD_version >= 800000
1258 if (adapter->hw.mac.type == e1000_82576)
1259 ifp->if_hwassist |= CSUM_SCTP;
1260 #endif
1261 }
1262
1263 if (ifp->if_capenable & IFCAP_TSO4)
1264 ifp->if_hwassist |= CSUM_TSO;
1265
1266 /* Configure for OS presence */
1267 igb_init_manageability(adapter);
1268
1269 /* Prepare transmit descriptors and buffers */
1270 igb_setup_transmit_structures(adapter);
1271 igb_initialize_transmit_units(adapter);
1272
1273 /* Setup Multicast table */
1274 igb_set_multi(adapter);
1275
1276 /*
1277 ** Figure out the desired mbuf pool
1278 ** for doing jumbo/packetsplit
1279 */
1280 if (adapter->max_frame_size <= 2048)
1281 adapter->rx_mbuf_sz = MCLBYTES;
1282 else if (adapter->max_frame_size <= 4096)
1283 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1284 else
1285 adapter->rx_mbuf_sz = MJUM9BYTES;
1286
1287 /* Prepare receive descriptors and buffers */
1288 if (igb_setup_receive_structures(adapter)) {
1289 device_printf(dev, "Could not setup receive structures\n");
1290 return;
1291 }
1292 igb_initialize_receive_units(adapter);
1293
1294 /* Enable VLAN support */
1295 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1296 igb_setup_vlan_hw_support(adapter);
1297
1298 /* Don't lose promiscuous settings */
1299 igb_set_promisc(adapter);
1300
1301 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1302 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1303
1304 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1305 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1306
1307 if (adapter->msix > 1) /* Set up queue routing */
1308 igb_configure_queues(adapter);
1309
1310 /* this clears any pending interrupts */
1311 E1000_READ_REG(&adapter->hw, E1000_ICR);
1312 #ifdef DEVICE_POLLING
1313 /*
1314 * Only enable interrupts if we are not polling, make sure
1315 * they are off otherwise.
1316 */
1317 if (ifp->if_capenable & IFCAP_POLLING)
1318 igb_disable_intr(adapter);
1319 else
1320 #endif /* DEVICE_POLLING */
1321 {
1322 igb_enable_intr(adapter);
1323 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1324 }
1325
1326 /* Set Energy Efficient Ethernet */
1327 e1000_set_eee_i350(&adapter->hw);
1328
1329 /* Don't reset the phy next time init gets called */
1330 adapter->hw.phy.reset_disable = TRUE;
1331 }
1332
1333 static void
1334 igb_init(void *arg)
1335 {
1336 struct adapter *adapter = arg;
1337
1338 IGB_CORE_LOCK(adapter);
1339 igb_init_locked(adapter);
1340 IGB_CORE_UNLOCK(adapter);
1341 }
1342
1343
1344 static void
1345 igb_handle_que(void *context, int pending)
1346 {
1347 struct igb_queue *que = context;
1348 struct adapter *adapter = que->adapter;
1349 struct tx_ring *txr = que->txr;
1350 struct ifnet *ifp = adapter->ifp;
1351
1352 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1353 bool more;
1354
1355 more = igb_rxeof(que, -1, NULL);
1356
1357 IGB_TX_LOCK(txr);
1358 if (igb_txeof(txr))
1359 more = TRUE;
1360 #if __FreeBSD_version >= 800000
1361 if (!drbr_empty(ifp, txr->br))
1362 igb_mq_start_locked(ifp, txr, NULL);
1363 #else
1364 igb_start_locked(txr, ifp);
1365 #endif
1366 IGB_TX_UNLOCK(txr);
1367 if (more || (ifp->if_drv_flags & IFF_DRV_OACTIVE)) {
1368 taskqueue_enqueue(que->tq, &que->que_task);
1369 return;
1370 }
1371 }
1372
1373 #ifdef DEVICE_POLLING
1374 if (ifp->if_capenable & IFCAP_POLLING)
1375 return;
1376 #endif
1377 /* Reenable this interrupt */
1378 if (que->eims)
1379 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1380 else
1381 igb_enable_intr(adapter);
1382 }
1383
1384 /* Deal with link in a sleepable context */
1385 static void
1386 igb_handle_link(void *context, int pending)
1387 {
1388 struct adapter *adapter = context;
1389
1390 adapter->hw.mac.get_link_status = 1;
1391 igb_update_link_status(adapter);
1392 }
1393
1394 /*********************************************************************
1395 *
1396 * MSI/Legacy Deferred
1397 * Interrupt Service routine
1398 *
1399 *********************************************************************/
1400 static int
1401 igb_irq_fast(void *arg)
1402 {
1403 struct adapter *adapter = arg;
1404 struct igb_queue *que = adapter->queues;
1405 u32 reg_icr;
1406
1407
1408 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1409
1410 /* Hot eject? */
1411 if (reg_icr == 0xffffffff)
1412 return FILTER_STRAY;
1413
1414 /* Definitely not our interrupt. */
1415 if (reg_icr == 0x0)
1416 return FILTER_STRAY;
1417
1418 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1419 return FILTER_STRAY;
1420
1421 /*
1422 * Mask interrupts until the taskqueue is finished running. This is
1423 * cheap, just assume that it is needed. This also works around the
1424 * MSI message reordering errata on certain systems.
1425 */
1426 igb_disable_intr(adapter);
1427 taskqueue_enqueue(que->tq, &que->que_task);
1428
1429 /* Link status change */
1430 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1431 taskqueue_enqueue(que->tq, &adapter->link_task);
1432
1433 if (reg_icr & E1000_ICR_RXO)
1434 adapter->rx_overruns++;
1435 return FILTER_HANDLED;
1436 }
1437
1438 #ifdef DEVICE_POLLING
1439 /*********************************************************************
1440 *
1441 * Legacy polling routine : if using this code you MUST be sure that
1442 * multiqueue is not defined, ie, set igb_num_queues to 1.
1443 *
1444 *********************************************************************/
1445 #if __FreeBSD_version >= 800000
1446 #define POLL_RETURN_COUNT(a) (a)
1447 static int
1448 #else
1449 #define POLL_RETURN_COUNT(a)
1450 static void
1451 #endif
1452 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1453 {
1454 struct adapter *adapter = ifp->if_softc;
1455 struct igb_queue *que = adapter->queues;
1456 struct tx_ring *txr = adapter->tx_rings;
1457 u32 reg_icr, rx_done = 0;
1458 u32 loop = IGB_MAX_LOOP;
1459 bool more;
1460
1461 IGB_CORE_LOCK(adapter);
1462 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1463 IGB_CORE_UNLOCK(adapter);
1464 return POLL_RETURN_COUNT(rx_done);
1465 }
1466
1467 if (cmd == POLL_AND_CHECK_STATUS) {
1468 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1469 /* Link status change */
1470 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1471 igb_handle_link(adapter, 0);
1472
1473 if (reg_icr & E1000_ICR_RXO)
1474 adapter->rx_overruns++;
1475 }
1476 IGB_CORE_UNLOCK(adapter);
1477
1478 igb_rxeof(que, count, &rx_done);
1479
1480 IGB_TX_LOCK(txr);
1481 do {
1482 more = igb_txeof(txr);
1483 } while (loop-- && more);
1484 #if __FreeBSD_version >= 800000
1485 if (!drbr_empty(ifp, txr->br))
1486 igb_mq_start_locked(ifp, txr, NULL);
1487 #else
1488 igb_start_locked(txr, ifp);
1489 #endif
1490 IGB_TX_UNLOCK(txr);
1491 return POLL_RETURN_COUNT(rx_done);
1492 }
1493 #endif /* DEVICE_POLLING */
1494
1495 /*********************************************************************
1496 *
1497 * MSIX TX Interrupt Service routine
1498 *
1499 **********************************************************************/
1500 static void
1501 igb_msix_que(void *arg)
1502 {
1503 struct igb_queue *que = arg;
1504 struct adapter *adapter = que->adapter;
1505 struct tx_ring *txr = que->txr;
1506 struct rx_ring *rxr = que->rxr;
1507 u32 newitr = 0;
1508 bool more_tx, more_rx;
1509
1510 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1511 ++que->irqs;
1512
1513 IGB_TX_LOCK(txr);
1514 more_tx = igb_txeof(txr);
1515 IGB_TX_UNLOCK(txr);
1516
1517 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
1518
1519 if (adapter->enable_aim == FALSE)
1520 goto no_calc;
1521 /*
1522 ** Do Adaptive Interrupt Moderation:
1523 ** - Write out last calculated setting
1524 ** - Calculate based on average size over
1525 ** the last interval.
1526 */
1527 if (que->eitr_setting)
1528 E1000_WRITE_REG(&adapter->hw,
1529 E1000_EITR(que->msix), que->eitr_setting);
1530
1531 que->eitr_setting = 0;
1532
1533 /* Idle, do nothing */
1534 if ((txr->bytes == 0) && (rxr->bytes == 0))
1535 goto no_calc;
1536
1537 /* Used half Default if sub-gig */
1538 if (adapter->link_speed != 1000)
1539 newitr = IGB_DEFAULT_ITR / 2;
1540 else {
1541 if ((txr->bytes) && (txr->packets))
1542 newitr = txr->bytes/txr->packets;
1543 if ((rxr->bytes) && (rxr->packets))
1544 newitr = max(newitr,
1545 (rxr->bytes / rxr->packets));
1546 newitr += 24; /* account for hardware frame, crc */
1547 /* set an upper boundary */
1548 newitr = min(newitr, 3000);
1549 /* Be nice to the mid range */
1550 if ((newitr > 300) && (newitr < 1200))
1551 newitr = (newitr / 3);
1552 else
1553 newitr = (newitr / 2);
1554 }
1555 newitr &= 0x7FFC; /* Mask invalid bits */
1556 if (adapter->hw.mac.type == e1000_82575)
1557 newitr |= newitr << 16;
1558 else
1559 newitr |= E1000_EITR_CNT_IGNR;
1560
1561 /* save for next interrupt */
1562 que->eitr_setting = newitr;
1563
1564 /* Reset state */
1565 txr->bytes = 0;
1566 txr->packets = 0;
1567 rxr->bytes = 0;
1568 rxr->packets = 0;
1569
1570 no_calc:
1571 /* Schedule a clean task if needed*/
1572 if (more_tx || more_rx ||
1573 (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE))
1574 taskqueue_enqueue(que->tq, &que->que_task);
1575 else
1576 /* Reenable this interrupt */
1577 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1578 return;
1579 }
1580
1581
1582 /*********************************************************************
1583 *
1584 * MSIX Link Interrupt Service routine
1585 *
1586 **********************************************************************/
1587
1588 static void
1589 igb_msix_link(void *arg)
1590 {
1591 struct adapter *adapter = arg;
1592 u32 icr;
1593
1594 ++adapter->link_irq;
1595 icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1596 if (!(icr & E1000_ICR_LSC))
1597 goto spurious;
1598 igb_handle_link(adapter, 0);
1599
1600 spurious:
1601 /* Rearm */
1602 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1603 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1604 return;
1605 }
1606
1607
1608 /*********************************************************************
1609 *
1610 * Media Ioctl callback
1611 *
1612 * This routine is called whenever the user queries the status of
1613 * the interface using ifconfig.
1614 *
1615 **********************************************************************/
1616 static void
1617 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1618 {
1619 struct adapter *adapter = ifp->if_softc;
1620 u_char fiber_type = IFM_1000_SX;
1621
1622 INIT_DEBUGOUT("igb_media_status: begin");
1623
1624 IGB_CORE_LOCK(adapter);
1625 igb_update_link_status(adapter);
1626
1627 ifmr->ifm_status = IFM_AVALID;
1628 ifmr->ifm_active = IFM_ETHER;
1629
1630 if (!adapter->link_active) {
1631 IGB_CORE_UNLOCK(adapter);
1632 return;
1633 }
1634
1635 ifmr->ifm_status |= IFM_ACTIVE;
1636
1637 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1638 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes))
1639 ifmr->ifm_active |= fiber_type | IFM_FDX;
1640 else {
1641 switch (adapter->link_speed) {
1642 case 10:
1643 ifmr->ifm_active |= IFM_10_T;
1644 break;
1645 case 100:
1646 ifmr->ifm_active |= IFM_100_TX;
1647 break;
1648 case 1000:
1649 ifmr->ifm_active |= IFM_1000_T;
1650 break;
1651 }
1652 if (adapter->link_duplex == FULL_DUPLEX)
1653 ifmr->ifm_active |= IFM_FDX;
1654 else
1655 ifmr->ifm_active |= IFM_HDX;
1656 }
1657 IGB_CORE_UNLOCK(adapter);
1658 }
1659
1660 /*********************************************************************
1661 *
1662 * Media Ioctl callback
1663 *
1664 * This routine is called when the user changes speed/duplex using
1665 * media/mediopt option with ifconfig.
1666 *
1667 **********************************************************************/
1668 static int
1669 igb_media_change(struct ifnet *ifp)
1670 {
1671 struct adapter *adapter = ifp->if_softc;
1672 struct ifmedia *ifm = &adapter->media;
1673
1674 INIT_DEBUGOUT("igb_media_change: begin");
1675
1676 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1677 return (EINVAL);
1678
1679 IGB_CORE_LOCK(adapter);
1680 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1681 case IFM_AUTO:
1682 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1683 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1684 break;
1685 case IFM_1000_LX:
1686 case IFM_1000_SX:
1687 case IFM_1000_T:
1688 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1689 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1690 break;
1691 case IFM_100_TX:
1692 adapter->hw.mac.autoneg = FALSE;
1693 adapter->hw.phy.autoneg_advertised = 0;
1694 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1695 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1696 else
1697 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1698 break;
1699 case IFM_10_T:
1700 adapter->hw.mac.autoneg = FALSE;
1701 adapter->hw.phy.autoneg_advertised = 0;
1702 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1703 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1704 else
1705 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1706 break;
1707 default:
1708 device_printf(adapter->dev, "Unsupported media type\n");
1709 }
1710
1711 igb_init_locked(adapter);
1712 IGB_CORE_UNLOCK(adapter);
1713
1714 return (0);
1715 }
1716
1717
1718 /*********************************************************************
1719 *
1720 * This routine maps the mbufs to Advanced TX descriptors.
1721 * used by the 82575 adapter.
1722 *
1723 **********************************************************************/
1724
1725 static int
1726 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1727 {
1728 struct adapter *adapter = txr->adapter;
1729 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1730 bus_dmamap_t map;
1731 struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped;
1732 union e1000_adv_tx_desc *txd = NULL;
1733 struct mbuf *m_head;
1734 u32 olinfo_status = 0, cmd_type_len = 0;
1735 int nsegs, i, j, error, first, last = 0;
1736 u32 hdrlen = 0;
1737
1738 m_head = *m_headp;
1739
1740
1741 /* Set basic descriptor constants */
1742 cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
1743 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
1744 if (m_head->m_flags & M_VLANTAG)
1745 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1746
1747 /*
1748 * Map the packet for DMA.
1749 *
1750 * Capture the first descriptor index,
1751 * this descriptor will have the index
1752 * of the EOP which is the only one that
1753 * now gets a DONE bit writeback.
1754 */
1755 first = txr->next_avail_desc;
1756 tx_buffer = &txr->tx_buffers[first];
1757 tx_buffer_mapped = tx_buffer;
1758 map = tx_buffer->map;
1759
1760 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1761 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1762
1763 if (error == EFBIG) {
1764 struct mbuf *m;
1765
1766 m = m_defrag(*m_headp, M_DONTWAIT);
1767 if (m == NULL) {
1768 adapter->mbuf_defrag_failed++;
1769 m_freem(*m_headp);
1770 *m_headp = NULL;
1771 return (ENOBUFS);
1772 }
1773 *m_headp = m;
1774
1775 /* Try it again */
1776 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1777 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1778
1779 if (error == ENOMEM) {
1780 adapter->no_tx_dma_setup++;
1781 return (error);
1782 } else if (error != 0) {
1783 adapter->no_tx_dma_setup++;
1784 m_freem(*m_headp);
1785 *m_headp = NULL;
1786 return (error);
1787 }
1788 } else if (error == ENOMEM) {
1789 adapter->no_tx_dma_setup++;
1790 return (error);
1791 } else if (error != 0) {
1792 adapter->no_tx_dma_setup++;
1793 m_freem(*m_headp);
1794 *m_headp = NULL;
1795 return (error);
1796 }
1797
1798 /* Check again to be sure we have enough descriptors */
1799 if (nsegs > (txr->tx_avail - 2)) {
1800 txr->no_desc_avail++;
1801 bus_dmamap_unload(txr->txtag, map);
1802 return (ENOBUFS);
1803 }
1804 m_head = *m_headp;
1805
1806 /*
1807 * Set up the context descriptor:
1808 * used when any hardware offload is done.
1809 * This includes CSUM, VLAN, and TSO. It
1810 * will use the first descriptor.
1811 */
1812 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1813 if (igb_tso_setup(txr, m_head, &hdrlen)) {
1814 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1815 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
1816 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1817 } else
1818 return (ENXIO);
1819 } else if (igb_tx_ctx_setup(txr, m_head))
1820 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1821
1822 /* Calculate payload length */
1823 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
1824 << E1000_ADVTXD_PAYLEN_SHIFT);
1825
1826 /* 82575 needs the queue index added */
1827 if (adapter->hw.mac.type == e1000_82575)
1828 olinfo_status |= txr->me << 4;
1829
1830 /* Set up our transmit descriptors */
1831 i = txr->next_avail_desc;
1832 for (j = 0; j < nsegs; j++) {
1833 bus_size_t seg_len;
1834 bus_addr_t seg_addr;
1835
1836 tx_buffer = &txr->tx_buffers[i];
1837 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
1838 seg_addr = segs[j].ds_addr;
1839 seg_len = segs[j].ds_len;
1840
1841 txd->read.buffer_addr = htole64(seg_addr);
1842 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len);
1843 txd->read.olinfo_status = htole32(olinfo_status);
1844 last = i;
1845 if (++i == adapter->num_tx_desc)
1846 i = 0;
1847 tx_buffer->m_head = NULL;
1848 tx_buffer->next_eop = -1;
1849 }
1850
1851 txr->next_avail_desc = i;
1852 txr->tx_avail -= nsegs;
1853
1854 tx_buffer->m_head = m_head;
1855 tx_buffer_mapped->map = tx_buffer->map;
1856 tx_buffer->map = map;
1857 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1858
1859 /*
1860 * Last Descriptor of Packet
1861 * needs End Of Packet (EOP)
1862 * and Report Status (RS)
1863 */
1864 txd->read.cmd_type_len |=
1865 htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS);
1866 /*
1867 * Keep track in the first buffer which
1868 * descriptor will be written back
1869 */
1870 tx_buffer = &txr->tx_buffers[first];
1871 tx_buffer->next_eop = last;
1872 txr->watchdog_time = ticks;
1873
1874 /*
1875 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
1876 * that this frame is available to transmit.
1877 */
1878 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1879 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1880 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1881 ++txr->tx_packets;
1882
1883 return (0);
1884
1885 }
1886
1887 static void
1888 igb_set_promisc(struct adapter *adapter)
1889 {
1890 struct ifnet *ifp = adapter->ifp;
1891 struct e1000_hw *hw = &adapter->hw;
1892 u32 reg;
1893
1894 if (adapter->vf_ifp) {
1895 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
1896 return;
1897 }
1898
1899 reg = E1000_READ_REG(hw, E1000_RCTL);
1900 if (ifp->if_flags & IFF_PROMISC) {
1901 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1902 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1903 } else if (ifp->if_flags & IFF_ALLMULTI) {
1904 reg |= E1000_RCTL_MPE;
1905 reg &= ~E1000_RCTL_UPE;
1906 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1907 }
1908 }
1909
1910 static void
1911 igb_disable_promisc(struct adapter *adapter)
1912 {
1913 struct e1000_hw *hw = &adapter->hw;
1914 u32 reg;
1915
1916 if (adapter->vf_ifp) {
1917 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
1918 return;
1919 }
1920 reg = E1000_READ_REG(hw, E1000_RCTL);
1921 reg &= (~E1000_RCTL_UPE);
1922 reg &= (~E1000_RCTL_MPE);
1923 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1924 }
1925
1926
1927 /*********************************************************************
1928 * Multicast Update
1929 *
1930 * This routine is called whenever multicast address list is updated.
1931 *
1932 **********************************************************************/
1933
1934 static void
1935 igb_set_multi(struct adapter *adapter)
1936 {
1937 struct ifnet *ifp = adapter->ifp;
1938 struct ifmultiaddr *ifma;
1939 u32 reg_rctl = 0;
1940 u8 *mta;
1941
1942 int mcnt = 0;
1943
1944 IOCTL_DEBUGOUT("igb_set_multi: begin");
1945
1946 mta = adapter->mta;
1947 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
1948 MAX_NUM_MULTICAST_ADDRESSES);
1949
1950 #if __FreeBSD_version < 800000
1951 IF_ADDR_LOCK(ifp);
1952 #else
1953 if_maddr_rlock(ifp);
1954 #endif
1955 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1956 if (ifma->ifma_addr->sa_family != AF_LINK)
1957 continue;
1958
1959 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1960 break;
1961
1962 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1963 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
1964 mcnt++;
1965 }
1966 #if __FreeBSD_version < 800000
1967 IF_ADDR_UNLOCK(ifp);
1968 #else
1969 if_maddr_runlock(ifp);
1970 #endif
1971
1972 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1973 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1974 reg_rctl |= E1000_RCTL_MPE;
1975 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1976 } else
1977 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
1978 }
1979
1980
1981 /*********************************************************************
1982 * Timer routine:
1983 * This routine checks for link status,
1984 * updates statistics, and does the watchdog.
1985 *
1986 **********************************************************************/
1987
1988 static void
1989 igb_local_timer(void *arg)
1990 {
1991 struct adapter *adapter = arg;
1992 device_t dev = adapter->dev;
1993 struct tx_ring *txr = adapter->tx_rings;
1994
1995
1996 IGB_CORE_LOCK_ASSERT(adapter);
1997
1998 igb_update_link_status(adapter);
1999 igb_update_stats_counters(adapter);
2000
2001 /*
2002 ** If flow control has paused us since last checking
2003 ** it invalidates the watchdog timing, so dont run it.
2004 */
2005 if (adapter->pause_frames) {
2006 adapter->pause_frames = 0;
2007 goto out;
2008 }
2009
2010 /*
2011 ** Watchdog: check for time since any descriptor was cleaned
2012 */
2013 for (int i = 0; i < adapter->num_queues; i++, txr++)
2014 if (txr->queue_status == IGB_QUEUE_HUNG)
2015 goto timeout;
2016 out:
2017 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
2018 #ifndef DEVICE_POLLING
2019 /* Schedule all queue interrupts - deadlock protection */
2020 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
2021 #endif
2022 return;
2023
2024 timeout:
2025 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2026 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2027 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2028 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2029 device_printf(dev,"TX(%d) desc avail = %d,"
2030 "Next TX to Clean = %d\n",
2031 txr->me, txr->tx_avail, txr->next_to_clean);
2032 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2033 adapter->watchdog_events++;
2034 igb_init_locked(adapter);
2035 }
2036
2037 static void
2038 igb_update_link_status(struct adapter *adapter)
2039 {
2040 struct e1000_hw *hw = &adapter->hw;
2041 struct ifnet *ifp = adapter->ifp;
2042 device_t dev = adapter->dev;
2043 struct tx_ring *txr = adapter->tx_rings;
2044 u32 link_check, thstat, ctrl;
2045
2046 link_check = thstat = ctrl = 0;
2047
2048 /* Get the cached link value or read for real */
2049 switch (hw->phy.media_type) {
2050 case e1000_media_type_copper:
2051 if (hw->mac.get_link_status) {
2052 /* Do the work to read phy */
2053 e1000_check_for_link(hw);
2054 link_check = !hw->mac.get_link_status;
2055 } else
2056 link_check = TRUE;
2057 break;
2058 case e1000_media_type_fiber:
2059 e1000_check_for_link(hw);
2060 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2061 E1000_STATUS_LU);
2062 break;
2063 case e1000_media_type_internal_serdes:
2064 e1000_check_for_link(hw);
2065 link_check = adapter->hw.mac.serdes_has_link;
2066 break;
2067 /* VF device is type_unknown */
2068 case e1000_media_type_unknown:
2069 e1000_check_for_link(hw);
2070 link_check = !hw->mac.get_link_status;
2071 /* Fall thru */
2072 default:
2073 break;
2074 }
2075
2076 /* Check for thermal downshift or shutdown */
2077 if (hw->mac.type == e1000_i350) {
2078 thstat = E1000_READ_REG(hw, E1000_THSTAT);
2079 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
2080 }
2081
2082 /* Now we check if a transition has happened */
2083 if (link_check && (adapter->link_active == 0)) {
2084 e1000_get_speed_and_duplex(&adapter->hw,
2085 &adapter->link_speed, &adapter->link_duplex);
2086 if (bootverbose)
2087 device_printf(dev, "Link is up %d Mbps %s\n",
2088 adapter->link_speed,
2089 ((adapter->link_duplex == FULL_DUPLEX) ?
2090 "Full Duplex" : "Half Duplex"));
2091 adapter->link_active = 1;
2092 ifp->if_baudrate = adapter->link_speed * 1000000;
2093 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2094 (thstat & E1000_THSTAT_LINK_THROTTLE))
2095 device_printf(dev, "Link: thermal downshift\n");
2096 /* This can sleep */
2097 if_link_state_change(ifp, LINK_STATE_UP);
2098 } else if (!link_check && (adapter->link_active == 1)) {
2099 ifp->if_baudrate = adapter->link_speed = 0;
2100 adapter->link_duplex = 0;
2101 if (bootverbose)
2102 device_printf(dev, "Link is Down\n");
2103 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2104 (thstat & E1000_THSTAT_PWR_DOWN))
2105 device_printf(dev, "Link: thermal shutdown\n");
2106 adapter->link_active = 0;
2107 /* This can sleep */
2108 if_link_state_change(ifp, LINK_STATE_DOWN);
2109 /* Turn off watchdogs */
2110 for (int i = 0; i < adapter->num_queues; i++, txr++)
2111 txr->queue_status = IGB_QUEUE_IDLE;
2112 }
2113 }
2114
2115 /*********************************************************************
2116 *
2117 * This routine disables all traffic on the adapter by issuing a
2118 * global reset on the MAC and deallocates TX/RX buffers.
2119 *
2120 **********************************************************************/
2121
2122 static void
2123 igb_stop(void *arg)
2124 {
2125 struct adapter *adapter = arg;
2126 struct ifnet *ifp = adapter->ifp;
2127 struct tx_ring *txr = adapter->tx_rings;
2128
2129 IGB_CORE_LOCK_ASSERT(adapter);
2130
2131 INIT_DEBUGOUT("igb_stop: begin");
2132
2133 igb_disable_intr(adapter);
2134
2135 callout_stop(&adapter->timer);
2136
2137 /* Tell the stack that the interface is no longer active */
2138 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2139
2140 /* Unarm watchdog timer. */
2141 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2142 IGB_TX_LOCK(txr);
2143 txr->queue_status = IGB_QUEUE_IDLE;
2144 IGB_TX_UNLOCK(txr);
2145 }
2146
2147 e1000_reset_hw(&adapter->hw);
2148 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2149
2150 e1000_led_off(&adapter->hw);
2151 e1000_cleanup_led(&adapter->hw);
2152 }
2153
2154
2155 /*********************************************************************
2156 *
2157 * Determine hardware revision.
2158 *
2159 **********************************************************************/
2160 static void
2161 igb_identify_hardware(struct adapter *adapter)
2162 {
2163 device_t dev = adapter->dev;
2164
2165 /* Make sure our PCI config space has the necessary stuff set */
2166 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2167 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2168 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2169 INIT_DEBUGOUT("Memory Access and/or Bus Master "
2170 "bits were not set!\n");
2171 adapter->hw.bus.pci_cmd_word |=
2172 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2173 pci_write_config(dev, PCIR_COMMAND,
2174 adapter->hw.bus.pci_cmd_word, 2);
2175 }
2176
2177 /* Save off the information about this board */
2178 adapter->hw.vendor_id = pci_get_vendor(dev);
2179 adapter->hw.device_id = pci_get_device(dev);
2180 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2181 adapter->hw.subsystem_vendor_id =
2182 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2183 adapter->hw.subsystem_device_id =
2184 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2185
2186 /* Set MAC type early for PCI setup */
2187 e1000_set_mac_type(&adapter->hw);
2188
2189 /* Are we a VF device? */
2190 if ((adapter->hw.mac.type == e1000_vfadapt) ||
2191 (adapter->hw.mac.type == e1000_vfadapt_i350))
2192 adapter->vf_ifp = 1;
2193 else
2194 adapter->vf_ifp = 0;
2195 }
2196
2197 static int
2198 igb_allocate_pci_resources(struct adapter *adapter)
2199 {
2200 device_t dev = adapter->dev;
2201 int rid;
2202
2203 rid = PCIR_BAR(0);
2204 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2205 &rid, RF_ACTIVE);
2206 if (adapter->pci_mem == NULL) {
2207 device_printf(dev, "Unable to allocate bus resource: memory\n");
2208 return (ENXIO);
2209 }
2210 adapter->osdep.mem_bus_space_tag =
2211 rman_get_bustag(adapter->pci_mem);
2212 adapter->osdep.mem_bus_space_handle =
2213 rman_get_bushandle(adapter->pci_mem);
2214 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2215
2216 adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2217
2218 /* This will setup either MSI/X or MSI */
2219 adapter->msix = igb_setup_msix(adapter);
2220 adapter->hw.back = &adapter->osdep;
2221
2222 return (0);
2223 }
2224
2225 /*********************************************************************
2226 *
2227 * Setup the Legacy or MSI Interrupt handler
2228 *
2229 **********************************************************************/
2230 static int
2231 igb_allocate_legacy(struct adapter *adapter)
2232 {
2233 device_t dev = adapter->dev;
2234 struct igb_queue *que = adapter->queues;
2235 struct tx_ring *txr = adapter->tx_rings;
2236 int error, rid = 0;
2237
2238 /* Turn off all interrupts */
2239 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2240
2241 /* MSI RID is 1 */
2242 if (adapter->msix == 1)
2243 rid = 1;
2244
2245 /* We allocate a single interrupt resource */
2246 adapter->res = bus_alloc_resource_any(dev,
2247 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2248 if (adapter->res == NULL) {
2249 device_printf(dev, "Unable to allocate bus resource: "
2250 "interrupt\n");
2251 return (ENXIO);
2252 }
2253
2254 #if __FreeBSD_version >= 800000
2255 TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
2256 #endif
2257
2258 /*
2259 * Try allocating a fast interrupt and the associated deferred
2260 * processing contexts.
2261 */
2262 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2263 /* Make tasklet for deferred link handling */
2264 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2265 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2266 taskqueue_thread_enqueue, &que->tq);
2267 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
2268 device_get_nameunit(adapter->dev));
2269 if ((error = bus_setup_intr(dev, adapter->res,
2270 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2271 adapter, &adapter->tag)) != 0) {
2272 device_printf(dev, "Failed to register fast interrupt "
2273 "handler: %d\n", error);
2274 taskqueue_free(que->tq);
2275 que->tq = NULL;
2276 return (error);
2277 }
2278
2279 return (0);
2280 }
2281
2282
2283 /*********************************************************************
2284 *
2285 * Setup the MSIX Queue Interrupt handlers:
2286 *
2287 **********************************************************************/
2288 static int
2289 igb_allocate_msix(struct adapter *adapter)
2290 {
2291 device_t dev = adapter->dev;
2292 struct igb_queue *que = adapter->queues;
2293 int error, rid, vector = 0;
2294
2295
2296 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2297 rid = vector +1;
2298 que->res = bus_alloc_resource_any(dev,
2299 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2300 if (que->res == NULL) {
2301 device_printf(dev,
2302 "Unable to allocate bus resource: "
2303 "MSIX Queue Interrupt\n");
2304 return (ENXIO);
2305 }
2306 error = bus_setup_intr(dev, que->res,
2307 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2308 igb_msix_que, que, &que->tag);
2309 if (error) {
2310 que->res = NULL;
2311 device_printf(dev, "Failed to register Queue handler");
2312 return (error);
2313 }
2314 #if __FreeBSD_version >= 800504
2315 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2316 #endif
2317 que->msix = vector;
2318 if (adapter->hw.mac.type == e1000_82575)
2319 que->eims = E1000_EICR_TX_QUEUE0 << i;
2320 else
2321 que->eims = 1 << vector;
2322 /*
2323 ** Bind the msix vector, and thus the
2324 ** rings to the corresponding cpu.
2325 */
2326 if (adapter->num_queues > 1)
2327 bus_bind_intr(dev, que->res, i);
2328 #if __FreeBSD_version >= 800000
2329 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
2330 que->txr);
2331 #endif
2332 /* Make tasklet for deferred handling */
2333 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2334 que->tq = taskqueue_create("igb_que", M_NOWAIT,
2335 taskqueue_thread_enqueue, &que->tq);
2336 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2337 device_get_nameunit(adapter->dev));
2338 }
2339
2340 /* And Link */
2341 rid = vector + 1;
2342 adapter->res = bus_alloc_resource_any(dev,
2343 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2344 if (adapter->res == NULL) {
2345 device_printf(dev,
2346 "Unable to allocate bus resource: "
2347 "MSIX Link Interrupt\n");
2348 return (ENXIO);
2349 }
2350 if ((error = bus_setup_intr(dev, adapter->res,
2351 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2352 igb_msix_link, adapter, &adapter->tag)) != 0) {
2353 device_printf(dev, "Failed to register Link handler");
2354 return (error);
2355 }
2356 #if __FreeBSD_version >= 800504
2357 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2358 #endif
2359 adapter->linkvec = vector;
2360
2361 return (0);
2362 }
2363
2364
2365 static void
2366 igb_configure_queues(struct adapter *adapter)
2367 {
2368 struct e1000_hw *hw = &adapter->hw;
2369 struct igb_queue *que;
2370 u32 tmp, ivar = 0, newitr = 0;
2371
2372 /* First turn on RSS capability */
2373 if (adapter->hw.mac.type != e1000_82575)
2374 E1000_WRITE_REG(hw, E1000_GPIE,
2375 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2376 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2377
2378 /* Turn on MSIX */
2379 switch (adapter->hw.mac.type) {
2380 case e1000_82580:
2381 case e1000_i350:
2382 case e1000_vfadapt:
2383 case e1000_vfadapt_i350:
2384 /* RX entries */
2385 for (int i = 0; i < adapter->num_queues; i++) {
2386 u32 index = i >> 1;
2387 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2388 que = &adapter->queues[i];
2389 if (i & 1) {
2390 ivar &= 0xFF00FFFF;
2391 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2392 } else {
2393 ivar &= 0xFFFFFF00;
2394 ivar |= que->msix | E1000_IVAR_VALID;
2395 }
2396 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2397 }
2398 /* TX entries */
2399 for (int i = 0; i < adapter->num_queues; i++) {
2400 u32 index = i >> 1;
2401 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2402 que = &adapter->queues[i];
2403 if (i & 1) {
2404 ivar &= 0x00FFFFFF;
2405 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2406 } else {
2407 ivar &= 0xFFFF00FF;
2408 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2409 }
2410 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2411 adapter->que_mask |= que->eims;
2412 }
2413
2414 /* And for the link interrupt */
2415 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2416 adapter->link_mask = 1 << adapter->linkvec;
2417 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2418 break;
2419 case e1000_82576:
2420 /* RX entries */
2421 for (int i = 0; i < adapter->num_queues; i++) {
2422 u32 index = i & 0x7; /* Each IVAR has two entries */
2423 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2424 que = &adapter->queues[i];
2425 if (i < 8) {
2426 ivar &= 0xFFFFFF00;
2427 ivar |= que->msix | E1000_IVAR_VALID;
2428 } else {
2429 ivar &= 0xFF00FFFF;
2430 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2431 }
2432 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2433 adapter->que_mask |= que->eims;
2434 }
2435 /* TX entries */
2436 for (int i = 0; i < adapter->num_queues; i++) {
2437 u32 index = i & 0x7; /* Each IVAR has two entries */
2438 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2439 que = &adapter->queues[i];
2440 if (i < 8) {
2441 ivar &= 0xFFFF00FF;
2442 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2443 } else {
2444 ivar &= 0x00FFFFFF;
2445 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2446 }
2447 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2448 adapter->que_mask |= que->eims;
2449 }
2450
2451 /* And for the link interrupt */
2452 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2453 adapter->link_mask = 1 << adapter->linkvec;
2454 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2455 break;
2456
2457 case e1000_82575:
2458 /* enable MSI-X support*/
2459 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2460 tmp |= E1000_CTRL_EXT_PBA_CLR;
2461 /* Auto-Mask interrupts upon ICR read. */
2462 tmp |= E1000_CTRL_EXT_EIAME;
2463 tmp |= E1000_CTRL_EXT_IRCA;
2464 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2465
2466 /* Queues */
2467 for (int i = 0; i < adapter->num_queues; i++) {
2468 que = &adapter->queues[i];
2469 tmp = E1000_EICR_RX_QUEUE0 << i;
2470 tmp |= E1000_EICR_TX_QUEUE0 << i;
2471 que->eims = tmp;
2472 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2473 i, que->eims);
2474 adapter->que_mask |= que->eims;
2475 }
2476
2477 /* Link */
2478 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2479 E1000_EIMS_OTHER);
2480 adapter->link_mask |= E1000_EIMS_OTHER;
2481 default:
2482 break;
2483 }
2484
2485 /* Set the starting interrupt rate */
2486 if (igb_max_interrupt_rate > 0)
2487 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
2488
2489 if (hw->mac.type == e1000_82575)
2490 newitr |= newitr << 16;
2491 else
2492 newitr |= E1000_EITR_CNT_IGNR;
2493
2494 for (int i = 0; i < adapter->num_queues; i++) {
2495 que = &adapter->queues[i];
2496 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2497 }
2498
2499 return;
2500 }
2501
2502
2503 static void
2504 igb_free_pci_resources(struct adapter *adapter)
2505 {
2506 struct igb_queue *que = adapter->queues;
2507 device_t dev = adapter->dev;
2508 int rid;
2509
2510 /*
2511 ** There is a slight possibility of a failure mode
2512 ** in attach that will result in entering this function
2513 ** before interrupt resources have been initialized, and
2514 ** in that case we do not want to execute the loops below
2515 ** We can detect this reliably by the state of the adapter
2516 ** res pointer.
2517 */
2518 if (adapter->res == NULL)
2519 goto mem;
2520
2521 /*
2522 * First release all the interrupt resources:
2523 */
2524 for (int i = 0; i < adapter->num_queues; i++, que++) {
2525 rid = que->msix + 1;
2526 if (que->tag != NULL) {
2527 bus_teardown_intr(dev, que->res, que->tag);
2528 que->tag = NULL;
2529 }
2530 if (que->res != NULL)
2531 bus_release_resource(dev,
2532 SYS_RES_IRQ, rid, que->res);
2533 }
2534
2535 /* Clean the Legacy or Link interrupt last */
2536 if (adapter->linkvec) /* we are doing MSIX */
2537 rid = adapter->linkvec + 1;
2538 else
2539 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2540
2541 que = adapter->queues;
2542 if (adapter->tag != NULL) {
2543 taskqueue_drain(que->tq, &adapter->link_task);
2544 bus_teardown_intr(dev, adapter->res, adapter->tag);
2545 adapter->tag = NULL;
2546 }
2547 if (adapter->res != NULL)
2548 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2549
2550 for (int i = 0; i < adapter->num_queues; i++, que++) {
2551 if (que->tq != NULL) {
2552 #if __FreeBSD_version >= 800000
2553 taskqueue_drain(que->tq, &que->txr->txq_task);
2554 #endif
2555 taskqueue_drain(que->tq, &que->que_task);
2556 taskqueue_free(que->tq);
2557 }
2558 }
2559 mem:
2560 if (adapter->msix)
2561 pci_release_msi(dev);
2562
2563 if (adapter->msix_mem != NULL)
2564 bus_release_resource(dev, SYS_RES_MEMORY,
2565 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2566
2567 if (adapter->pci_mem != NULL)
2568 bus_release_resource(dev, SYS_RES_MEMORY,
2569 PCIR_BAR(0), adapter->pci_mem);
2570
2571 }
2572
2573 /*
2574 * Setup Either MSI/X or MSI
2575 */
2576 static int
2577 igb_setup_msix(struct adapter *adapter)
2578 {
2579 device_t dev = adapter->dev;
2580 int rid, want, queues, msgs;
2581
2582 /* tuneable override */
2583 if (igb_enable_msix == 0)
2584 goto msi;
2585
2586 /* First try MSI/X */
2587 rid = PCIR_BAR(IGB_MSIX_BAR);
2588 adapter->msix_mem = bus_alloc_resource_any(dev,
2589 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2590 if (!adapter->msix_mem) {
2591 /* May not be enabled */
2592 device_printf(adapter->dev,
2593 "Unable to map MSIX table \n");
2594 goto msi;
2595 }
2596
2597 msgs = pci_msix_count(dev);
2598 if (msgs == 0) { /* system has msix disabled */
2599 bus_release_resource(dev, SYS_RES_MEMORY,
2600 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2601 adapter->msix_mem = NULL;
2602 goto msi;
2603 }
2604
2605 /* Figure out a reasonable auto config value */
2606 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2607
2608 /* Manual override */
2609 if (igb_num_queues != 0)
2610 queues = igb_num_queues;
2611 if (queues > 8) /* max queues */
2612 queues = 8;
2613
2614 /* Can have max of 4 queues on 82575 */
2615 if ((adapter->hw.mac.type == e1000_82575) && (queues > 4))
2616 queues = 4;
2617
2618 /* Limit the VF devices to one queue */
2619 if (adapter->vf_ifp)
2620 queues = 1;
2621
2622 /*
2623 ** One vector (RX/TX pair) per queue
2624 ** plus an additional for Link interrupt
2625 */
2626 want = queues + 1;
2627 if (msgs >= want)
2628 msgs = want;
2629 else {
2630 device_printf(adapter->dev,
2631 "MSIX Configuration Problem, "
2632 "%d vectors configured, but %d queues wanted!\n",
2633 msgs, want);
2634 return (ENXIO);
2635 }
2636 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2637 device_printf(adapter->dev,
2638 "Using MSIX interrupts with %d vectors\n", msgs);
2639 adapter->num_queues = queues;
2640 return (msgs);
2641 }
2642 msi:
2643 msgs = pci_msi_count(dev);
2644 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2645 device_printf(adapter->dev,"Using MSI interrupt\n");
2646 return (msgs);
2647 }
2648
2649 /*********************************************************************
2650 *
2651 * Set up an fresh starting state
2652 *
2653 **********************************************************************/
2654 static void
2655 igb_reset(struct adapter *adapter)
2656 {
2657 device_t dev = adapter->dev;
2658 struct e1000_hw *hw = &adapter->hw;
2659 struct e1000_fc_info *fc = &hw->fc;
2660 struct ifnet *ifp = adapter->ifp;
2661 u32 pba = 0;
2662 u16 hwm;
2663
2664 INIT_DEBUGOUT("igb_reset: begin");
2665
2666 /* Let the firmware know the OS is in control */
2667 igb_get_hw_control(adapter);
2668
2669 /*
2670 * Packet Buffer Allocation (PBA)
2671 * Writing PBA sets the receive portion of the buffer
2672 * the remainder is used for the transmit buffer.
2673 */
2674 switch (hw->mac.type) {
2675 case e1000_82575:
2676 pba = E1000_PBA_32K;
2677 break;
2678 case e1000_82576:
2679 case e1000_vfadapt:
2680 pba = E1000_READ_REG(hw, E1000_RXPBS);
2681 pba &= E1000_RXPBS_SIZE_MASK_82576;
2682 break;
2683 case e1000_82580:
2684 case e1000_i350:
2685 case e1000_vfadapt_i350:
2686 pba = E1000_READ_REG(hw, E1000_RXPBS);
2687 pba = e1000_rxpbs_adjust_82580(pba);
2688 break;
2689 pba = E1000_PBA_35K;
2690 default:
2691 break;
2692 }
2693
2694 /* Special needs in case of Jumbo frames */
2695 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
2696 u32 tx_space, min_tx, min_rx;
2697 pba = E1000_READ_REG(hw, E1000_PBA);
2698 tx_space = pba >> 16;
2699 pba &= 0xffff;
2700 min_tx = (adapter->max_frame_size +
2701 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
2702 min_tx = roundup2(min_tx, 1024);
2703 min_tx >>= 10;
2704 min_rx = adapter->max_frame_size;
2705 min_rx = roundup2(min_rx, 1024);
2706 min_rx >>= 10;
2707 if (tx_space < min_tx &&
2708 ((min_tx - tx_space) < pba)) {
2709 pba = pba - (min_tx - tx_space);
2710 /*
2711 * if short on rx space, rx wins
2712 * and must trump tx adjustment
2713 */
2714 if (pba < min_rx)
2715 pba = min_rx;
2716 }
2717 E1000_WRITE_REG(hw, E1000_PBA, pba);
2718 }
2719
2720 INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
2721
2722 /*
2723 * These parameters control the automatic generation (Tx) and
2724 * response (Rx) to Ethernet PAUSE frames.
2725 * - High water mark should allow for at least two frames to be
2726 * received after sending an XOFF.
2727 * - Low water mark works best when it is very near the high water mark.
2728 * This allows the receiver to restart by sending XON when it has
2729 * drained a bit.
2730 */
2731 hwm = min(((pba << 10) * 9 / 10),
2732 ((pba << 10) - 2 * adapter->max_frame_size));
2733
2734 if (hw->mac.type < e1000_82576) {
2735 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
2736 fc->low_water = fc->high_water - 8;
2737 } else {
2738 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
2739 fc->low_water = fc->high_water - 16;
2740 }
2741
2742 fc->pause_time = IGB_FC_PAUSE_TIME;
2743 fc->send_xon = TRUE;
2744 if (fc->requested_mode)
2745 fc->current_mode = fc->requested_mode;
2746 else
2747 fc->current_mode = e1000_fc_full;
2748
2749 adapter->fc = fc->current_mode;
2750
2751 /* Issue a global reset */
2752 e1000_reset_hw(hw);
2753 E1000_WRITE_REG(hw, E1000_WUC, 0);
2754
2755 if (e1000_init_hw(hw) < 0)
2756 device_printf(dev, "Hardware Initialization Failed\n");
2757
2758 /* Setup DMA Coalescing */
2759 if (hw->mac.type == e1000_i350) {
2760 u32 reg = ~E1000_DMACR_DMAC_EN;
2761
2762 if (adapter->dmac == 0) { /* Disabling it */
2763 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2764 goto reset_out;
2765 }
2766
2767 hwm = (pba - 4) << 10;
2768 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
2769 & E1000_DMACR_DMACTHR_MASK);
2770
2771 /* transition to L0x or L1 if available..*/
2772 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
2773
2774 /* timer = value in adapter->dmac in 32usec intervals */
2775 reg |= (adapter->dmac >> 5);
2776 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2777
2778 /* No lower threshold */
2779 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
2780
2781 /* set hwm to PBA - 2 * max frame size */
2782 E1000_WRITE_REG(hw, E1000_FCRTC, hwm);
2783
2784 /* Set the interval before transition */
2785 reg = E1000_READ_REG(hw, E1000_DMCTLX);
2786 reg |= 0x800000FF; /* 255 usec */
2787 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
2788
2789 /* free space in tx packet buffer to wake from DMA coal */
2790 E1000_WRITE_REG(hw, E1000_DMCTXTH,
2791 (20480 - (2 * adapter->max_frame_size)) >> 6);
2792
2793 /* make low power state decision controlled by DMA coal */
2794 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2795 E1000_WRITE_REG(hw, E1000_PCIEMISC,
2796 reg | E1000_PCIEMISC_LX_DECISION);
2797 device_printf(dev, "DMA Coalescing enabled\n");
2798 }
2799
2800 reset_out:
2801 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
2802 e1000_get_phy_info(hw);
2803 e1000_check_for_link(hw);
2804 return;
2805 }
2806
2807 /*********************************************************************
2808 *
2809 * Setup networking device structure and register an interface.
2810 *
2811 **********************************************************************/
2812 static int
2813 igb_setup_interface(device_t dev, struct adapter *adapter)
2814 {
2815 struct ifnet *ifp;
2816
2817 INIT_DEBUGOUT("igb_setup_interface: begin");
2818
2819 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2820 if (ifp == NULL) {
2821 device_printf(dev, "can not allocate ifnet structure\n");
2822 return (-1);
2823 }
2824 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2825 ifp->if_mtu = ETHERMTU;
2826 ifp->if_init = igb_init;
2827 ifp->if_softc = adapter;
2828 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2829 ifp->if_ioctl = igb_ioctl;
2830 #if __FreeBSD_version >= 800000
2831 ifp->if_transmit = igb_mq_start;
2832 ifp->if_qflush = igb_qflush;
2833 #else
2834 ifp->if_start = igb_start;
2835 #endif
2836 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2837 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2838 IFQ_SET_READY(&ifp->if_snd);
2839
2840 ether_ifattach(ifp, adapter->hw.mac.addr);
2841
2842 ifp->if_capabilities = ifp->if_capenable = 0;
2843
2844 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2845 ifp->if_capabilities |= IFCAP_TSO4;
2846 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2847 ifp->if_capenable = ifp->if_capabilities;
2848
2849 /* Don't enable LRO by default */
2850 ifp->if_capabilities |= IFCAP_LRO;
2851
2852 #ifdef DEVICE_POLLING
2853 ifp->if_capabilities |= IFCAP_POLLING;
2854 #endif
2855
2856 /*
2857 * Tell the upper layer(s) we
2858 * support full VLAN capability.
2859 */
2860 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2861 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2862 | IFCAP_VLAN_HWTSO
2863 | IFCAP_VLAN_MTU;
2864 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
2865 | IFCAP_VLAN_HWTSO
2866 | IFCAP_VLAN_MTU;
2867
2868 /*
2869 ** Don't turn this on by default, if vlans are
2870 ** created on another pseudo device (eg. lagg)
2871 ** then vlan events are not passed thru, breaking
2872 ** operation, but with HW FILTER off it works. If
2873 ** using vlans directly on the igb driver you can
2874 ** enable this and get full hardware tag filtering.
2875 */
2876 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2877
2878 /*
2879 * Specify the media types supported by this adapter and register
2880 * callbacks to update media and link information
2881 */
2882 ifmedia_init(&adapter->media, IFM_IMASK,
2883 igb_media_change, igb_media_status);
2884 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2885 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2886 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
2887 0, NULL);
2888 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2889 } else {
2890 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2891 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2892 0, NULL);
2893 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2894 0, NULL);
2895 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2896 0, NULL);
2897 if (adapter->hw.phy.type != e1000_phy_ife) {
2898 ifmedia_add(&adapter->media,
2899 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2900 ifmedia_add(&adapter->media,
2901 IFM_ETHER | IFM_1000_T, 0, NULL);
2902 }
2903 }
2904 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2905 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2906 return (0);
2907 }
2908
2909
2910 /*
2911 * Manage DMA'able memory.
2912 */
2913 static void
2914 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2915 {
2916 if (error)
2917 return;
2918 *(bus_addr_t *) arg = segs[0].ds_addr;
2919 }
2920
2921 static int
2922 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
2923 struct igb_dma_alloc *dma, int mapflags)
2924 {
2925 int error;
2926
2927 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2928 IGB_DBA_ALIGN, 0, /* alignment, bounds */
2929 BUS_SPACE_MAXADDR, /* lowaddr */
2930 BUS_SPACE_MAXADDR, /* highaddr */
2931 NULL, NULL, /* filter, filterarg */
2932 size, /* maxsize */
2933 1, /* nsegments */
2934 size, /* maxsegsize */
2935 0, /* flags */
2936 NULL, /* lockfunc */
2937 NULL, /* lockarg */
2938 &dma->dma_tag);
2939 if (error) {
2940 device_printf(adapter->dev,
2941 "%s: bus_dma_tag_create failed: %d\n",
2942 __func__, error);
2943 goto fail_0;
2944 }
2945
2946 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2947 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
2948 if (error) {
2949 device_printf(adapter->dev,
2950 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2951 __func__, (uintmax_t)size, error);
2952 goto fail_2;
2953 }
2954
2955 dma->dma_paddr = 0;
2956 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2957 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2958 if (error || dma->dma_paddr == 0) {
2959 device_printf(adapter->dev,
2960 "%s: bus_dmamap_load failed: %d\n",
2961 __func__, error);
2962 goto fail_3;
2963 }
2964
2965 return (0);
2966
2967 fail_3:
2968 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2969 fail_2:
2970 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2971 bus_dma_tag_destroy(dma->dma_tag);
2972 fail_0:
2973 dma->dma_map = NULL;
2974 dma->dma_tag = NULL;
2975
2976 return (error);
2977 }
2978
2979 static void
2980 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
2981 {
2982 if (dma->dma_tag == NULL)
2983 return;
2984 if (dma->dma_map != NULL) {
2985 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2986 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2987 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2988 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2989 dma->dma_map = NULL;
2990 }
2991 bus_dma_tag_destroy(dma->dma_tag);
2992 dma->dma_tag = NULL;
2993 }
2994
2995
2996 /*********************************************************************
2997 *
2998 * Allocate memory for the transmit and receive rings, and then
2999 * the descriptors associated with each, called only once at attach.
3000 *
3001 **********************************************************************/
3002 static int
3003 igb_allocate_queues(struct adapter *adapter)
3004 {
3005 device_t dev = adapter->dev;
3006 struct igb_queue *que = NULL;
3007 struct tx_ring *txr = NULL;
3008 struct rx_ring *rxr = NULL;
3009 int rsize, tsize, error = E1000_SUCCESS;
3010 int txconf = 0, rxconf = 0;
3011
3012 /* First allocate the top level queue structs */
3013 if (!(adapter->queues =
3014 (struct igb_queue *) malloc(sizeof(struct igb_queue) *
3015 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3016 device_printf(dev, "Unable to allocate queue memory\n");
3017 error = ENOMEM;
3018 goto fail;
3019 }
3020
3021 /* Next allocate the TX ring struct memory */
3022 if (!(adapter->tx_rings =
3023 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3024 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3025 device_printf(dev, "Unable to allocate TX ring memory\n");
3026 error = ENOMEM;
3027 goto tx_fail;
3028 }
3029
3030 /* Now allocate the RX */
3031 if (!(adapter->rx_rings =
3032 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3033 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3034 device_printf(dev, "Unable to allocate RX ring memory\n");
3035 error = ENOMEM;
3036 goto rx_fail;
3037 }
3038
3039 tsize = roundup2(adapter->num_tx_desc *
3040 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
3041 /*
3042 * Now set up the TX queues, txconf is needed to handle the
3043 * possibility that things fail midcourse and we need to
3044 * undo memory gracefully
3045 */
3046 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3047 /* Set up some basics */
3048 txr = &adapter->tx_rings[i];
3049 txr->adapter = adapter;
3050 txr->me = i;
3051
3052 /* Initialize the TX lock */
3053 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3054 device_get_nameunit(dev), txr->me);
3055 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3056
3057 if (igb_dma_malloc(adapter, tsize,
3058 &txr->txdma, BUS_DMA_NOWAIT)) {
3059 device_printf(dev,
3060 "Unable to allocate TX Descriptor memory\n");
3061 error = ENOMEM;
3062 goto err_tx_desc;
3063 }
3064 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
3065 bzero((void *)txr->tx_base, tsize);
3066
3067 /* Now allocate transmit buffers for the ring */
3068 if (igb_allocate_transmit_buffers(txr)) {
3069 device_printf(dev,
3070 "Critical Failure setting up transmit buffers\n");
3071 error = ENOMEM;
3072 goto err_tx_desc;
3073 }
3074 #if __FreeBSD_version >= 800000
3075 /* Allocate a buf ring */
3076 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF,
3077 M_WAITOK, &txr->tx_mtx);
3078 #endif
3079 }
3080
3081 /*
3082 * Next the RX queues...
3083 */
3084 rsize = roundup2(adapter->num_rx_desc *
3085 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3086 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3087 rxr = &adapter->rx_rings[i];
3088 rxr->adapter = adapter;
3089 rxr->me = i;
3090
3091 /* Initialize the RX lock */
3092 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3093 device_get_nameunit(dev), txr->me);
3094 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3095
3096 if (igb_dma_malloc(adapter, rsize,
3097 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3098 device_printf(dev,
3099 "Unable to allocate RxDescriptor memory\n");
3100 error = ENOMEM;
3101 goto err_rx_desc;
3102 }
3103 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3104 bzero((void *)rxr->rx_base, rsize);
3105
3106 /* Allocate receive buffers for the ring*/
3107 if (igb_allocate_receive_buffers(rxr)) {
3108 device_printf(dev,
3109 "Critical Failure setting up receive buffers\n");
3110 error = ENOMEM;
3111 goto err_rx_desc;
3112 }
3113 }
3114
3115 /*
3116 ** Finally set up the queue holding structs
3117 */
3118 for (int i = 0; i < adapter->num_queues; i++) {
3119 que = &adapter->queues[i];
3120 que->adapter = adapter;
3121 que->txr = &adapter->tx_rings[i];
3122 que->rxr = &adapter->rx_rings[i];
3123 }
3124
3125 return (0);
3126
3127 err_rx_desc:
3128 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3129 igb_dma_free(adapter, &rxr->rxdma);
3130 err_tx_desc:
3131 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3132 igb_dma_free(adapter, &txr->txdma);
3133 free(adapter->rx_rings, M_DEVBUF);
3134 rx_fail:
3135 #if __FreeBSD_version >= 800000
3136 buf_ring_free(txr->br, M_DEVBUF);
3137 #endif
3138 free(adapter->tx_rings, M_DEVBUF);
3139 tx_fail:
3140 free(adapter->queues, M_DEVBUF);
3141 fail:
3142 return (error);
3143 }
3144
3145 /*********************************************************************
3146 *
3147 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3148 * the information needed to transmit a packet on the wire. This is
3149 * called only once at attach, setup is done every reset.
3150 *
3151 **********************************************************************/
3152 static int
3153 igb_allocate_transmit_buffers(struct tx_ring *txr)
3154 {
3155 struct adapter *adapter = txr->adapter;
3156 device_t dev = adapter->dev;
3157 struct igb_tx_buffer *txbuf;
3158 int error, i;
3159
3160 /*
3161 * Setup DMA descriptor areas.
3162 */
3163 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3164 1, 0, /* alignment, bounds */
3165 BUS_SPACE_MAXADDR, /* lowaddr */
3166 BUS_SPACE_MAXADDR, /* highaddr */
3167 NULL, NULL, /* filter, filterarg */
3168 IGB_TSO_SIZE, /* maxsize */
3169 IGB_MAX_SCATTER, /* nsegments */
3170 PAGE_SIZE, /* maxsegsize */
3171 0, /* flags */
3172 NULL, /* lockfunc */
3173 NULL, /* lockfuncarg */
3174 &txr->txtag))) {
3175 device_printf(dev,"Unable to allocate TX DMA tag\n");
3176 goto fail;
3177 }
3178
3179 if (!(txr->tx_buffers =
3180 (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) *
3181 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3182 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3183 error = ENOMEM;
3184 goto fail;
3185 }
3186
3187 /* Create the descriptor buffer dma maps */
3188 txbuf = txr->tx_buffers;
3189 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3190 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3191 if (error != 0) {
3192 device_printf(dev, "Unable to create TX DMA map\n");
3193 goto fail;
3194 }
3195 }
3196
3197 return 0;
3198 fail:
3199 /* We free all, it handles case where we are in the middle */
3200 igb_free_transmit_structures(adapter);
3201 return (error);
3202 }
3203
3204 /*********************************************************************
3205 *
3206 * Initialize a transmit ring.
3207 *
3208 **********************************************************************/
3209 static void
3210 igb_setup_transmit_ring(struct tx_ring *txr)
3211 {
3212 struct adapter *adapter = txr->adapter;
3213 struct igb_tx_buffer *txbuf;
3214 int i;
3215
3216 /* Clear the old descriptor contents */
3217 IGB_TX_LOCK(txr);
3218 bzero((void *)txr->tx_base,
3219 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3220 /* Reset indices */
3221 txr->next_avail_desc = 0;
3222 txr->next_to_clean = 0;
3223
3224 /* Free any existing tx buffers. */
3225 txbuf = txr->tx_buffers;
3226 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3227 if (txbuf->m_head != NULL) {
3228 bus_dmamap_sync(txr->txtag, txbuf->map,
3229 BUS_DMASYNC_POSTWRITE);
3230 bus_dmamap_unload(txr->txtag, txbuf->map);
3231 m_freem(txbuf->m_head);
3232 txbuf->m_head = NULL;
3233 }
3234 /* clear the watch index */
3235 txbuf->next_eop = -1;
3236 }
3237
3238 /* Set number of descriptors available */
3239 txr->tx_avail = adapter->num_tx_desc;
3240
3241 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3242 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3243 IGB_TX_UNLOCK(txr);
3244 }
3245
3246 /*********************************************************************
3247 *
3248 * Initialize all transmit rings.
3249 *
3250 **********************************************************************/
3251 static void
3252 igb_setup_transmit_structures(struct adapter *adapter)
3253 {
3254 struct tx_ring *txr = adapter->tx_rings;
3255
3256 for (int i = 0; i < adapter->num_queues; i++, txr++)
3257 igb_setup_transmit_ring(txr);
3258
3259 return;
3260 }
3261
3262 /*********************************************************************
3263 *
3264 * Enable transmit unit.
3265 *
3266 **********************************************************************/
3267 static void
3268 igb_initialize_transmit_units(struct adapter *adapter)
3269 {
3270 struct tx_ring *txr = adapter->tx_rings;
3271 struct e1000_hw *hw = &adapter->hw;
3272 u32 tctl, txdctl;
3273
3274 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3275 tctl = txdctl = 0;
3276
3277 /* Setup the Tx Descriptor Rings */
3278 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3279 u64 bus_addr = txr->txdma.dma_paddr;
3280
3281 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3282 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3283 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3284 (uint32_t)(bus_addr >> 32));
3285 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3286 (uint32_t)bus_addr);
3287
3288 /* Setup the HW Tx Head and Tail descriptor pointers */
3289 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3290 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3291
3292 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3293 E1000_READ_REG(hw, E1000_TDBAL(i)),
3294 E1000_READ_REG(hw, E1000_TDLEN(i)));
3295
3296 txr->queue_status = IGB_QUEUE_IDLE;
3297
3298 txdctl |= IGB_TX_PTHRESH;
3299 txdctl |= IGB_TX_HTHRESH << 8;
3300 txdctl |= IGB_TX_WTHRESH << 16;
3301 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3302 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3303 }
3304
3305 if (adapter->vf_ifp)
3306 return;
3307
3308 e1000_config_collision_dist(hw);
3309
3310 /* Program the Transmit Control Register */
3311 tctl = E1000_READ_REG(hw, E1000_TCTL);
3312 tctl &= ~E1000_TCTL_CT;
3313 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3314 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3315
3316 /* This write will effectively turn on the transmit unit. */
3317 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3318 }
3319
3320 /*********************************************************************
3321 *
3322 * Free all transmit rings.
3323 *
3324 **********************************************************************/
3325 static void
3326 igb_free_transmit_structures(struct adapter *adapter)
3327 {
3328 struct tx_ring *txr = adapter->tx_rings;
3329
3330 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3331 IGB_TX_LOCK(txr);
3332 igb_free_transmit_buffers(txr);
3333 igb_dma_free(adapter, &txr->txdma);
3334 IGB_TX_UNLOCK(txr);
3335 IGB_TX_LOCK_DESTROY(txr);
3336 }
3337 free(adapter->tx_rings, M_DEVBUF);
3338 }
3339
3340 /*********************************************************************
3341 *
3342 * Free transmit ring related data structures.
3343 *
3344 **********************************************************************/
3345 static void
3346 igb_free_transmit_buffers(struct tx_ring *txr)
3347 {
3348 struct adapter *adapter = txr->adapter;
3349 struct igb_tx_buffer *tx_buffer;
3350 int i;
3351
3352 INIT_DEBUGOUT("free_transmit_ring: begin");
3353
3354 if (txr->tx_buffers == NULL)
3355 return;
3356
3357 tx_buffer = txr->tx_buffers;
3358 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3359 if (tx_buffer->m_head != NULL) {
3360 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3361 BUS_DMASYNC_POSTWRITE);
3362 bus_dmamap_unload(txr->txtag,
3363 tx_buffer->map);
3364 m_freem(tx_buffer->m_head);
3365 tx_buffer->m_head = NULL;
3366 if (tx_buffer->map != NULL) {
3367 bus_dmamap_destroy(txr->txtag,
3368 tx_buffer->map);
3369 tx_buffer->map = NULL;
3370 }
3371 } else if (tx_buffer->map != NULL) {
3372 bus_dmamap_unload(txr->txtag,
3373 tx_buffer->map);
3374 bus_dmamap_destroy(txr->txtag,
3375 tx_buffer->map);
3376 tx_buffer->map = NULL;
3377 }
3378 }
3379 #if __FreeBSD_version >= 800000
3380 if (txr->br != NULL)
3381 buf_ring_free(txr->br, M_DEVBUF);
3382 #endif
3383 if (txr->tx_buffers != NULL) {
3384 free(txr->tx_buffers, M_DEVBUF);
3385 txr->tx_buffers = NULL;
3386 }
3387 if (txr->txtag != NULL) {
3388 bus_dma_tag_destroy(txr->txtag);
3389 txr->txtag = NULL;
3390 }
3391 return;
3392 }
3393
3394 /**********************************************************************
3395 *
3396 * Setup work for hardware segmentation offload (TSO)
3397 *
3398 **********************************************************************/
3399 static boolean_t
3400 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
3401 {
3402 struct adapter *adapter = txr->adapter;
3403 struct e1000_adv_tx_context_desc *TXD;
3404 struct igb_tx_buffer *tx_buffer;
3405 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3406 u32 mss_l4len_idx = 0;
3407 u16 vtag = 0;
3408 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3409 struct ether_vlan_header *eh;
3410 struct ip *ip;
3411 struct tcphdr *th;
3412
3413
3414 /*
3415 * Determine where frame payload starts.
3416 * Jump over vlan headers if already present
3417 */
3418 eh = mtod(mp, struct ether_vlan_header *);
3419 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3420 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3421 else
3422 ehdrlen = ETHER_HDR_LEN;
3423
3424 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3425 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3426 return FALSE;
3427
3428 /* Only supports IPV4 for now */
3429 ctxd = txr->next_avail_desc;
3430 tx_buffer = &txr->tx_buffers[ctxd];
3431 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3432
3433 ip = (struct ip *)(mp->m_data + ehdrlen);
3434 if (ip->ip_p != IPPROTO_TCP)
3435 return FALSE; /* 0 */
3436 ip->ip_sum = 0;
3437 ip_hlen = ip->ip_hl << 2;
3438 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3439 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3440 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3441 tcp_hlen = th->th_off << 2;
3442 /*
3443 * Calculate header length, this is used
3444 * in the transmit desc in igb_xmit
3445 */
3446 *hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3447
3448 /* VLAN MACLEN IPLEN */
3449 if (mp->m_flags & M_VLANTAG) {
3450 vtag = htole16(mp->m_pkthdr.ether_vtag);
3451 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3452 }
3453
3454 vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT);
3455 vlan_macip_lens |= ip_hlen;
3456 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3457
3458 /* ADV DTYPE TUCMD */
3459 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3460 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3461 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3462 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3463
3464 /* MSS L4LEN IDX */
3465 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3466 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3467 /* 82575 needs the queue index added */
3468 if (adapter->hw.mac.type == e1000_82575)
3469 mss_l4len_idx |= txr->me << 4;
3470 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3471
3472 TXD->seqnum_seed = htole32(0);
3473 tx_buffer->m_head = NULL;
3474 tx_buffer->next_eop = -1;
3475
3476 if (++ctxd == adapter->num_tx_desc)
3477 ctxd = 0;
3478
3479 txr->tx_avail--;
3480 txr->next_avail_desc = ctxd;
3481 return TRUE;
3482 }
3483
3484
3485 /*********************************************************************
3486 *
3487 * Context Descriptor setup for VLAN or CSUM
3488 *
3489 **********************************************************************/
3490
3491 static bool
3492 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3493 {
3494 struct adapter *adapter = txr->adapter;
3495 struct e1000_adv_tx_context_desc *TXD;
3496 struct igb_tx_buffer *tx_buffer;
3497 u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx;
3498 struct ether_vlan_header *eh;
3499 struct ip *ip = NULL;
3500 struct ip6_hdr *ip6;
3501 int ehdrlen, ctxd, ip_hlen = 0;
3502 u16 etype, vtag = 0;
3503 u8 ipproto = 0;
3504 bool offload = TRUE;
3505
3506 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3507 offload = FALSE;
3508
3509 vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0;
3510 ctxd = txr->next_avail_desc;
3511 tx_buffer = &txr->tx_buffers[ctxd];
3512 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3513
3514 /*
3515 ** In advanced descriptors the vlan tag must
3516 ** be placed into the context descriptor, thus
3517 ** we need to be here just for that setup.
3518 */
3519 if (mp->m_flags & M_VLANTAG) {
3520 vtag = htole16(mp->m_pkthdr.ether_vtag);
3521 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3522 } else if (offload == FALSE)
3523 return FALSE;
3524
3525 /*
3526 * Determine where frame payload starts.
3527 * Jump over vlan headers if already present,
3528 * helpful for QinQ too.
3529 */
3530 eh = mtod(mp, struct ether_vlan_header *);
3531 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3532 etype = ntohs(eh->evl_proto);
3533 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3534 } else {
3535 etype = ntohs(eh->evl_encap_proto);
3536 ehdrlen = ETHER_HDR_LEN;
3537 }
3538
3539 /* Set the ether header length */
3540 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3541
3542 switch (etype) {
3543 case ETHERTYPE_IP:
3544 ip = (struct ip *)(mp->m_data + ehdrlen);
3545 ip_hlen = ip->ip_hl << 2;
3546 if (mp->m_len < ehdrlen + ip_hlen) {
3547 offload = FALSE;
3548 break;
3549 }
3550 ipproto = ip->ip_p;
3551 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3552 break;
3553 case ETHERTYPE_IPV6:
3554 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3555 ip_hlen = sizeof(struct ip6_hdr);
3556 ipproto = ip6->ip6_nxt;
3557 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3558 break;
3559 default:
3560 offload = FALSE;
3561 break;
3562 }
3563
3564 vlan_macip_lens |= ip_hlen;
3565 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3566
3567 switch (ipproto) {
3568 case IPPROTO_TCP:
3569 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3570 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3571 break;
3572 case IPPROTO_UDP:
3573 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3574 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3575 break;
3576 #if __FreeBSD_version >= 800000
3577 case IPPROTO_SCTP:
3578 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3579 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3580 break;
3581 #endif
3582 default:
3583 offload = FALSE;
3584 break;
3585 }
3586
3587 /* 82575 needs the queue index added */
3588 if (adapter->hw.mac.type == e1000_82575)
3589 mss_l4len_idx = txr->me << 4;
3590
3591 /* Now copy bits into descriptor */
3592 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3593 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3594 TXD->seqnum_seed = htole32(0);
3595 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3596
3597 tx_buffer->m_head = NULL;
3598 tx_buffer->next_eop = -1;
3599
3600 /* We've consumed the first desc, adjust counters */
3601 if (++ctxd == adapter->num_tx_desc)
3602 ctxd = 0;
3603 txr->next_avail_desc = ctxd;
3604 --txr->tx_avail;
3605
3606 return (offload);
3607 }
3608
3609
3610 /**********************************************************************
3611 *
3612 * Examine each tx_buffer in the used queue. If the hardware is done
3613 * processing the packet then free associated resources. The
3614 * tx_buffer is put back on the free queue.
3615 *
3616 * TRUE return means there's work in the ring to clean, FALSE its empty.
3617 **********************************************************************/
3618 static bool
3619 igb_txeof(struct tx_ring *txr)
3620 {
3621 struct adapter *adapter = txr->adapter;
3622 int first, last, done, processed;
3623 struct igb_tx_buffer *tx_buffer;
3624 struct e1000_tx_desc *tx_desc, *eop_desc;
3625 struct ifnet *ifp = adapter->ifp;
3626
3627 IGB_TX_LOCK_ASSERT(txr);
3628
3629 if (txr->tx_avail == adapter->num_tx_desc) {
3630 txr->queue_status = IGB_QUEUE_IDLE;
3631 return FALSE;
3632 }
3633
3634 processed = 0;
3635 first = txr->next_to_clean;
3636 tx_desc = &txr->tx_base[first];
3637 tx_buffer = &txr->tx_buffers[first];
3638 last = tx_buffer->next_eop;
3639 eop_desc = &txr->tx_base[last];
3640
3641 /*
3642 * What this does is get the index of the
3643 * first descriptor AFTER the EOP of the
3644 * first packet, that way we can do the
3645 * simple comparison on the inner while loop.
3646 */
3647 if (++last == adapter->num_tx_desc)
3648 last = 0;
3649 done = last;
3650
3651 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3652 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3653
3654 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3655 /* We clean the range of the packet */
3656 while (first != done) {
3657 tx_desc->upper.data = 0;
3658 tx_desc->lower.data = 0;
3659 tx_desc->buffer_addr = 0;
3660 ++txr->tx_avail;
3661 ++processed;
3662
3663 if (tx_buffer->m_head) {
3664 txr->bytes +=
3665 tx_buffer->m_head->m_pkthdr.len;
3666 bus_dmamap_sync(txr->txtag,
3667 tx_buffer->map,
3668 BUS_DMASYNC_POSTWRITE);
3669 bus_dmamap_unload(txr->txtag,
3670 tx_buffer->map);
3671
3672 m_freem(tx_buffer->m_head);
3673 tx_buffer->m_head = NULL;
3674 }
3675 tx_buffer->next_eop = -1;
3676 txr->watchdog_time = ticks;
3677
3678 if (++first == adapter->num_tx_desc)
3679 first = 0;
3680
3681 tx_buffer = &txr->tx_buffers[first];
3682 tx_desc = &txr->tx_base[first];
3683 }
3684 ++txr->packets;
3685 ++ifp->if_opackets;
3686 /* See if we can continue to the next packet */
3687 last = tx_buffer->next_eop;
3688 if (last != -1) {
3689 eop_desc = &txr->tx_base[last];
3690 /* Get new done point */
3691 if (++last == adapter->num_tx_desc) last = 0;
3692 done = last;
3693 } else
3694 break;
3695 }
3696 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3697 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3698
3699 txr->next_to_clean = first;
3700
3701 /*
3702 ** Watchdog calculation, we know there's
3703 ** work outstanding or the first return
3704 ** would have been taken, so none processed
3705 ** for too long indicates a hang.
3706 */
3707 if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
3708 txr->queue_status = IGB_QUEUE_HUNG;
3709
3710 /*
3711 * If we have a minimum free, clear IFF_DRV_OACTIVE
3712 * to tell the stack that it is OK to send packets.
3713 */
3714 if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) {
3715 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3716 /* All clean, turn off the watchdog */
3717 if (txr->tx_avail == adapter->num_tx_desc) {
3718 txr->queue_status = IGB_QUEUE_IDLE;
3719 return (FALSE);
3720 }
3721 }
3722 return (TRUE);
3723 }
3724
3725 /*********************************************************************
3726 *
3727 * Refresh mbuf buffers for RX descriptor rings
3728 * - now keeps its own state so discards due to resource
3729 * exhaustion are unnecessary, if an mbuf cannot be obtained
3730 * it just returns, keeping its placeholder, thus it can simply
3731 * be recalled to try again.
3732 *
3733 **********************************************************************/
3734 static void
3735 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
3736 {
3737 struct adapter *adapter = rxr->adapter;
3738 bus_dma_segment_t hseg[1];
3739 bus_dma_segment_t pseg[1];
3740 struct igb_rx_buf *rxbuf;
3741 struct mbuf *mh, *mp;
3742 int i, j, nsegs, error;
3743 bool refreshed = FALSE;
3744
3745 i = j = rxr->next_to_refresh;
3746 /*
3747 ** Get one descriptor beyond
3748 ** our work mark to control
3749 ** the loop.
3750 */
3751 if (++j == adapter->num_rx_desc)
3752 j = 0;
3753
3754 while (j != limit) {
3755 rxbuf = &rxr->rx_buffers[i];
3756 /* No hdr mbuf used with header split off */
3757 if (rxr->hdr_split == FALSE)
3758 goto no_split;
3759 if (rxbuf->m_head == NULL) {
3760 mh = m_gethdr(M_DONTWAIT, MT_DATA);
3761 if (mh == NULL)
3762 goto update;
3763 } else
3764 mh = rxbuf->m_head;
3765
3766 mh->m_pkthdr.len = mh->m_len = MHLEN;
3767 mh->m_len = MHLEN;
3768 mh->m_flags |= M_PKTHDR;
3769 /* Get the memory mapping */
3770 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3771 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
3772 if (error != 0) {
3773 printf("Refresh mbufs: hdr dmamap load"
3774 " failure - %d\n", error);
3775 m_free(mh);
3776 rxbuf->m_head = NULL;
3777 goto update;
3778 }
3779 rxbuf->m_head = mh;
3780 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3781 BUS_DMASYNC_PREREAD);
3782 rxr->rx_base[i].read.hdr_addr =
3783 htole64(hseg[0].ds_addr);
3784 no_split:
3785 if (rxbuf->m_pack == NULL) {
3786 mp = m_getjcl(M_DONTWAIT, MT_DATA,
3787 M_PKTHDR, adapter->rx_mbuf_sz);
3788 if (mp == NULL)
3789 goto update;
3790 } else
3791 mp = rxbuf->m_pack;
3792
3793 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3794 /* Get the memory mapping */
3795 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3796 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
3797 if (error != 0) {
3798 printf("Refresh mbufs: payload dmamap load"
3799 " failure - %d\n", error);
3800 m_free(mp);
3801 rxbuf->m_pack = NULL;
3802 goto update;
3803 }
3804 rxbuf->m_pack = mp;
3805 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3806 BUS_DMASYNC_PREREAD);
3807 rxr->rx_base[i].read.pkt_addr =
3808 htole64(pseg[0].ds_addr);
3809 refreshed = TRUE; /* I feel wefreshed :) */
3810
3811 i = j; /* our next is precalculated */
3812 rxr->next_to_refresh = i;
3813 if (++j == adapter->num_rx_desc)
3814 j = 0;
3815 }
3816 update:
3817 if (refreshed) /* update tail */
3818 E1000_WRITE_REG(&adapter->hw,
3819 E1000_RDT(rxr->me), rxr->next_to_refresh);
3820 return;
3821 }
3822
3823
3824 /*********************************************************************
3825 *
3826 * Allocate memory for rx_buffer structures. Since we use one
3827 * rx_buffer per received packet, the maximum number of rx_buffer's
3828 * that we'll need is equal to the number of receive descriptors
3829 * that we've allocated.
3830 *
3831 **********************************************************************/
3832 static int
3833 igb_allocate_receive_buffers(struct rx_ring *rxr)
3834 {
3835 struct adapter *adapter = rxr->adapter;
3836 device_t dev = adapter->dev;
3837 struct igb_rx_buf *rxbuf;
3838 int i, bsize, error;
3839
3840 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
3841 if (!(rxr->rx_buffers =
3842 (struct igb_rx_buf *) malloc(bsize,
3843 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3844 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3845 error = ENOMEM;
3846 goto fail;
3847 }
3848
3849 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3850 1, 0, /* alignment, bounds */
3851 BUS_SPACE_MAXADDR, /* lowaddr */
3852 BUS_SPACE_MAXADDR, /* highaddr */
3853 NULL, NULL, /* filter, filterarg */
3854 MSIZE, /* maxsize */
3855 1, /* nsegments */
3856 MSIZE, /* maxsegsize */
3857 0, /* flags */
3858 NULL, /* lockfunc */
3859 NULL, /* lockfuncarg */
3860 &rxr->htag))) {
3861 device_printf(dev, "Unable to create RX DMA tag\n");
3862 goto fail;
3863 }
3864
3865 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3866 1, 0, /* alignment, bounds */
3867 BUS_SPACE_MAXADDR, /* lowaddr */
3868 BUS_SPACE_MAXADDR, /* highaddr */
3869 NULL, NULL, /* filter, filterarg */
3870 MJUM9BYTES, /* maxsize */
3871 1, /* nsegments */
3872 MJUM9BYTES, /* maxsegsize */
3873 0, /* flags */
3874 NULL, /* lockfunc */
3875 NULL, /* lockfuncarg */
3876 &rxr->ptag))) {
3877 device_printf(dev, "Unable to create RX payload DMA tag\n");
3878 goto fail;
3879 }
3880
3881 for (i = 0; i < adapter->num_rx_desc; i++) {
3882 rxbuf = &rxr->rx_buffers[i];
3883 error = bus_dmamap_create(rxr->htag,
3884 BUS_DMA_NOWAIT, &rxbuf->hmap);
3885 if (error) {
3886 device_printf(dev,
3887 "Unable to create RX head DMA maps\n");
3888 goto fail;
3889 }
3890 error = bus_dmamap_create(rxr->ptag,
3891 BUS_DMA_NOWAIT, &rxbuf->pmap);
3892 if (error) {
3893 device_printf(dev,
3894 "Unable to create RX packet DMA maps\n");
3895 goto fail;
3896 }
3897 }
3898
3899 return (0);
3900
3901 fail:
3902 /* Frees all, but can handle partial completion */
3903 igb_free_receive_structures(adapter);
3904 return (error);
3905 }
3906
3907
3908 static void
3909 igb_free_receive_ring(struct rx_ring *rxr)
3910 {
3911 struct adapter *adapter = rxr->adapter;
3912 struct igb_rx_buf *rxbuf;
3913
3914
3915 for (int i = 0; i < adapter->num_rx_desc; i++) {
3916 rxbuf = &rxr->rx_buffers[i];
3917 if (rxbuf->m_head != NULL) {
3918 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3919 BUS_DMASYNC_POSTREAD);
3920 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3921 rxbuf->m_head->m_flags |= M_PKTHDR;
3922 m_freem(rxbuf->m_head);
3923 }
3924 if (rxbuf->m_pack != NULL) {
3925 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3926 BUS_DMASYNC_POSTREAD);
3927 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3928 rxbuf->m_pack->m_flags |= M_PKTHDR;
3929 m_freem(rxbuf->m_pack);
3930 }
3931 rxbuf->m_head = NULL;
3932 rxbuf->m_pack = NULL;
3933 }
3934 }
3935
3936
3937 /*********************************************************************
3938 *
3939 * Initialize a receive ring and its buffers.
3940 *
3941 **********************************************************************/
3942 static int
3943 igb_setup_receive_ring(struct rx_ring *rxr)
3944 {
3945 struct adapter *adapter;
3946 struct ifnet *ifp;
3947 device_t dev;
3948 struct igb_rx_buf *rxbuf;
3949 bus_dma_segment_t pseg[1], hseg[1];
3950 struct lro_ctrl *lro = &rxr->lro;
3951 int rsize, nsegs, error = 0;
3952
3953 adapter = rxr->adapter;
3954 dev = adapter->dev;
3955 ifp = adapter->ifp;
3956
3957 /* Clear the ring contents */
3958 IGB_RX_LOCK(rxr);
3959 rsize = roundup2(adapter->num_rx_desc *
3960 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3961 bzero((void *)rxr->rx_base, rsize);
3962
3963 /*
3964 ** Free current RX buffer structures and their mbufs
3965 */
3966 igb_free_receive_ring(rxr);
3967
3968 /* Configure for header split? */
3969 if (igb_header_split)
3970 rxr->hdr_split = TRUE;
3971
3972 /* Now replenish the ring mbufs */
3973 for (int j = 0; j < adapter->num_rx_desc; ++j) {
3974 struct mbuf *mh, *mp;
3975
3976 rxbuf = &rxr->rx_buffers[j];
3977 if (rxr->hdr_split == FALSE)
3978 goto skip_head;
3979
3980 /* First the header */
3981 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA);
3982 if (rxbuf->m_head == NULL) {
3983 error = ENOBUFS;
3984 goto fail;
3985 }
3986 m_adj(rxbuf->m_head, ETHER_ALIGN);
3987 mh = rxbuf->m_head;
3988 mh->m_len = mh->m_pkthdr.len = MHLEN;
3989 mh->m_flags |= M_PKTHDR;
3990 /* Get the memory mapping */
3991 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3992 rxbuf->hmap, rxbuf->m_head, hseg,
3993 &nsegs, BUS_DMA_NOWAIT);
3994 if (error != 0) /* Nothing elegant to do here */
3995 goto fail;
3996 bus_dmamap_sync(rxr->htag,
3997 rxbuf->hmap, BUS_DMASYNC_PREREAD);
3998 /* Update descriptor */
3999 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4000
4001 skip_head:
4002 /* Now the payload cluster */
4003 rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA,
4004 M_PKTHDR, adapter->rx_mbuf_sz);
4005 if (rxbuf->m_pack == NULL) {
4006 error = ENOBUFS;
4007 goto fail;
4008 }
4009 mp = rxbuf->m_pack;
4010 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4011 /* Get the memory mapping */
4012 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4013 rxbuf->pmap, mp, pseg,
4014 &nsegs, BUS_DMA_NOWAIT);
4015 if (error != 0)
4016 goto fail;
4017 bus_dmamap_sync(rxr->ptag,
4018 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4019 /* Update descriptor */
4020 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4021 }
4022
4023 /* Setup our descriptor indices */
4024 rxr->next_to_check = 0;
4025 rxr->next_to_refresh = adapter->num_rx_desc - 1;
4026 rxr->lro_enabled = FALSE;
4027 rxr->rx_split_packets = 0;
4028 rxr->rx_bytes = 0;
4029
4030 rxr->fmp = NULL;
4031 rxr->lmp = NULL;
4032 rxr->discard = FALSE;
4033
4034 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4035 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4036
4037 /*
4038 ** Now set up the LRO interface, we
4039 ** also only do head split when LRO
4040 ** is enabled, since so often they
4041 ** are undesireable in similar setups.
4042 */
4043 if (ifp->if_capenable & IFCAP_LRO) {
4044 error = tcp_lro_init(lro);
4045 if (error) {
4046 device_printf(dev, "LRO Initialization failed!\n");
4047 goto fail;
4048 }
4049 INIT_DEBUGOUT("RX LRO Initialized\n");
4050 rxr->lro_enabled = TRUE;
4051 lro->ifp = adapter->ifp;
4052 }
4053
4054 IGB_RX_UNLOCK(rxr);
4055 return (0);
4056
4057 fail:
4058 igb_free_receive_ring(rxr);
4059 IGB_RX_UNLOCK(rxr);
4060 return (error);
4061 }
4062
4063
4064 /*********************************************************************
4065 *
4066 * Initialize all receive rings.
4067 *
4068 **********************************************************************/
4069 static int
4070 igb_setup_receive_structures(struct adapter *adapter)
4071 {
4072 struct rx_ring *rxr = adapter->rx_rings;
4073 int i;
4074
4075 for (i = 0; i < adapter->num_queues; i++, rxr++)
4076 if (igb_setup_receive_ring(rxr))
4077 goto fail;
4078
4079 return (0);
4080 fail:
4081 /*
4082 * Free RX buffers allocated so far, we will only handle
4083 * the rings that completed, the failing case will have
4084 * cleaned up for itself. 'i' is the endpoint.
4085 */
4086 for (int j = 0; j > i; ++j) {
4087 rxr = &adapter->rx_rings[i];
4088 IGB_RX_LOCK(rxr);
4089 igb_free_receive_ring(rxr);
4090 IGB_RX_UNLOCK(rxr);
4091 }
4092
4093 return (ENOBUFS);
4094 }
4095
4096 /*********************************************************************
4097 *
4098 * Enable receive unit.
4099 *
4100 **********************************************************************/
4101 static void
4102 igb_initialize_receive_units(struct adapter *adapter)
4103 {
4104 struct rx_ring *rxr = adapter->rx_rings;
4105 struct ifnet *ifp = adapter->ifp;
4106 struct e1000_hw *hw = &adapter->hw;
4107 u32 rctl, rxcsum, psize, srrctl = 0;
4108
4109 INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
4110
4111 /*
4112 * Make sure receives are disabled while setting
4113 * up the descriptor ring
4114 */
4115 rctl = E1000_READ_REG(hw, E1000_RCTL);
4116 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4117
4118 /*
4119 ** Set up for header split
4120 */
4121 if (igb_header_split) {
4122 /* Use a standard mbuf for the header */
4123 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4124 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4125 } else
4126 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4127
4128 /*
4129 ** Set up for jumbo frames
4130 */
4131 if (ifp->if_mtu > ETHERMTU) {
4132 rctl |= E1000_RCTL_LPE;
4133 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
4134 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4135 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4136 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
4137 srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4138 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4139 }
4140 /* Set maximum packet len */
4141 psize = adapter->max_frame_size;
4142 /* are we on a vlan? */
4143 if (adapter->ifp->if_vlantrunk != NULL)
4144 psize += VLAN_TAG_SIZE;
4145 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
4146 } else {
4147 rctl &= ~E1000_RCTL_LPE;
4148 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4149 rctl |= E1000_RCTL_SZ_2048;
4150 }
4151
4152 /* Setup the Base and Length of the Rx Descriptor Rings */
4153 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4154 u64 bus_addr = rxr->rxdma.dma_paddr;
4155 u32 rxdctl;
4156
4157 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4158 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4159 E1000_WRITE_REG(hw, E1000_RDBAH(i),
4160 (uint32_t)(bus_addr >> 32));
4161 E1000_WRITE_REG(hw, E1000_RDBAL(i),
4162 (uint32_t)bus_addr);
4163 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
4164 /* Enable this Queue */
4165 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
4166 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4167 rxdctl &= 0xFFF00000;
4168 rxdctl |= IGB_RX_PTHRESH;
4169 rxdctl |= IGB_RX_HTHRESH << 8;
4170 rxdctl |= IGB_RX_WTHRESH << 16;
4171 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
4172 }
4173
4174 /*
4175 ** Setup for RX MultiQueue
4176 */
4177 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4178 if (adapter->num_queues >1) {
4179 u32 random[10], mrqc, shift = 0;
4180 union igb_reta {
4181 u32 dword;
4182 u8 bytes[4];
4183 } reta;
4184
4185 arc4rand(&random, sizeof(random), 0);
4186 if (adapter->hw.mac.type == e1000_82575)
4187 shift = 6;
4188 /* Warning FM follows */
4189 for (int i = 0; i < 128; i++) {
4190 reta.bytes[i & 3] =
4191 (i % adapter->num_queues) << shift;
4192 if ((i & 3) == 3)
4193 E1000_WRITE_REG(hw,
4194 E1000_RETA(i >> 2), reta.dword);
4195 }
4196 /* Now fill in hash table */
4197 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
4198 for (int i = 0; i < 10; i++)
4199 E1000_WRITE_REG_ARRAY(hw,
4200 E1000_RSSRK(0), i, random[i]);
4201
4202 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
4203 E1000_MRQC_RSS_FIELD_IPV4_TCP);
4204 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
4205 E1000_MRQC_RSS_FIELD_IPV6_TCP);
4206 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
4207 E1000_MRQC_RSS_FIELD_IPV6_UDP);
4208 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
4209 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
4210
4211 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
4212
4213 /*
4214 ** NOTE: Receive Full-Packet Checksum Offload
4215 ** is mutually exclusive with Multiqueue. However
4216 ** this is not the same as TCP/IP checksums which
4217 ** still work.
4218 */
4219 rxcsum |= E1000_RXCSUM_PCSD;
4220 #if __FreeBSD_version >= 800000
4221 /* For SCTP Offload */
4222 if ((hw->mac.type == e1000_82576)
4223 && (ifp->if_capenable & IFCAP_RXCSUM))
4224 rxcsum |= E1000_RXCSUM_CRCOFL;
4225 #endif
4226 } else {
4227 /* Non RSS setup */
4228 if (ifp->if_capenable & IFCAP_RXCSUM) {
4229 rxcsum |= E1000_RXCSUM_IPPCSE;
4230 #if __FreeBSD_version >= 800000
4231 if (adapter->hw.mac.type == e1000_82576)
4232 rxcsum |= E1000_RXCSUM_CRCOFL;
4233 #endif
4234 } else
4235 rxcsum &= ~E1000_RXCSUM_TUOFL;
4236 }
4237 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4238
4239 /* Setup the Receive Control Register */
4240 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4241 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4242 E1000_RCTL_RDMTS_HALF |
4243 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4244 /* Strip CRC bytes. */
4245 rctl |= E1000_RCTL_SECRC;
4246 /* Make sure VLAN Filters are off */
4247 rctl &= ~E1000_RCTL_VFE;
4248 /* Don't store bad packets */
4249 rctl &= ~E1000_RCTL_SBP;
4250
4251 /* Enable Receives */
4252 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4253
4254 /*
4255 * Setup the HW Rx Head and Tail Descriptor Pointers
4256 * - needs to be after enable
4257 */
4258 for (int i = 0; i < adapter->num_queues; i++) {
4259 rxr = &adapter->rx_rings[i];
4260 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
4261 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
4262 }
4263 return;
4264 }
4265
4266 /*********************************************************************
4267 *
4268 * Free receive rings.
4269 *
4270 **********************************************************************/
4271 static void
4272 igb_free_receive_structures(struct adapter *adapter)
4273 {
4274 struct rx_ring *rxr = adapter->rx_rings;
4275
4276 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4277 struct lro_ctrl *lro = &rxr->lro;
4278 igb_free_receive_buffers(rxr);
4279 tcp_lro_free(lro);
4280 igb_dma_free(adapter, &rxr->rxdma);
4281 }
4282
4283 free(adapter->rx_rings, M_DEVBUF);
4284 }
4285
4286 /*********************************************************************
4287 *
4288 * Free receive ring data structures.
4289 *
4290 **********************************************************************/
4291 static void
4292 igb_free_receive_buffers(struct rx_ring *rxr)
4293 {
4294 struct adapter *adapter = rxr->adapter;
4295 struct igb_rx_buf *rxbuf;
4296 int i;
4297
4298 INIT_DEBUGOUT("free_receive_structures: begin");
4299
4300 /* Cleanup any existing buffers */
4301 if (rxr->rx_buffers != NULL) {
4302 for (i = 0; i < adapter->num_rx_desc; i++) {
4303 rxbuf = &rxr->rx_buffers[i];
4304 if (rxbuf->m_head != NULL) {
4305 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4306 BUS_DMASYNC_POSTREAD);
4307 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4308 rxbuf->m_head->m_flags |= M_PKTHDR;
4309 m_freem(rxbuf->m_head);
4310 }
4311 if (rxbuf->m_pack != NULL) {
4312 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4313 BUS_DMASYNC_POSTREAD);
4314 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4315 rxbuf->m_pack->m_flags |= M_PKTHDR;
4316 m_freem(rxbuf->m_pack);
4317 }
4318 rxbuf->m_head = NULL;
4319 rxbuf->m_pack = NULL;
4320 if (rxbuf->hmap != NULL) {
4321 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4322 rxbuf->hmap = NULL;
4323 }
4324 if (rxbuf->pmap != NULL) {
4325 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4326 rxbuf->pmap = NULL;
4327 }
4328 }
4329 if (rxr->rx_buffers != NULL) {
4330 free(rxr->rx_buffers, M_DEVBUF);
4331 rxr->rx_buffers = NULL;
4332 }
4333 }
4334
4335 if (rxr->htag != NULL) {
4336 bus_dma_tag_destroy(rxr->htag);
4337 rxr->htag = NULL;
4338 }
4339 if (rxr->ptag != NULL) {
4340 bus_dma_tag_destroy(rxr->ptag);
4341 rxr->ptag = NULL;
4342 }
4343 }
4344
4345 static __inline void
4346 igb_rx_discard(struct rx_ring *rxr, int i)
4347 {
4348 struct igb_rx_buf *rbuf;
4349
4350 rbuf = &rxr->rx_buffers[i];
4351
4352 /* Partially received? Free the chain */
4353 if (rxr->fmp != NULL) {
4354 rxr->fmp->m_flags |= M_PKTHDR;
4355 m_freem(rxr->fmp);
4356 rxr->fmp = NULL;
4357 rxr->lmp = NULL;
4358 }
4359
4360 /*
4361 ** With advanced descriptors the writeback
4362 ** clobbers the buffer addrs, so its easier
4363 ** to just free the existing mbufs and take
4364 ** the normal refresh path to get new buffers
4365 ** and mapping.
4366 */
4367 if (rbuf->m_head) {
4368 m_free(rbuf->m_head);
4369 rbuf->m_head = NULL;
4370 }
4371
4372 if (rbuf->m_pack) {
4373 m_free(rbuf->m_pack);
4374 rbuf->m_pack = NULL;
4375 }
4376
4377 return;
4378 }
4379
4380 static __inline void
4381 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4382 {
4383
4384 /*
4385 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4386 * should be computed by hardware. Also it should not have VLAN tag in
4387 * ethernet header.
4388 */
4389 if (rxr->lro_enabled &&
4390 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4391 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4392 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4393 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4394 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4395 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4396 /*
4397 * Send to the stack if:
4398 ** - LRO not enabled, or
4399 ** - no LRO resources, or
4400 ** - lro enqueue fails
4401 */
4402 if (rxr->lro.lro_cnt != 0)
4403 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4404 return;
4405 }
4406 IGB_RX_UNLOCK(rxr);
4407 (*ifp->if_input)(ifp, m);
4408 IGB_RX_LOCK(rxr);
4409 }
4410
4411 /*********************************************************************
4412 *
4413 * This routine executes in interrupt context. It replenishes
4414 * the mbufs in the descriptor and sends data which has been
4415 * dma'ed into host memory to upper layer.
4416 *
4417 * We loop at most count times if count is > 0, or until done if
4418 * count < 0.
4419 *
4420 * Return TRUE if more to clean, FALSE otherwise
4421 *********************************************************************/
4422 static bool
4423 igb_rxeof(struct igb_queue *que, int count, int *done)
4424 {
4425 struct adapter *adapter = que->adapter;
4426 struct rx_ring *rxr = que->rxr;
4427 struct ifnet *ifp = adapter->ifp;
4428 struct lro_ctrl *lro = &rxr->lro;
4429 struct lro_entry *queued;
4430 int i, processed = 0, rxdone = 0;
4431 u32 ptype, staterr = 0;
4432 union e1000_adv_rx_desc *cur;
4433
4434 IGB_RX_LOCK(rxr);
4435 /* Sync the ring. */
4436 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4437 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4438
4439 /* Main clean loop */
4440 for (i = rxr->next_to_check; count != 0;) {
4441 struct mbuf *sendmp, *mh, *mp;
4442 struct igb_rx_buf *rxbuf;
4443 u16 hlen, plen, hdr, vtag;
4444 bool eop = FALSE;
4445
4446 cur = &rxr->rx_base[i];
4447 staterr = le32toh(cur->wb.upper.status_error);
4448 if ((staterr & E1000_RXD_STAT_DD) == 0)
4449 break;
4450 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4451 break;
4452 count--;
4453 sendmp = mh = mp = NULL;
4454 cur->wb.upper.status_error = 0;
4455 rxbuf = &rxr->rx_buffers[i];
4456 plen = le16toh(cur->wb.upper.length);
4457 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
4458 if ((adapter->hw.mac.type == e1000_i350) &&
4459 (staterr & E1000_RXDEXT_STATERR_LB))
4460 vtag = be16toh(cur->wb.upper.vlan);
4461 else
4462 vtag = le16toh(cur->wb.upper.vlan);
4463 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4464 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
4465
4466 /* Make sure all segments of a bad packet are discarded */
4467 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) ||
4468 (rxr->discard)) {
4469 ifp->if_ierrors++;
4470 ++rxr->rx_discarded;
4471 if (!eop) /* Catch subsequent segs */
4472 rxr->discard = TRUE;
4473 else
4474 rxr->discard = FALSE;
4475 igb_rx_discard(rxr, i);
4476 goto next_desc;
4477 }
4478
4479 /*
4480 ** The way the hardware is configured to
4481 ** split, it will ONLY use the header buffer
4482 ** when header split is enabled, otherwise we
4483 ** get normal behavior, ie, both header and
4484 ** payload are DMA'd into the payload buffer.
4485 **
4486 ** The fmp test is to catch the case where a
4487 ** packet spans multiple descriptors, in that
4488 ** case only the first header is valid.
4489 */
4490 if (rxr->hdr_split && rxr->fmp == NULL) {
4491 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4492 E1000_RXDADV_HDRBUFLEN_SHIFT;
4493 if (hlen > IGB_HDR_BUF)
4494 hlen = IGB_HDR_BUF;
4495 mh = rxr->rx_buffers[i].m_head;
4496 mh->m_len = hlen;
4497 /* clear buf pointer for refresh */
4498 rxbuf->m_head = NULL;
4499 /*
4500 ** Get the payload length, this
4501 ** could be zero if its a small
4502 ** packet.
4503 */
4504 if (plen > 0) {
4505 mp = rxr->rx_buffers[i].m_pack;
4506 mp->m_len = plen;
4507 mh->m_next = mp;
4508 /* clear buf pointer */
4509 rxbuf->m_pack = NULL;
4510 rxr->rx_split_packets++;
4511 }
4512 } else {
4513 /*
4514 ** Either no header split, or a
4515 ** secondary piece of a fragmented
4516 ** split packet.
4517 */
4518 mh = rxr->rx_buffers[i].m_pack;
4519 mh->m_len = plen;
4520 /* clear buf info for refresh */
4521 rxbuf->m_pack = NULL;
4522 }
4523
4524 ++processed; /* So we know when to refresh */
4525
4526 /* Initial frame - setup */
4527 if (rxr->fmp == NULL) {
4528 mh->m_pkthdr.len = mh->m_len;
4529 /* Save the head of the chain */
4530 rxr->fmp = mh;
4531 rxr->lmp = mh;
4532 if (mp != NULL) {
4533 /* Add payload if split */
4534 mh->m_pkthdr.len += mp->m_len;
4535 rxr->lmp = mh->m_next;
4536 }
4537 } else {
4538 /* Chain mbuf's together */
4539 rxr->lmp->m_next = mh;
4540 rxr->lmp = rxr->lmp->m_next;
4541 rxr->fmp->m_pkthdr.len += mh->m_len;
4542 }
4543
4544 if (eop) {
4545 rxr->fmp->m_pkthdr.rcvif = ifp;
4546 ifp->if_ipackets++;
4547 rxr->rx_packets++;
4548 /* capture data for AIM */
4549 rxr->packets++;
4550 rxr->bytes += rxr->fmp->m_pkthdr.len;
4551 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4552
4553 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4554 igb_rx_checksum(staterr, rxr->fmp, ptype);
4555
4556 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4557 (staterr & E1000_RXD_STAT_VP) != 0) {
4558 rxr->fmp->m_pkthdr.ether_vtag = vtag;
4559 rxr->fmp->m_flags |= M_VLANTAG;
4560 }
4561 #if __FreeBSD_version >= 800000
4562 rxr->fmp->m_pkthdr.flowid = que->msix;
4563 rxr->fmp->m_flags |= M_FLOWID;
4564 #endif
4565 sendmp = rxr->fmp;
4566 /* Make sure to set M_PKTHDR. */
4567 sendmp->m_flags |= M_PKTHDR;
4568 rxr->fmp = NULL;
4569 rxr->lmp = NULL;
4570 }
4571
4572 next_desc:
4573 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4574 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4575
4576 /* Advance our pointers to the next descriptor. */
4577 if (++i == adapter->num_rx_desc)
4578 i = 0;
4579 /*
4580 ** Send to the stack or LRO
4581 */
4582 if (sendmp != NULL) {
4583 rxr->next_to_check = i;
4584 igb_rx_input(rxr, ifp, sendmp, ptype);
4585 i = rxr->next_to_check;
4586 rxdone++;
4587 }
4588
4589 /* Every 8 descriptors we go to refresh mbufs */
4590 if (processed == 8) {
4591 igb_refresh_mbufs(rxr, i);
4592 processed = 0;
4593 }
4594 }
4595
4596 /* Catch any remainders */
4597 if (igb_rx_unrefreshed(rxr))
4598 igb_refresh_mbufs(rxr, i);
4599
4600 rxr->next_to_check = i;
4601
4602 /*
4603 * Flush any outstanding LRO work
4604 */
4605 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4606 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4607 tcp_lro_flush(lro, queued);
4608 }
4609
4610 if (done != NULL)
4611 *done = rxdone;
4612
4613 IGB_RX_UNLOCK(rxr);
4614 return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4615 }
4616
4617 /*********************************************************************
4618 *
4619 * Verify that the hardware indicated that the checksum is valid.
4620 * Inform the stack about the status of checksum so that stack
4621 * doesn't spend time verifying the checksum.
4622 *
4623 *********************************************************************/
4624 static void
4625 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
4626 {
4627 u16 status = (u16)staterr;
4628 u8 errors = (u8) (staterr >> 24);
4629 int sctp;
4630
4631 /* Ignore Checksum bit is set */
4632 if (status & E1000_RXD_STAT_IXSM) {
4633 mp->m_pkthdr.csum_flags = 0;
4634 return;
4635 }
4636
4637 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4638 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
4639 sctp = 1;
4640 else
4641 sctp = 0;
4642 if (status & E1000_RXD_STAT_IPCS) {
4643 /* Did it pass? */
4644 if (!(errors & E1000_RXD_ERR_IPE)) {
4645 /* IP Checksum Good */
4646 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4647 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4648 } else
4649 mp->m_pkthdr.csum_flags = 0;
4650 }
4651
4652 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
4653 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4654 #if __FreeBSD_version >= 800000
4655 if (sctp) /* reassign */
4656 type = CSUM_SCTP_VALID;
4657 #endif
4658 /* Did it pass? */
4659 if (!(errors & E1000_RXD_ERR_TCPE)) {
4660 mp->m_pkthdr.csum_flags |= type;
4661 if (sctp == 0)
4662 mp->m_pkthdr.csum_data = htons(0xffff);
4663 }
4664 }
4665 return;
4666 }
4667
4668 /*
4669 * This routine is run via an vlan
4670 * config EVENT
4671 */
4672 static void
4673 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4674 {
4675 struct adapter *adapter = ifp->if_softc;
4676 u32 index, bit;
4677
4678 if (ifp->if_softc != arg) /* Not our event */
4679 return;
4680
4681 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4682 return;
4683
4684 IGB_CORE_LOCK(adapter);
4685 index = (vtag >> 5) & 0x7F;
4686 bit = vtag & 0x1F;
4687 adapter->shadow_vfta[index] |= (1 << bit);
4688 ++adapter->num_vlans;
4689 /* Change hw filter setting */
4690 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4691 igb_setup_vlan_hw_support(adapter);
4692 IGB_CORE_UNLOCK(adapter);
4693 }
4694
4695 /*
4696 * This routine is run via an vlan
4697 * unconfig EVENT
4698 */
4699 static void
4700 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4701 {
4702 struct adapter *adapter = ifp->if_softc;
4703 u32 index, bit;
4704
4705 if (ifp->if_softc != arg)
4706 return;
4707
4708 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4709 return;
4710
4711 IGB_CORE_LOCK(adapter);
4712 index = (vtag >> 5) & 0x7F;
4713 bit = vtag & 0x1F;
4714 adapter->shadow_vfta[index] &= ~(1 << bit);
4715 --adapter->num_vlans;
4716 /* Change hw filter setting */
4717 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4718 igb_setup_vlan_hw_support(adapter);
4719 IGB_CORE_UNLOCK(adapter);
4720 }
4721
4722 static void
4723 igb_setup_vlan_hw_support(struct adapter *adapter)
4724 {
4725 struct e1000_hw *hw = &adapter->hw;
4726 struct ifnet *ifp = adapter->ifp;
4727 u32 reg;
4728
4729 if (adapter->vf_ifp) {
4730 e1000_rlpml_set_vf(hw,
4731 adapter->max_frame_size + VLAN_TAG_SIZE);
4732 return;
4733 }
4734
4735 reg = E1000_READ_REG(hw, E1000_CTRL);
4736 reg |= E1000_CTRL_VME;
4737 E1000_WRITE_REG(hw, E1000_CTRL, reg);
4738
4739 /* Enable the Filter Table */
4740 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4741 reg = E1000_READ_REG(hw, E1000_RCTL);
4742 reg &= ~E1000_RCTL_CFIEN;
4743 reg |= E1000_RCTL_VFE;
4744 E1000_WRITE_REG(hw, E1000_RCTL, reg);
4745 }
4746
4747 /* Update the frame size */
4748 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4749 adapter->max_frame_size + VLAN_TAG_SIZE);
4750
4751 /* Don't bother with table if no vlans */
4752 if ((adapter->num_vlans == 0) ||
4753 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
4754 return;
4755 /*
4756 ** A soft reset zero's out the VFTA, so
4757 ** we need to repopulate it now.
4758 */
4759 for (int i = 0; i < IGB_VFTA_SIZE; i++)
4760 if (adapter->shadow_vfta[i] != 0) {
4761 if (adapter->vf_ifp)
4762 e1000_vfta_set_vf(hw,
4763 adapter->shadow_vfta[i], TRUE);
4764 else
4765 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4766 i, adapter->shadow_vfta[i]);
4767 }
4768 }
4769
4770 static void
4771 igb_enable_intr(struct adapter *adapter)
4772 {
4773 /* With RSS set up what to auto clear */
4774 if (adapter->msix_mem) {
4775 u32 mask = (adapter->que_mask | adapter->link_mask);
4776 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
4777 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
4778 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
4779 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4780 E1000_IMS_LSC);
4781 } else {
4782 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4783 IMS_ENABLE_MASK);
4784 }
4785 E1000_WRITE_FLUSH(&adapter->hw);
4786
4787 return;
4788 }
4789
4790 static void
4791 igb_disable_intr(struct adapter *adapter)
4792 {
4793 if (adapter->msix_mem) {
4794 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
4795 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
4796 }
4797 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
4798 E1000_WRITE_FLUSH(&adapter->hw);
4799 return;
4800 }
4801
4802 /*
4803 * Bit of a misnomer, what this really means is
4804 * to enable OS management of the system... aka
4805 * to disable special hardware management features
4806 */
4807 static void
4808 igb_init_manageability(struct adapter *adapter)
4809 {
4810 if (adapter->has_manage) {
4811 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4812 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4813
4814 /* disable hardware interception of ARP */
4815 manc &= ~(E1000_MANC_ARP_EN);
4816
4817 /* enable receiving management packets to the host */
4818 manc |= E1000_MANC_EN_MNG2HOST;
4819 manc2h |= 1 << 5; /* Mng Port 623 */
4820 manc2h |= 1 << 6; /* Mng Port 664 */
4821 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4822 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4823 }
4824 }
4825
4826 /*
4827 * Give control back to hardware management
4828 * controller if there is one.
4829 */
4830 static void
4831 igb_release_manageability(struct adapter *adapter)
4832 {
4833 if (adapter->has_manage) {
4834 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4835
4836 /* re-enable hardware interception of ARP */
4837 manc |= E1000_MANC_ARP_EN;
4838 manc &= ~E1000_MANC_EN_MNG2HOST;
4839
4840 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4841 }
4842 }
4843
4844 /*
4845 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
4846 * For ASF and Pass Through versions of f/w this means that
4847 * the driver is loaded.
4848 *
4849 */
4850 static void
4851 igb_get_hw_control(struct adapter *adapter)
4852 {
4853 u32 ctrl_ext;
4854
4855 if (adapter->vf_ifp)
4856 return;
4857
4858 /* Let firmware know the driver has taken over */
4859 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4860 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4861 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4862 }
4863
4864 /*
4865 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
4866 * For ASF and Pass Through versions of f/w this means that the
4867 * driver is no longer loaded.
4868 *
4869 */
4870 static void
4871 igb_release_hw_control(struct adapter *adapter)
4872 {
4873 u32 ctrl_ext;
4874
4875 if (adapter->vf_ifp)
4876 return;
4877
4878 /* Let firmware taken over control of h/w */
4879 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4880 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4881 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4882 }
4883
4884 static int
4885 igb_is_valid_ether_addr(uint8_t *addr)
4886 {
4887 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4888
4889 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4890 return (FALSE);
4891 }
4892
4893 return (TRUE);
4894 }
4895
4896
4897 /*
4898 * Enable PCI Wake On Lan capability
4899 */
4900 static void
4901 igb_enable_wakeup(device_t dev)
4902 {
4903 u16 cap, status;
4904 u8 id;
4905
4906 /* First find the capabilities pointer*/
4907 cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4908 /* Read the PM Capabilities */
4909 id = pci_read_config(dev, cap, 1);
4910 if (id != PCIY_PMG) /* Something wrong */
4911 return;
4912 /* OK, we have the power capabilities, so
4913 now get the status register */
4914 cap += PCIR_POWER_STATUS;
4915 status = pci_read_config(dev, cap, 2);
4916 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4917 pci_write_config(dev, cap, status, 2);
4918 return;
4919 }
4920
4921 static void
4922 igb_led_func(void *arg, int onoff)
4923 {
4924 struct adapter *adapter = arg;
4925
4926 IGB_CORE_LOCK(adapter);
4927 if (onoff) {
4928 e1000_setup_led(&adapter->hw);
4929 e1000_led_on(&adapter->hw);
4930 } else {
4931 e1000_led_off(&adapter->hw);
4932 e1000_cleanup_led(&adapter->hw);
4933 }
4934 IGB_CORE_UNLOCK(adapter);
4935 }
4936
4937 /**********************************************************************
4938 *
4939 * Update the board statistics counters.
4940 *
4941 **********************************************************************/
4942 static void
4943 igb_update_stats_counters(struct adapter *adapter)
4944 {
4945 struct ifnet *ifp;
4946 struct e1000_hw *hw = &adapter->hw;
4947 struct e1000_hw_stats *stats;
4948
4949 /*
4950 ** The virtual function adapter has only a
4951 ** small controlled set of stats, do only
4952 ** those and return.
4953 */
4954 if (adapter->vf_ifp) {
4955 igb_update_vf_stats_counters(adapter);
4956 return;
4957 }
4958
4959 stats = (struct e1000_hw_stats *)adapter->stats;
4960
4961 if(adapter->hw.phy.media_type == e1000_media_type_copper ||
4962 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
4963 stats->symerrs +=
4964 E1000_READ_REG(hw,E1000_SYMERRS);
4965 stats->sec += E1000_READ_REG(hw, E1000_SEC);
4966 }
4967
4968 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
4969 stats->mpc += E1000_READ_REG(hw, E1000_MPC);
4970 stats->scc += E1000_READ_REG(hw, E1000_SCC);
4971 stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
4972
4973 stats->mcc += E1000_READ_REG(hw, E1000_MCC);
4974 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
4975 stats->colc += E1000_READ_REG(hw, E1000_COLC);
4976 stats->dc += E1000_READ_REG(hw, E1000_DC);
4977 stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
4978 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
4979 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
4980 /*
4981 ** For watchdog management we need to know if we have been
4982 ** paused during the last interval, so capture that here.
4983 */
4984 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4985 stats->xoffrxc += adapter->pause_frames;
4986 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
4987 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
4988 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
4989 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
4990 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
4991 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
4992 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
4993 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
4994 stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
4995 stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
4996 stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
4997 stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
4998
4999 /* For the 64-bit byte counters the low dword must be read first. */
5000 /* Both registers clear on the read of the high dword */
5001
5002 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
5003 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
5004 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
5005 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
5006
5007 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
5008 stats->ruc += E1000_READ_REG(hw, E1000_RUC);
5009 stats->rfc += E1000_READ_REG(hw, E1000_RFC);
5010 stats->roc += E1000_READ_REG(hw, E1000_ROC);
5011 stats->rjc += E1000_READ_REG(hw, E1000_RJC);
5012
5013 stats->tor += E1000_READ_REG(hw, E1000_TORH);
5014 stats->tot += E1000_READ_REG(hw, E1000_TOTH);
5015
5016 stats->tpr += E1000_READ_REG(hw, E1000_TPR);
5017 stats->tpt += E1000_READ_REG(hw, E1000_TPT);
5018 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5019 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5020 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5021 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5022 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5023 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5024 stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
5025 stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
5026
5027 /* Interrupt Counts */
5028
5029 stats->iac += E1000_READ_REG(hw, E1000_IAC);
5030 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5031 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5032 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5033 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5034 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5035 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5036 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5037 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5038
5039 /* Host to Card Statistics */
5040
5041 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
5042 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
5043 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
5044 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
5045 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
5046 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
5047 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
5048 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
5049 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
5050 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
5051 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
5052 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
5053 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
5054 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
5055
5056 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5057 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5058 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5059 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
5060 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5061 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5062
5063 ifp = adapter->ifp;
5064 ifp->if_collisions = stats->colc;
5065
5066 /* Rx Errors */
5067 ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc +
5068 stats->crcerrs + stats->algnerrc +
5069 stats->ruc + stats->roc + stats->mpc + stats->cexterr;
5070
5071 /* Tx Errors */
5072 ifp->if_oerrors = stats->ecol +
5073 stats->latecol + adapter->watchdog_events;
5074
5075 /* Driver specific counters */
5076 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
5077 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
5078 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
5079 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
5080 adapter->packet_buf_alloc_tx =
5081 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
5082 adapter->packet_buf_alloc_rx =
5083 (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
5084 }
5085
5086
5087 /**********************************************************************
5088 *
5089 * Initialize the VF board statistics counters.
5090 *
5091 **********************************************************************/
5092 static void
5093 igb_vf_init_stats(struct adapter *adapter)
5094 {
5095 struct e1000_hw *hw = &adapter->hw;
5096 struct e1000_vf_stats *stats;
5097
5098 stats = (struct e1000_vf_stats *)adapter->stats;
5099 if (stats == NULL)
5100 return;
5101 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
5102 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
5103 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
5104 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
5105 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
5106 }
5107
5108 /**********************************************************************
5109 *
5110 * Update the VF board statistics counters.
5111 *
5112 **********************************************************************/
5113 static void
5114 igb_update_vf_stats_counters(struct adapter *adapter)
5115 {
5116 struct e1000_hw *hw = &adapter->hw;
5117 struct e1000_vf_stats *stats;
5118
5119 if (adapter->link_speed == 0)
5120 return;
5121
5122 stats = (struct e1000_vf_stats *)adapter->stats;
5123
5124 UPDATE_VF_REG(E1000_VFGPRC,
5125 stats->last_gprc, stats->gprc);
5126 UPDATE_VF_REG(E1000_VFGORC,
5127 stats->last_gorc, stats->gorc);
5128 UPDATE_VF_REG(E1000_VFGPTC,
5129 stats->last_gptc, stats->gptc);
5130 UPDATE_VF_REG(E1000_VFGOTC,
5131 stats->last_gotc, stats->gotc);
5132 UPDATE_VF_REG(E1000_VFMPRC,
5133 stats->last_mprc, stats->mprc);
5134 }
5135
5136 /* Export a single 32-bit register via a read-only sysctl. */
5137 static int
5138 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5139 {
5140 struct adapter *adapter;
5141 u_int val;
5142
5143 adapter = oidp->oid_arg1;
5144 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5145 return (sysctl_handle_int(oidp, &val, 0, req));
5146 }
5147
5148 /*
5149 ** Tuneable interrupt rate handler
5150 */
5151 static int
5152 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5153 {
5154 struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1);
5155 int error;
5156 u32 reg, usec, rate;
5157
5158 reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
5159 usec = ((reg & 0x7FFC) >> 2);
5160 if (usec > 0)
5161 rate = 1000000 / usec;
5162 else
5163 rate = 0;
5164 error = sysctl_handle_int(oidp, &rate, 0, req);
5165 if (error || !req->newptr)
5166 return error;
5167 return 0;
5168 }
5169
5170 /*
5171 * Add sysctl variables, one per statistic, to the system.
5172 */
5173 static void
5174 igb_add_hw_stats(struct adapter *adapter)
5175 {
5176 device_t dev = adapter->dev;
5177
5178 struct tx_ring *txr = adapter->tx_rings;
5179 struct rx_ring *rxr = adapter->rx_rings;
5180
5181 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5182 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5183 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5184 struct e1000_hw_stats *stats = adapter->stats;
5185
5186 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
5187 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
5188
5189 #define QUEUE_NAME_LEN 32
5190 char namebuf[QUEUE_NAME_LEN];
5191
5192 /* Driver Statistics */
5193 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "link_irq",
5194 CTLFLAG_RD, &adapter->link_irq, 0,
5195 "Link MSIX IRQ Handled");
5196 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5197 CTLFLAG_RD, &adapter->dropped_pkts,
5198 "Driver dropped packets");
5199 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5200 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5201 "Driver tx dma failure in xmit");
5202 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5203 CTLFLAG_RD, &adapter->rx_overruns,
5204 "RX overruns");
5205 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5206 CTLFLAG_RD, &adapter->watchdog_events,
5207 "Watchdog timeouts");
5208
5209 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
5210 CTLFLAG_RD, &adapter->device_control,
5211 "Device Control Register");
5212 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
5213 CTLFLAG_RD, &adapter->rx_control,
5214 "Receiver Control Register");
5215 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
5216 CTLFLAG_RD, &adapter->int_mask,
5217 "Interrupt Mask");
5218 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
5219 CTLFLAG_RD, &adapter->eint_mask,
5220 "Extended Interrupt Mask");
5221 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
5222 CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
5223 "Transmit Buffer Packet Allocation");
5224 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
5225 CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
5226 "Receive Buffer Packet Allocation");
5227 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5228 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5229 "Flow Control High Watermark");
5230 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5231 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5232 "Flow Control Low Watermark");
5233
5234 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5235 struct lro_ctrl *lro = &rxr->lro;
5236
5237 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5238 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5239 CTLFLAG_RD, NULL, "Queue Name");
5240 queue_list = SYSCTL_CHILDREN(queue_node);
5241
5242 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5243 CTLFLAG_RD, &adapter->queues[i],
5244 sizeof(&adapter->queues[i]),
5245 igb_sysctl_interrupt_rate_handler,
5246 "IU", "Interrupt Rate");
5247
5248 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5249 CTLFLAG_RD, adapter, E1000_TDH(txr->me),
5250 igb_sysctl_reg_handler, "IU",
5251 "Transmit Descriptor Head");
5252 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5253 CTLFLAG_RD, adapter, E1000_TDT(txr->me),
5254 igb_sysctl_reg_handler, "IU",
5255 "Transmit Descriptor Tail");
5256 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5257 CTLFLAG_RD, &txr->no_desc_avail,
5258 "Queue No Descriptor Available");
5259 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5260 CTLFLAG_RD, &txr->tx_packets,
5261 "Queue Packets Transmitted");
5262
5263 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5264 CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
5265 igb_sysctl_reg_handler, "IU",
5266 "Receive Descriptor Head");
5267 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5268 CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
5269 igb_sysctl_reg_handler, "IU",
5270 "Receive Descriptor Tail");
5271 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5272 CTLFLAG_RD, &rxr->rx_packets,
5273 "Queue Packets Received");
5274 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5275 CTLFLAG_RD, &rxr->rx_bytes,
5276 "Queue Bytes Received");
5277 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
5278 CTLFLAG_RD, &lro->lro_queued, 0,
5279 "LRO Queued");
5280 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
5281 CTLFLAG_RD, &lro->lro_flushed, 0,
5282 "LRO Flushed");
5283 }
5284
5285 /* MAC stats get their own sub node */
5286
5287 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5288 CTLFLAG_RD, NULL, "MAC Statistics");
5289 stat_list = SYSCTL_CHILDREN(stat_node);
5290
5291 /*
5292 ** VF adapter has a very limited set of stats
5293 ** since its not managing the metal, so to speak.
5294 */
5295 if (adapter->vf_ifp) {
5296 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5297 CTLFLAG_RD, &stats->gprc,
5298 "Good Packets Received");
5299 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5300 CTLFLAG_RD, &stats->gptc,
5301 "Good Packets Transmitted");
5302 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5303 CTLFLAG_RD, &stats->gorc,
5304 "Good Octets Received");
5305 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5306 CTLFLAG_RD, &stats->gotc,
5307 "Good Octets Transmitted");
5308 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5309 CTLFLAG_RD, &stats->mprc,
5310 "Multicast Packets Received");
5311 return;
5312 }
5313
5314 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5315 CTLFLAG_RD, &stats->ecol,
5316 "Excessive collisions");
5317 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
5318 CTLFLAG_RD, &stats->scc,
5319 "Single collisions");
5320 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5321 CTLFLAG_RD, &stats->mcc,
5322 "Multiple collisions");
5323 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
5324 CTLFLAG_RD, &stats->latecol,
5325 "Late collisions");
5326 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
5327 CTLFLAG_RD, &stats->colc,
5328 "Collision Count");
5329 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5330 CTLFLAG_RD, &stats->symerrs,
5331 "Symbol Errors");
5332 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5333 CTLFLAG_RD, &stats->sec,
5334 "Sequence Errors");
5335 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
5336 CTLFLAG_RD, &stats->dc,
5337 "Defer Count");
5338 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5339 CTLFLAG_RD, &stats->mpc,
5340 "Missed Packets");
5341 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5342 CTLFLAG_RD, &stats->rnbc,
5343 "Receive No Buffers");
5344 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5345 CTLFLAG_RD, &stats->ruc,
5346 "Receive Undersize");
5347 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5348 CTLFLAG_RD, &stats->rfc,
5349 "Fragmented Packets Received ");
5350 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5351 CTLFLAG_RD, &stats->roc,
5352 "Oversized Packets Received");
5353 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5354 CTLFLAG_RD, &stats->rjc,
5355 "Recevied Jabber");
5356 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5357 CTLFLAG_RD, &stats->rxerrc,
5358 "Receive Errors");
5359 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5360 CTLFLAG_RD, &stats->crcerrs,
5361 "CRC errors");
5362 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5363 CTLFLAG_RD, &stats->algnerrc,
5364 "Alignment Errors");
5365 /* On 82575 these are collision counts */
5366 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5367 CTLFLAG_RD, &stats->cexterr,
5368 "Collision/Carrier extension errors");
5369 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5370 CTLFLAG_RD, &stats->xonrxc,
5371 "XON Received");
5372 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5373 CTLFLAG_RD, &stats->xontxc,
5374 "XON Transmitted");
5375 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5376 CTLFLAG_RD, &stats->xoffrxc,
5377 "XOFF Received");
5378 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5379 CTLFLAG_RD, &stats->xofftxc,
5380 "XOFF Transmitted");
5381 /* Packet Reception Stats */
5382 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5383 CTLFLAG_RD, &stats->tpr,
5384 "Total Packets Received ");
5385 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5386 CTLFLAG_RD, &stats->gprc,
5387 "Good Packets Received");
5388 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5389 CTLFLAG_RD, &stats->bprc,
5390 "Broadcast Packets Received");
5391 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5392 CTLFLAG_RD, &stats->mprc,
5393 "Multicast Packets Received");
5394 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5395 CTLFLAG_RD, &stats->prc64,
5396 "64 byte frames received ");
5397 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5398 CTLFLAG_RD, &stats->prc127,
5399 "65-127 byte frames received");
5400 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5401 CTLFLAG_RD, &stats->prc255,
5402 "128-255 byte frames received");
5403 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5404 CTLFLAG_RD, &stats->prc511,
5405 "256-511 byte frames received");
5406 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5407 CTLFLAG_RD, &stats->prc1023,
5408 "512-1023 byte frames received");
5409 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5410 CTLFLAG_RD, &stats->prc1522,
5411 "1023-1522 byte frames received");
5412 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5413 CTLFLAG_RD, &stats->gorc,
5414 "Good Octets Received");
5415
5416 /* Packet Transmission Stats */
5417 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5418 CTLFLAG_RD, &stats->gotc,
5419 "Good Octets Transmitted");
5420 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5421 CTLFLAG_RD, &stats->tpt,
5422 "Total Packets Transmitted");
5423 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5424 CTLFLAG_RD, &stats->gptc,
5425 "Good Packets Transmitted");
5426 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5427 CTLFLAG_RD, &stats->bptc,
5428 "Broadcast Packets Transmitted");
5429 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5430 CTLFLAG_RD, &stats->mptc,
5431 "Multicast Packets Transmitted");
5432 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5433 CTLFLAG_RD, &stats->ptc64,
5434 "64 byte frames transmitted ");
5435 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5436 CTLFLAG_RD, &stats->ptc127,
5437 "65-127 byte frames transmitted");
5438 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5439 CTLFLAG_RD, &stats->ptc255,
5440 "128-255 byte frames transmitted");
5441 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5442 CTLFLAG_RD, &stats->ptc511,
5443 "256-511 byte frames transmitted");
5444 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5445 CTLFLAG_RD, &stats->ptc1023,
5446 "512-1023 byte frames transmitted");
5447 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5448 CTLFLAG_RD, &stats->ptc1522,
5449 "1024-1522 byte frames transmitted");
5450 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5451 CTLFLAG_RD, &stats->tsctc,
5452 "TSO Contexts Transmitted");
5453 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5454 CTLFLAG_RD, &stats->tsctfc,
5455 "TSO Contexts Failed");
5456
5457
5458 /* Interrupt Stats */
5459
5460 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5461 CTLFLAG_RD, NULL, "Interrupt Statistics");
5462 int_list = SYSCTL_CHILDREN(int_node);
5463
5464 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
5465 CTLFLAG_RD, &stats->iac,
5466 "Interrupt Assertion Count");
5467
5468 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5469 CTLFLAG_RD, &stats->icrxptc,
5470 "Interrupt Cause Rx Pkt Timer Expire Count");
5471
5472 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5473 CTLFLAG_RD, &stats->icrxatc,
5474 "Interrupt Cause Rx Abs Timer Expire Count");
5475
5476 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5477 CTLFLAG_RD, &stats->ictxptc,
5478 "Interrupt Cause Tx Pkt Timer Expire Count");
5479
5480 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5481 CTLFLAG_RD, &stats->ictxatc,
5482 "Interrupt Cause Tx Abs Timer Expire Count");
5483
5484 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5485 CTLFLAG_RD, &stats->ictxqec,
5486 "Interrupt Cause Tx Queue Empty Count");
5487
5488 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5489 CTLFLAG_RD, &stats->ictxqmtc,
5490 "Interrupt Cause Tx Queue Min Thresh Count");
5491
5492 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5493 CTLFLAG_RD, &stats->icrxdmtc,
5494 "Interrupt Cause Rx Desc Min Thresh Count");
5495
5496 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5497 CTLFLAG_RD, &stats->icrxoc,
5498 "Interrupt Cause Receiver Overrun Count");
5499
5500 /* Host to Card Stats */
5501
5502 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
5503 CTLFLAG_RD, NULL,
5504 "Host to Card Statistics");
5505
5506 host_list = SYSCTL_CHILDREN(host_node);
5507
5508 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
5509 CTLFLAG_RD, &stats->cbtmpc,
5510 "Circuit Breaker Tx Packet Count");
5511
5512 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
5513 CTLFLAG_RD, &stats->htdpmc,
5514 "Host Transmit Discarded Packets");
5515
5516 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
5517 CTLFLAG_RD, &stats->rpthc,
5518 "Rx Packets To Host");
5519
5520 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
5521 CTLFLAG_RD, &stats->cbrmpc,
5522 "Circuit Breaker Rx Packet Count");
5523
5524 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
5525 CTLFLAG_RD, &stats->cbrdpc,
5526 "Circuit Breaker Rx Dropped Count");
5527
5528 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
5529 CTLFLAG_RD, &stats->hgptc,
5530 "Host Good Packets Tx Count");
5531
5532 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
5533 CTLFLAG_RD, &stats->htcbdpc,
5534 "Host Tx Circuit Breaker Dropped Count");
5535
5536 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
5537 CTLFLAG_RD, &stats->hgorc,
5538 "Host Good Octets Received Count");
5539
5540 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
5541 CTLFLAG_RD, &stats->hgotc,
5542 "Host Good Octets Transmit Count");
5543
5544 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
5545 CTLFLAG_RD, &stats->lenerrs,
5546 "Length Errors");
5547
5548 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
5549 CTLFLAG_RD, &stats->scvpc,
5550 "SerDes/SGMII Code Violation Pkt Count");
5551
5552 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
5553 CTLFLAG_RD, &stats->hrmpc,
5554 "Header Redirection Missed Packet Count");
5555 }
5556
5557
5558 /**********************************************************************
5559 *
5560 * This routine provides a way to dump out the adapter eeprom,
5561 * often a useful debug/service tool. This only dumps the first
5562 * 32 words, stuff that matters is in that extent.
5563 *
5564 **********************************************************************/
5565 static int
5566 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5567 {
5568 struct adapter *adapter;
5569 int error;
5570 int result;
5571
5572 result = -1;
5573 error = sysctl_handle_int(oidp, &result, 0, req);
5574
5575 if (error || !req->newptr)
5576 return (error);
5577
5578 /*
5579 * This value will cause a hex dump of the
5580 * first 32 16-bit words of the EEPROM to
5581 * the screen.
5582 */
5583 if (result == 1) {
5584 adapter = (struct adapter *)arg1;
5585 igb_print_nvm_info(adapter);
5586 }
5587
5588 return (error);
5589 }
5590
5591 static void
5592 igb_print_nvm_info(struct adapter *adapter)
5593 {
5594 u16 eeprom_data;
5595 int i, j, row = 0;
5596
5597 /* Its a bit crude, but it gets the job done */
5598 printf("\nInterface EEPROM Dump:\n");
5599 printf("Offset\n0x0000 ");
5600 for (i = 0, j = 0; i < 32; i++, j++) {
5601 if (j == 8) { /* Make the offset block */
5602 j = 0; ++row;
5603 printf("\n0x00%x0 ",row);
5604 }
5605 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5606 printf("%04x ", eeprom_data);
5607 }
5608 printf("\n");
5609 }
5610
5611 static void
5612 igb_set_sysctl_value(struct adapter *adapter, const char *name,
5613 const char *description, int *limit, int value)
5614 {
5615 *limit = value;
5616 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5617 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5618 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5619 }
5620
5621 /*
5622 ** Set flow control using sysctl:
5623 ** Flow control values:
5624 ** 0 - off
5625 ** 1 - rx pause
5626 ** 2 - tx pause
5627 ** 3 - full
5628 */
5629 static int
5630 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
5631 {
5632 int error;
5633 struct adapter *adapter = (struct adapter *) arg1;
5634
5635 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5636
5637 if ((error) || (req->newptr == NULL))
5638 return (error);
5639
5640 switch (adapter->fc) {
5641 case e1000_fc_rx_pause:
5642 case e1000_fc_tx_pause:
5643 case e1000_fc_full:
5644 adapter->hw.fc.requested_mode = adapter->fc;
5645 break;
5646 case e1000_fc_none:
5647 default:
5648 adapter->hw.fc.requested_mode = e1000_fc_none;
5649 }
5650
5651 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
5652 e1000_force_mac_fc(&adapter->hw);
5653 return (error);
5654 }
5655
5656 /*
5657 ** Manage DMA Coalesce:
5658 ** Control values:
5659 ** 0/1 - off/on
5660 ** Legal timer values are:
5661 ** 250,500,1000-10000 in thousands
5662 */
5663 static int
5664 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
5665 {
5666 struct adapter *adapter = (struct adapter *) arg1;
5667 int error;
5668
5669 error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
5670
5671 if ((error) || (req->newptr == NULL))
5672 return (error);
5673
5674 switch (adapter->dmac) {
5675 case 0:
5676 /*Disabling */
5677 break;
5678 case 1: /* Just enable and use default */
5679 adapter->dmac = 1000;
5680 break;
5681 case 250:
5682 case 500:
5683 case 1000:
5684 case 2000:
5685 case 3000:
5686 case 4000:
5687 case 5000:
5688 case 6000:
5689 case 7000:
5690 case 8000:
5691 case 9000:
5692 case 10000:
5693 /* Legal values - allow */
5694 break;
5695 default:
5696 /* Do nothing, illegal value */
5697 adapter->dmac = 0;
5698 return (error);
5699 }
5700 /* Reinit the interface */
5701 igb_init(adapter);
5702 return (error);
5703 }
Cache object: d136c969d0d3549046d89bc45713b833
|