1 /**************************************************************************
2 SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3
4 Copyright (c) 2007-2009, Chelsio Inc.
5 All rights reserved.
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
12
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
28
29 ***************************************************************************/
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "opt_inet6.h"
35 #include "opt_inet.h"
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/module.h>
41 #include <sys/bus.h>
42 #include <sys/conf.h>
43 #include <machine/bus.h>
44 #include <machine/resource.h>
45 #include <sys/rman.h>
46 #include <sys/queue.h>
47 #include <sys/sysctl.h>
48 #include <sys/taskqueue.h>
49
50 #include <sys/proc.h>
51 #include <sys/sbuf.h>
52 #include <sys/sched.h>
53 #include <sys/smp.h>
54 #include <sys/systm.h>
55 #include <sys/syslog.h>
56 #include <sys/socket.h>
57 #include <sys/sglist.h>
58
59 #include <net/if.h>
60 #include <net/if_var.h>
61 #include <net/bpf.h>
62 #include <net/ethernet.h>
63 #include <net/if_vlan_var.h>
64
65 #include <netinet/in_systm.h>
66 #include <netinet/in.h>
67 #include <netinet/ip.h>
68 #include <netinet/ip6.h>
69 #include <netinet/tcp.h>
70
71 #include <dev/pci/pcireg.h>
72 #include <dev/pci/pcivar.h>
73
74 #include <vm/vm.h>
75 #include <vm/pmap.h>
76
77 #include <cxgb_include.h>
78 #include <sys/mvec.h>
79
80 int txq_fills = 0;
81 int multiq_tx_enable = 1;
82
83 #ifdef TCP_OFFLOAD
84 CTASSERT(NUM_CPL_HANDLERS >= NUM_CPL_CMDS);
85 #endif
86
87 extern struct sysctl_oid_list sysctl__hw_cxgb_children;
88 int cxgb_txq_buf_ring_size = TX_ETH_Q_SIZE;
89 SYSCTL_INT(_hw_cxgb, OID_AUTO, txq_mr_size, CTLFLAG_RDTUN, &cxgb_txq_buf_ring_size, 0,
90 "size of per-queue mbuf ring");
91
92 static int cxgb_tx_coalesce_force = 0;
93 SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_coalesce_force, CTLFLAG_RWTUN,
94 &cxgb_tx_coalesce_force, 0,
95 "coalesce small packets into a single work request regardless of ring state");
96
97 #define COALESCE_START_DEFAULT TX_ETH_Q_SIZE>>1
98 #define COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3))
99 #define COALESCE_STOP_DEFAULT TX_ETH_Q_SIZE>>2
100 #define COALESCE_STOP_MIN TX_ETH_Q_SIZE>>5
101 #define TX_RECLAIM_DEFAULT TX_ETH_Q_SIZE>>5
102 #define TX_RECLAIM_MAX TX_ETH_Q_SIZE>>2
103 #define TX_RECLAIM_MIN TX_ETH_Q_SIZE>>6
104
105
106 static int cxgb_tx_coalesce_enable_start = COALESCE_START_DEFAULT;
107 SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_start, CTLFLAG_RWTUN,
108 &cxgb_tx_coalesce_enable_start, 0,
109 "coalesce enable threshold");
110 static int cxgb_tx_coalesce_enable_stop = COALESCE_STOP_DEFAULT;
111 SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_stop, CTLFLAG_RWTUN,
112 &cxgb_tx_coalesce_enable_stop, 0,
113 "coalesce disable threshold");
114 static int cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT;
115 SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_reclaim_threshold, CTLFLAG_RWTUN,
116 &cxgb_tx_reclaim_threshold, 0,
117 "tx cleaning minimum threshold");
118
119 /*
120 * XXX don't re-enable this until TOE stops assuming
121 * we have an m_ext
122 */
123 static int recycle_enable = 0;
124
125 extern int cxgb_use_16k_clusters;
126 extern int nmbjumbop;
127 extern int nmbjumbo9;
128 extern int nmbjumbo16;
129
130 #define USE_GTS 0
131
132 #define SGE_RX_SM_BUF_SIZE 1536
133 #define SGE_RX_DROP_THRES 16
134 #define SGE_RX_COPY_THRES 128
135
136 /*
137 * Period of the Tx buffer reclaim timer. This timer does not need to run
138 * frequently as Tx buffers are usually reclaimed by new Tx packets.
139 */
140 #define TX_RECLAIM_PERIOD (hz >> 1)
141
142 /*
143 * Values for sge_txq.flags
144 */
145 enum {
146 TXQ_RUNNING = 1 << 0, /* fetch engine is running */
147 TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */
148 };
149
150 struct tx_desc {
151 uint64_t flit[TX_DESC_FLITS];
152 } __packed;
153
154 struct rx_desc {
155 uint32_t addr_lo;
156 uint32_t len_gen;
157 uint32_t gen2;
158 uint32_t addr_hi;
159 } __packed;
160
161 struct rsp_desc { /* response queue descriptor */
162 struct rss_header rss_hdr;
163 uint32_t flags;
164 uint32_t len_cq;
165 uint8_t imm_data[47];
166 uint8_t intr_gen;
167 } __packed;
168
169 #define RX_SW_DESC_MAP_CREATED (1 << 0)
170 #define TX_SW_DESC_MAP_CREATED (1 << 1)
171 #define RX_SW_DESC_INUSE (1 << 3)
172 #define TX_SW_DESC_MAPPED (1 << 4)
173
174 #define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0)
175 #define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP)
176 #define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP)
177 #define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP)
178
179 struct tx_sw_desc { /* SW state per Tx descriptor */
180 struct mbuf *m;
181 bus_dmamap_t map;
182 int flags;
183 };
184
185 struct rx_sw_desc { /* SW state per Rx descriptor */
186 caddr_t rxsd_cl;
187 struct mbuf *m;
188 bus_dmamap_t map;
189 int flags;
190 };
191
192 struct txq_state {
193 unsigned int compl;
194 unsigned int gen;
195 unsigned int pidx;
196 };
197
198 struct refill_fl_cb_arg {
199 int error;
200 bus_dma_segment_t seg;
201 int nseg;
202 };
203
204
205 /*
206 * Maps a number of flits to the number of Tx descriptors that can hold them.
207 * The formula is
208 *
209 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
210 *
211 * HW allows up to 4 descriptors to be combined into a WR.
212 */
213 static uint8_t flit_desc_map[] = {
214 0,
215 #if SGE_NUM_GENBITS == 1
216 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
217 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
218 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
219 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
220 #elif SGE_NUM_GENBITS == 2
221 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
222 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
223 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
224 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
225 #else
226 # error "SGE_NUM_GENBITS must be 1 or 2"
227 #endif
228 };
229
230 #define TXQ_LOCK_ASSERT(qs) mtx_assert(&(qs)->lock, MA_OWNED)
231 #define TXQ_TRYLOCK(qs) mtx_trylock(&(qs)->lock)
232 #define TXQ_LOCK(qs) mtx_lock(&(qs)->lock)
233 #define TXQ_UNLOCK(qs) mtx_unlock(&(qs)->lock)
234 #define TXQ_RING_EMPTY(qs) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
235 #define TXQ_RING_NEEDS_ENQUEUE(qs) \
236 drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
237 #define TXQ_RING_FLUSH(qs) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
238 #define TXQ_RING_DEQUEUE_COND(qs, func, arg) \
239 drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg)
240 #define TXQ_RING_DEQUEUE(qs) \
241 drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
242
243 int cxgb_debug = 0;
244
245 static void sge_timer_cb(void *arg);
246 static void sge_timer_reclaim(void *arg, int ncount);
247 static void sge_txq_reclaim_handler(void *arg, int ncount);
248 static void cxgb_start_locked(struct sge_qset *qs);
249
250 /*
251 * XXX need to cope with bursty scheduling by looking at a wider
252 * window than we are now for determining the need for coalescing
253 *
254 */
255 static __inline uint64_t
256 check_pkt_coalesce(struct sge_qset *qs)
257 {
258 struct adapter *sc;
259 struct sge_txq *txq;
260 uint8_t *fill;
261
262 if (__predict_false(cxgb_tx_coalesce_force))
263 return (1);
264 txq = &qs->txq[TXQ_ETH];
265 sc = qs->port->adapter;
266 fill = &sc->tunq_fill[qs->idx];
267
268 if (cxgb_tx_coalesce_enable_start > COALESCE_START_MAX)
269 cxgb_tx_coalesce_enable_start = COALESCE_START_MAX;
270 if (cxgb_tx_coalesce_enable_stop < COALESCE_STOP_MIN)
271 cxgb_tx_coalesce_enable_start = COALESCE_STOP_MIN;
272 /*
273 * if the hardware transmit queue is more than 1/8 full
274 * we mark it as coalescing - we drop back from coalescing
275 * when we go below 1/32 full and there are no packets enqueued,
276 * this provides us with some degree of hysteresis
277 */
278 if (*fill != 0 && (txq->in_use <= cxgb_tx_coalesce_enable_stop) &&
279 TXQ_RING_EMPTY(qs) && (qs->coalescing == 0))
280 *fill = 0;
281 else if (*fill == 0 && (txq->in_use >= cxgb_tx_coalesce_enable_start))
282 *fill = 1;
283
284 return (sc->tunq_coalesce);
285 }
286
287 #ifdef __LP64__
288 static void
289 set_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo)
290 {
291 uint64_t wr_hilo;
292 #if _BYTE_ORDER == _LITTLE_ENDIAN
293 wr_hilo = wr_hi;
294 wr_hilo |= (((uint64_t)wr_lo)<<32);
295 #else
296 wr_hilo = wr_lo;
297 wr_hilo |= (((uint64_t)wr_hi)<<32);
298 #endif
299 wrp->wrh_hilo = wr_hilo;
300 }
301 #else
302 static void
303 set_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo)
304 {
305
306 wrp->wrh_hi = wr_hi;
307 wmb();
308 wrp->wrh_lo = wr_lo;
309 }
310 #endif
311
312 struct coalesce_info {
313 int count;
314 int nbytes;
315 int noncoal;
316 };
317
318 static int
319 coalesce_check(struct mbuf *m, void *arg)
320 {
321 struct coalesce_info *ci = arg;
322
323 if ((m->m_next != NULL) ||
324 ((mtod(m, vm_offset_t) & PAGE_MASK) + m->m_len > PAGE_SIZE))
325 ci->noncoal = 1;
326
327 if ((ci->count == 0) || (ci->noncoal == 0 && (ci->count < 7) &&
328 (ci->nbytes + m->m_len <= 10500))) {
329 ci->count++;
330 ci->nbytes += m->m_len;
331 return (1);
332 }
333 return (0);
334 }
335
336 static struct mbuf *
337 cxgb_dequeue(struct sge_qset *qs)
338 {
339 struct mbuf *m, *m_head, *m_tail;
340 struct coalesce_info ci;
341
342
343 if (check_pkt_coalesce(qs) == 0)
344 return TXQ_RING_DEQUEUE(qs);
345
346 m_head = m_tail = NULL;
347 ci.count = ci.nbytes = ci.noncoal = 0;
348 do {
349 m = TXQ_RING_DEQUEUE_COND(qs, coalesce_check, &ci);
350 if (m_head == NULL) {
351 m_tail = m_head = m;
352 } else if (m != NULL) {
353 m_tail->m_nextpkt = m;
354 m_tail = m;
355 }
356 } while (m != NULL);
357 if (ci.count > 7)
358 panic("trying to coalesce %d packets in to one WR", ci.count);
359 return (m_head);
360 }
361
362 /**
363 * reclaim_completed_tx - reclaims completed Tx descriptors
364 * @adapter: the adapter
365 * @q: the Tx queue to reclaim completed descriptors from
366 *
367 * Reclaims Tx descriptors that the SGE has indicated it has processed,
368 * and frees the associated buffers if possible. Called with the Tx
369 * queue's lock held.
370 */
371 static __inline int
372 reclaim_completed_tx(struct sge_qset *qs, int reclaim_min, int queue)
373 {
374 struct sge_txq *q = &qs->txq[queue];
375 int reclaim = desc_reclaimable(q);
376
377 if ((cxgb_tx_reclaim_threshold > TX_RECLAIM_MAX) ||
378 (cxgb_tx_reclaim_threshold < TX_RECLAIM_MIN))
379 cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT;
380
381 if (reclaim < reclaim_min)
382 return (0);
383
384 mtx_assert(&qs->lock, MA_OWNED);
385 if (reclaim > 0) {
386 t3_free_tx_desc(qs, reclaim, queue);
387 q->cleaned += reclaim;
388 q->in_use -= reclaim;
389 }
390 if (isset(&qs->txq_stopped, TXQ_ETH))
391 clrbit(&qs->txq_stopped, TXQ_ETH);
392
393 return (reclaim);
394 }
395
396 #ifdef DEBUGNET
397 int
398 cxgb_debugnet_poll_tx(struct sge_qset *qs)
399 {
400
401 return (reclaim_completed_tx(qs, TX_RECLAIM_MAX, TXQ_ETH));
402 }
403 #endif
404
405 /**
406 * should_restart_tx - are there enough resources to restart a Tx queue?
407 * @q: the Tx queue
408 *
409 * Checks if there are enough descriptors to restart a suspended Tx queue.
410 */
411 static __inline int
412 should_restart_tx(const struct sge_txq *q)
413 {
414 unsigned int r = q->processed - q->cleaned;
415
416 return q->in_use - r < (q->size >> 1);
417 }
418
419 /**
420 * t3_sge_init - initialize SGE
421 * @adap: the adapter
422 * @p: the SGE parameters
423 *
424 * Performs SGE initialization needed every time after a chip reset.
425 * We do not initialize any of the queue sets here, instead the driver
426 * top-level must request those individually. We also do not enable DMA
427 * here, that should be done after the queues have been set up.
428 */
429 void
430 t3_sge_init(adapter_t *adap, struct sge_params *p)
431 {
432 u_int ctrl, ups;
433
434 ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */
435
436 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
437 F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
438 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
439 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
440 #if SGE_NUM_GENBITS == 1
441 ctrl |= F_EGRGENCTRL;
442 #endif
443 if (adap->params.rev > 0) {
444 if (!(adap->flags & (USING_MSIX | USING_MSI)))
445 ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
446 }
447 t3_write_reg(adap, A_SG_CONTROL, ctrl);
448 t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
449 V_LORCQDRBTHRSH(512));
450 t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
451 t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
452 V_TIMEOUT(200 * core_ticks_per_usec(adap)));
453 t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
454 adap->params.rev < T3_REV_C ? 1000 : 500);
455 t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
456 t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
457 t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
458 t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
459 t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
460 }
461
462
463 /**
464 * sgl_len - calculates the size of an SGL of the given capacity
465 * @n: the number of SGL entries
466 *
467 * Calculates the number of flits needed for a scatter/gather list that
468 * can hold the given number of entries.
469 */
470 static __inline unsigned int
471 sgl_len(unsigned int n)
472 {
473 return ((3 * n) / 2 + (n & 1));
474 }
475
476 /**
477 * get_imm_packet - return the next ingress packet buffer from a response
478 * @resp: the response descriptor containing the packet data
479 *
480 * Return a packet containing the immediate data of the given response.
481 */
482 static int
483 get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m)
484 {
485
486 if (resp->rss_hdr.opcode == CPL_RX_DATA) {
487 const struct cpl_rx_data *cpl = (const void *)&resp->imm_data[0];
488 m->m_len = sizeof(*cpl) + ntohs(cpl->len);
489 } else if (resp->rss_hdr.opcode == CPL_RX_PKT) {
490 const struct cpl_rx_pkt *cpl = (const void *)&resp->imm_data[0];
491 m->m_len = sizeof(*cpl) + ntohs(cpl->len);
492 } else
493 m->m_len = IMMED_PKT_SIZE;
494 m->m_ext.ext_buf = NULL;
495 m->m_ext.ext_type = 0;
496 memcpy(mtod(m, uint8_t *), resp->imm_data, m->m_len);
497 return (0);
498 }
499
500 static __inline u_int
501 flits_to_desc(u_int n)
502 {
503 return (flit_desc_map[n]);
504 }
505
506 #define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
507 F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
508 V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
509 F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
510 F_HIRCQPARITYERROR)
511 #define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
512 #define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
513 F_RSPQDISABLED)
514
515 /**
516 * t3_sge_err_intr_handler - SGE async event interrupt handler
517 * @adapter: the adapter
518 *
519 * Interrupt handler for SGE asynchronous (non-data) events.
520 */
521 void
522 t3_sge_err_intr_handler(adapter_t *adapter)
523 {
524 unsigned int v, status;
525
526 status = t3_read_reg(adapter, A_SG_INT_CAUSE);
527 if (status & SGE_PARERR)
528 CH_ALERT(adapter, "SGE parity error (0x%x)\n",
529 status & SGE_PARERR);
530 if (status & SGE_FRAMINGERR)
531 CH_ALERT(adapter, "SGE framing error (0x%x)\n",
532 status & SGE_FRAMINGERR);
533 if (status & F_RSPQCREDITOVERFOW)
534 CH_ALERT(adapter, "SGE response queue credit overflow\n");
535
536 if (status & F_RSPQDISABLED) {
537 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
538
539 CH_ALERT(adapter,
540 "packet delivered to disabled response queue (0x%x)\n",
541 (v >> S_RSPQ0DISABLED) & 0xff);
542 }
543
544 t3_write_reg(adapter, A_SG_INT_CAUSE, status);
545 if (status & SGE_FATALERR)
546 t3_fatal_err(adapter);
547 }
548
549 void
550 t3_sge_prep(adapter_t *adap, struct sge_params *p)
551 {
552 int i, nqsets, fl_q_size, jumbo_q_size, use_16k, jumbo_buf_size;
553
554 nqsets = min(SGE_QSETS / adap->params.nports, mp_ncpus);
555 nqsets *= adap->params.nports;
556
557 fl_q_size = min(nmbclusters/(3*nqsets), FL_Q_SIZE);
558
559 while (!powerof2(fl_q_size))
560 fl_q_size--;
561
562 use_16k = cxgb_use_16k_clusters != -1 ? cxgb_use_16k_clusters :
563 is_offload(adap);
564
565 if (use_16k) {
566 jumbo_q_size = min(nmbjumbo16/(3*nqsets), JUMBO_Q_SIZE);
567 jumbo_buf_size = MJUM16BYTES;
568 } else {
569 jumbo_q_size = min(nmbjumbo9/(3*nqsets), JUMBO_Q_SIZE);
570 jumbo_buf_size = MJUM9BYTES;
571 }
572 while (!powerof2(jumbo_q_size))
573 jumbo_q_size--;
574
575 if (fl_q_size < (FL_Q_SIZE / 4) || jumbo_q_size < (JUMBO_Q_SIZE / 2))
576 device_printf(adap->dev,
577 "Insufficient clusters and/or jumbo buffers.\n");
578
579 p->max_pkt_size = jumbo_buf_size - sizeof(struct cpl_rx_data);
580
581 for (i = 0; i < SGE_QSETS; ++i) {
582 struct qset_params *q = p->qset + i;
583
584 if (adap->params.nports > 2) {
585 q->coalesce_usecs = 50;
586 } else {
587 #ifdef INVARIANTS
588 q->coalesce_usecs = 10;
589 #else
590 q->coalesce_usecs = 5;
591 #endif
592 }
593 q->polling = 0;
594 q->rspq_size = RSPQ_Q_SIZE;
595 q->fl_size = fl_q_size;
596 q->jumbo_size = jumbo_q_size;
597 q->jumbo_buf_size = jumbo_buf_size;
598 q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE;
599 q->txq_size[TXQ_OFLD] = is_offload(adap) ? TX_OFLD_Q_SIZE : 16;
600 q->txq_size[TXQ_CTRL] = TX_CTRL_Q_SIZE;
601 q->cong_thres = 0;
602 }
603 }
604
605 int
606 t3_sge_alloc(adapter_t *sc)
607 {
608
609 /* The parent tag. */
610 if (bus_dma_tag_create( bus_get_dma_tag(sc->dev),/* PCI parent */
611 1, 0, /* algnmnt, boundary */
612 BUS_SPACE_MAXADDR, /* lowaddr */
613 BUS_SPACE_MAXADDR, /* highaddr */
614 NULL, NULL, /* filter, filterarg */
615 BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
616 BUS_SPACE_UNRESTRICTED, /* nsegments */
617 BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
618 0, /* flags */
619 NULL, NULL, /* lock, lockarg */
620 &sc->parent_dmat)) {
621 device_printf(sc->dev, "Cannot allocate parent DMA tag\n");
622 return (ENOMEM);
623 }
624
625 /*
626 * DMA tag for normal sized RX frames
627 */
628 if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR,
629 BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1,
630 MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) {
631 device_printf(sc->dev, "Cannot allocate RX DMA tag\n");
632 return (ENOMEM);
633 }
634
635 /*
636 * DMA tag for jumbo sized RX frames.
637 */
638 if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR,
639 BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES,
640 BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) {
641 device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n");
642 return (ENOMEM);
643 }
644
645 /*
646 * DMA tag for TX frames.
647 */
648 if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR,
649 BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS,
650 TX_MAX_SIZE, BUS_DMA_ALLOCNOW,
651 NULL, NULL, &sc->tx_dmat)) {
652 device_printf(sc->dev, "Cannot allocate TX DMA tag\n");
653 return (ENOMEM);
654 }
655
656 return (0);
657 }
658
659 int
660 t3_sge_free(struct adapter * sc)
661 {
662
663 if (sc->tx_dmat != NULL)
664 bus_dma_tag_destroy(sc->tx_dmat);
665
666 if (sc->rx_jumbo_dmat != NULL)
667 bus_dma_tag_destroy(sc->rx_jumbo_dmat);
668
669 if (sc->rx_dmat != NULL)
670 bus_dma_tag_destroy(sc->rx_dmat);
671
672 if (sc->parent_dmat != NULL)
673 bus_dma_tag_destroy(sc->parent_dmat);
674
675 return (0);
676 }
677
678 void
679 t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
680 {
681
682 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);
683 qs->rspq.polling = 0 /* p->polling */;
684 }
685
686 #if !defined(__i386__) && !defined(__amd64__)
687 static void
688 refill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
689 {
690 struct refill_fl_cb_arg *cb_arg = arg;
691
692 cb_arg->error = error;
693 cb_arg->seg = segs[0];
694 cb_arg->nseg = nseg;
695
696 }
697 #endif
698 /**
699 * refill_fl - refill an SGE free-buffer list
700 * @sc: the controller softc
701 * @q: the free-list to refill
702 * @n: the number of new buffers to allocate
703 *
704 * (Re)populate an SGE free-buffer list with up to @n new packet buffers.
705 * The caller must assure that @n does not exceed the queue's capacity.
706 */
707 static void
708 refill_fl(adapter_t *sc, struct sge_fl *q, int n)
709 {
710 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
711 struct rx_desc *d = &q->desc[q->pidx];
712 struct refill_fl_cb_arg cb_arg;
713 struct mbuf *m;
714 caddr_t cl;
715 int err;
716
717 cb_arg.error = 0;
718 while (n--) {
719 /*
720 * We allocate an uninitialized mbuf + cluster, mbuf is
721 * initialized after rx.
722 */
723 if (q->zone == zone_pack) {
724 if ((m = m_getcl(M_NOWAIT, MT_NOINIT, M_PKTHDR)) == NULL)
725 break;
726 cl = m->m_ext.ext_buf;
727 } else {
728 if ((cl = m_cljget(NULL, M_NOWAIT, q->buf_size)) == NULL)
729 break;
730 if ((m = m_gethdr_raw(M_NOWAIT, 0)) == NULL) {
731 uma_zfree(q->zone, cl);
732 break;
733 }
734 }
735 if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) {
736 if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) {
737 log(LOG_WARNING, "bus_dmamap_create failed %d\n", err);
738 uma_zfree(q->zone, cl);
739 goto done;
740 }
741 sd->flags |= RX_SW_DESC_MAP_CREATED;
742 }
743 #if !defined(__i386__) && !defined(__amd64__)
744 err = bus_dmamap_load(q->entry_tag, sd->map,
745 cl, q->buf_size, refill_fl_cb, &cb_arg, 0);
746
747 if (err != 0 || cb_arg.error) {
748 if (q->zone != zone_pack)
749 uma_zfree(q->zone, cl);
750 m_free(m);
751 goto done;
752 }
753 #else
754 cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)cl);
755 #endif
756 sd->flags |= RX_SW_DESC_INUSE;
757 sd->rxsd_cl = cl;
758 sd->m = m;
759 d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff);
760 d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff);
761 d->len_gen = htobe32(V_FLD_GEN1(q->gen));
762 d->gen2 = htobe32(V_FLD_GEN2(q->gen));
763
764 d++;
765 sd++;
766
767 if (++q->pidx == q->size) {
768 q->pidx = 0;
769 q->gen ^= 1;
770 sd = q->sdesc;
771 d = q->desc;
772 }
773 q->credits++;
774 q->db_pending++;
775 }
776
777 done:
778 if (q->db_pending >= 32) {
779 q->db_pending = 0;
780 t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
781 }
782 }
783
784
785 /**
786 * free_rx_bufs - free the Rx buffers on an SGE free list
787 * @sc: the controle softc
788 * @q: the SGE free list to clean up
789 *
790 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
791 * this queue should be stopped before calling this function.
792 */
793 static void
794 free_rx_bufs(adapter_t *sc, struct sge_fl *q)
795 {
796 u_int cidx = q->cidx;
797
798 while (q->credits--) {
799 struct rx_sw_desc *d = &q->sdesc[cidx];
800
801 if (d->flags & RX_SW_DESC_INUSE) {
802 bus_dmamap_unload(q->entry_tag, d->map);
803 bus_dmamap_destroy(q->entry_tag, d->map);
804 if (q->zone == zone_pack) {
805 m_init(d->m, M_NOWAIT, MT_DATA, M_EXT);
806 uma_zfree(zone_pack, d->m);
807 } else {
808 m_init(d->m, M_NOWAIT, MT_DATA, 0);
809 m_free_raw(d->m);
810 uma_zfree(q->zone, d->rxsd_cl);
811 }
812 }
813
814 d->rxsd_cl = NULL;
815 d->m = NULL;
816 if (++cidx == q->size)
817 cidx = 0;
818 }
819 }
820
821 static __inline void
822 __refill_fl(adapter_t *adap, struct sge_fl *fl)
823 {
824 refill_fl(adap, fl, min(16U, fl->size - fl->credits));
825 }
826
827 static __inline void
828 __refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max)
829 {
830 uint32_t reclaimable = fl->size - fl->credits;
831
832 if (reclaimable > 0)
833 refill_fl(adap, fl, min(max, reclaimable));
834 }
835
836 /**
837 * recycle_rx_buf - recycle a receive buffer
838 * @adapter: the adapter
839 * @q: the SGE free list
840 * @idx: index of buffer to recycle
841 *
842 * Recycles the specified buffer on the given free list by adding it at
843 * the next available slot on the list.
844 */
845 static void
846 recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx)
847 {
848 struct rx_desc *from = &q->desc[idx];
849 struct rx_desc *to = &q->desc[q->pidx];
850
851 q->sdesc[q->pidx] = q->sdesc[idx];
852 to->addr_lo = from->addr_lo; // already big endian
853 to->addr_hi = from->addr_hi; // likewise
854 wmb(); /* necessary ? */
855 to->len_gen = htobe32(V_FLD_GEN1(q->gen));
856 to->gen2 = htobe32(V_FLD_GEN2(q->gen));
857 q->credits++;
858
859 if (++q->pidx == q->size) {
860 q->pidx = 0;
861 q->gen ^= 1;
862 }
863 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
864 }
865
866 static void
867 alloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
868 {
869 uint32_t *addr;
870
871 addr = arg;
872 *addr = segs[0].ds_addr;
873 }
874
875 static int
876 alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size,
877 bus_addr_t *phys, void *desc, void *sdesc, bus_dma_tag_t *tag,
878 bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag)
879 {
880 size_t len = nelem * elem_size;
881 void *s = NULL;
882 void *p = NULL;
883 int err;
884
885 if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0,
886 BUS_SPACE_MAXADDR_32BIT,
887 BUS_SPACE_MAXADDR, NULL, NULL, len, 1,
888 len, 0, NULL, NULL, tag)) != 0) {
889 device_printf(sc->dev, "Cannot allocate descriptor tag\n");
890 return (ENOMEM);
891 }
892
893 if ((err = bus_dmamem_alloc(*tag, (void **)&p, BUS_DMA_NOWAIT,
894 map)) != 0) {
895 device_printf(sc->dev, "Cannot allocate descriptor memory\n");
896 return (ENOMEM);
897 }
898
899 bus_dmamap_load(*tag, *map, p, len, alloc_ring_cb, phys, 0);
900 bzero(p, len);
901 *(void **)desc = p;
902
903 if (sw_size) {
904 len = nelem * sw_size;
905 s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO);
906 *(void **)sdesc = s;
907 }
908 if (parent_entry_tag == NULL)
909 return (0);
910
911 if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0,
912 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
913 NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS,
914 TX_MAX_SIZE, BUS_DMA_ALLOCNOW,
915 NULL, NULL, entry_tag)) != 0) {
916 device_printf(sc->dev, "Cannot allocate descriptor entry tag\n");
917 return (ENOMEM);
918 }
919 return (0);
920 }
921
922 static void
923 sge_slow_intr_handler(void *arg, int ncount)
924 {
925 adapter_t *sc = arg;
926
927 t3_slow_intr_handler(sc);
928 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
929 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
930 }
931
932 /**
933 * sge_timer_cb - perform periodic maintenance of an SGE qset
934 * @data: the SGE queue set to maintain
935 *
936 * Runs periodically from a timer to perform maintenance of an SGE queue
937 * set. It performs two tasks:
938 *
939 * a) Cleans up any completed Tx descriptors that may still be pending.
940 * Normal descriptor cleanup happens when new packets are added to a Tx
941 * queue so this timer is relatively infrequent and does any cleanup only
942 * if the Tx queue has not seen any new packets in a while. We make a
943 * best effort attempt to reclaim descriptors, in that we don't wait
944 * around if we cannot get a queue's lock (which most likely is because
945 * someone else is queueing new packets and so will also handle the clean
946 * up). Since control queues use immediate data exclusively we don't
947 * bother cleaning them up here.
948 *
949 * b) Replenishes Rx queues that have run out due to memory shortage.
950 * Normally new Rx buffers are added when existing ones are consumed but
951 * when out of memory a queue can become empty. We try to add only a few
952 * buffers here, the queue will be replenished fully as these new buffers
953 * are used up if memory shortage has subsided.
954 *
955 * c) Return coalesced response queue credits in case a response queue is
956 * starved.
957 *
958 * d) Ring doorbells for T304 tunnel queues since we have seen doorbell
959 * fifo overflows and the FW doesn't implement any recovery scheme yet.
960 */
961 static void
962 sge_timer_cb(void *arg)
963 {
964 adapter_t *sc = arg;
965 if ((sc->flags & USING_MSIX) == 0) {
966
967 struct port_info *pi;
968 struct sge_qset *qs;
969 struct sge_txq *txq;
970 int i, j;
971 int reclaim_ofl, refill_rx;
972
973 if (sc->open_device_map == 0)
974 return;
975
976 for (i = 0; i < sc->params.nports; i++) {
977 pi = &sc->port[i];
978 for (j = 0; j < pi->nqsets; j++) {
979 qs = &sc->sge.qs[pi->first_qset + j];
980 txq = &qs->txq[0];
981 reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned;
982 refill_rx = ((qs->fl[0].credits < qs->fl[0].size) ||
983 (qs->fl[1].credits < qs->fl[1].size));
984 if (reclaim_ofl || refill_rx) {
985 taskqueue_enqueue(sc->tq, &pi->timer_reclaim_task);
986 break;
987 }
988 }
989 }
990 }
991
992 if (sc->params.nports > 2) {
993 int i;
994
995 for_each_port(sc, i) {
996 struct port_info *pi = &sc->port[i];
997
998 t3_write_reg(sc, A_SG_KDOORBELL,
999 F_SELEGRCNTX |
1000 (FW_TUNNEL_SGEEC_START + pi->first_qset));
1001 }
1002 }
1003 if (((sc->flags & USING_MSIX) == 0 || sc->params.nports > 2) &&
1004 sc->open_device_map != 0)
1005 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
1006 }
1007
1008 /*
1009 * This is meant to be a catch-all function to keep sge state private
1010 * to sge.c
1011 *
1012 */
1013 int
1014 t3_sge_init_adapter(adapter_t *sc)
1015 {
1016 callout_init(&sc->sge_timer_ch, 1);
1017 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
1018 TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc);
1019 return (0);
1020 }
1021
1022 int
1023 t3_sge_reset_adapter(adapter_t *sc)
1024 {
1025 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
1026 return (0);
1027 }
1028
1029 int
1030 t3_sge_init_port(struct port_info *pi)
1031 {
1032 TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi);
1033 return (0);
1034 }
1035
1036 /**
1037 * refill_rspq - replenish an SGE response queue
1038 * @adapter: the adapter
1039 * @q: the response queue to replenish
1040 * @credits: how many new responses to make available
1041 *
1042 * Replenishes a response queue by making the supplied number of responses
1043 * available to HW.
1044 */
1045 static __inline void
1046 refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits)
1047 {
1048
1049 /* mbufs are allocated on demand when a rspq entry is processed. */
1050 t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN,
1051 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
1052 }
1053
1054 static void
1055 sge_txq_reclaim_handler(void *arg, int ncount)
1056 {
1057 struct sge_qset *qs = arg;
1058 int i;
1059
1060 for (i = 0; i < 3; i++)
1061 reclaim_completed_tx(qs, 16, i);
1062 }
1063
1064 static void
1065 sge_timer_reclaim(void *arg, int ncount)
1066 {
1067 struct port_info *pi = arg;
1068 int i, nqsets = pi->nqsets;
1069 adapter_t *sc = pi->adapter;
1070 struct sge_qset *qs;
1071 struct mtx *lock;
1072
1073 KASSERT((sc->flags & USING_MSIX) == 0,
1074 ("can't call timer reclaim for msi-x"));
1075
1076 for (i = 0; i < nqsets; i++) {
1077 qs = &sc->sge.qs[pi->first_qset + i];
1078
1079 reclaim_completed_tx(qs, 16, TXQ_OFLD);
1080 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock :
1081 &sc->sge.qs[0].rspq.lock;
1082
1083 if (mtx_trylock(lock)) {
1084 /* XXX currently assume that we are *NOT* polling */
1085 uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS);
1086
1087 if (qs->fl[0].credits < qs->fl[0].size - 16)
1088 __refill_fl(sc, &qs->fl[0]);
1089 if (qs->fl[1].credits < qs->fl[1].size - 16)
1090 __refill_fl(sc, &qs->fl[1]);
1091
1092 if (status & (1 << qs->rspq.cntxt_id)) {
1093 if (qs->rspq.credits) {
1094 refill_rspq(sc, &qs->rspq, 1);
1095 qs->rspq.credits--;
1096 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS,
1097 1 << qs->rspq.cntxt_id);
1098 }
1099 }
1100 mtx_unlock(lock);
1101 }
1102 }
1103 }
1104
1105 /**
1106 * init_qset_cntxt - initialize an SGE queue set context info
1107 * @qs: the queue set
1108 * @id: the queue set id
1109 *
1110 * Initializes the TIDs and context ids for the queues of a queue set.
1111 */
1112 static void
1113 init_qset_cntxt(struct sge_qset *qs, u_int id)
1114 {
1115
1116 qs->rspq.cntxt_id = id;
1117 qs->fl[0].cntxt_id = 2 * id;
1118 qs->fl[1].cntxt_id = 2 * id + 1;
1119 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
1120 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
1121 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
1122 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
1123 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
1124
1125 /* XXX: a sane limit is needed instead of INT_MAX */
1126 mbufq_init(&qs->txq[TXQ_ETH].sendq, INT_MAX);
1127 mbufq_init(&qs->txq[TXQ_OFLD].sendq, INT_MAX);
1128 mbufq_init(&qs->txq[TXQ_CTRL].sendq, INT_MAX);
1129 }
1130
1131
1132 static void
1133 txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs)
1134 {
1135 txq->in_use += ndesc;
1136 /*
1137 * XXX we don't handle stopping of queue
1138 * presumably start handles this when we bump against the end
1139 */
1140 txqs->gen = txq->gen;
1141 txq->unacked += ndesc;
1142 txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5);
1143 txq->unacked &= 31;
1144 txqs->pidx = txq->pidx;
1145 txq->pidx += ndesc;
1146 #ifdef INVARIANTS
1147 if (((txqs->pidx > txq->cidx) &&
1148 (txq->pidx < txqs->pidx) &&
1149 (txq->pidx >= txq->cidx)) ||
1150 ((txqs->pidx < txq->cidx) &&
1151 (txq->pidx >= txq-> cidx)) ||
1152 ((txqs->pidx < txq->cidx) &&
1153 (txq->cidx < txqs->pidx)))
1154 panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d",
1155 txqs->pidx, txq->pidx, txq->cidx);
1156 #endif
1157 if (txq->pidx >= txq->size) {
1158 txq->pidx -= txq->size;
1159 txq->gen ^= 1;
1160 }
1161
1162 }
1163
1164 /**
1165 * calc_tx_descs - calculate the number of Tx descriptors for a packet
1166 * @m: the packet mbufs
1167 * @nsegs: the number of segments
1168 *
1169 * Returns the number of Tx descriptors needed for the given Ethernet
1170 * packet. Ethernet packets require addition of WR and CPL headers.
1171 */
1172 static __inline unsigned int
1173 calc_tx_descs(const struct mbuf *m, int nsegs)
1174 {
1175 unsigned int flits;
1176
1177 if (m->m_pkthdr.len <= PIO_LEN)
1178 return 1;
1179
1180 flits = sgl_len(nsegs) + 2;
1181 if (m->m_pkthdr.csum_flags & CSUM_TSO)
1182 flits++;
1183
1184 return flits_to_desc(flits);
1185 }
1186
1187 /**
1188 * make_sgl - populate a scatter/gather list for a packet
1189 * @sgp: the SGL to populate
1190 * @segs: the packet dma segments
1191 * @nsegs: the number of segments
1192 *
1193 * Generates a scatter/gather list for the buffers that make up a packet
1194 * and returns the SGL size in 8-byte words. The caller must size the SGL
1195 * appropriately.
1196 */
1197 static __inline void
1198 make_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs)
1199 {
1200 int i, idx;
1201
1202 for (idx = 0, i = 0; i < nsegs; i++) {
1203 /*
1204 * firmware doesn't like empty segments
1205 */
1206 if (segs[i].ds_len == 0)
1207 continue;
1208 if (i && idx == 0)
1209 ++sgp;
1210
1211 sgp->len[idx] = htobe32(segs[i].ds_len);
1212 sgp->addr[idx] = htobe64(segs[i].ds_addr);
1213 idx ^= 1;
1214 }
1215
1216 if (idx) {
1217 sgp->len[idx] = 0;
1218 sgp->addr[idx] = 0;
1219 }
1220 }
1221
1222 /**
1223 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
1224 * @adap: the adapter
1225 * @q: the Tx queue
1226 *
1227 * Ring the doorbell if a Tx queue is asleep. There is a natural race,
1228 * where the HW is going to sleep just after we checked, however,
1229 * then the interrupt handler will detect the outstanding TX packet
1230 * and ring the doorbell for us.
1231 *
1232 * When GTS is disabled we unconditionally ring the doorbell.
1233 */
1234 static __inline void
1235 check_ring_tx_db(adapter_t *adap, struct sge_txq *q, int mustring)
1236 {
1237 #if USE_GTS
1238 clear_bit(TXQ_LAST_PKT_DB, &q->flags);
1239 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
1240 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1241 #ifdef T3_TRACE
1242 T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d",
1243 q->cntxt_id);
1244 #endif
1245 t3_write_reg(adap, A_SG_KDOORBELL,
1246 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1247 }
1248 #else
1249 if (mustring || ++q->db_pending >= 32) {
1250 wmb(); /* write descriptors before telling HW */
1251 t3_write_reg(adap, A_SG_KDOORBELL,
1252 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1253 q->db_pending = 0;
1254 }
1255 #endif
1256 }
1257
1258 static __inline void
1259 wr_gen2(struct tx_desc *d, unsigned int gen)
1260 {
1261 #if SGE_NUM_GENBITS == 2
1262 d->flit[TX_DESC_FLITS - 1] = htobe64(gen);
1263 #endif
1264 }
1265
1266 /**
1267 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
1268 * @ndesc: number of Tx descriptors spanned by the SGL
1269 * @txd: first Tx descriptor to be written
1270 * @txqs: txq state (generation and producer index)
1271 * @txq: the SGE Tx queue
1272 * @sgl: the SGL
1273 * @flits: number of flits to the start of the SGL in the first descriptor
1274 * @sgl_flits: the SGL size in flits
1275 * @wr_hi: top 32 bits of WR header based on WR type (big endian)
1276 * @wr_lo: low 32 bits of WR header based on WR type (big endian)
1277 *
1278 * Write a work request header and an associated SGL. If the SGL is
1279 * small enough to fit into one Tx descriptor it has already been written
1280 * and we just need to write the WR header. Otherwise we distribute the
1281 * SGL across the number of descriptors it spans.
1282 */
1283 static void
1284 write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs,
1285 const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits,
1286 unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo)
1287 {
1288
1289 struct work_request_hdr *wrp = (struct work_request_hdr *)txd;
1290
1291 if (__predict_true(ndesc == 1)) {
1292 set_wr_hdr(wrp, htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1293 V_WR_SGLSFLT(flits)) | wr_hi,
1294 htonl(V_WR_LEN(flits + sgl_flits) | V_WR_GEN(txqs->gen)) |
1295 wr_lo);
1296
1297 wr_gen2(txd, txqs->gen);
1298
1299 } else {
1300 unsigned int ogen = txqs->gen;
1301 const uint64_t *fp = (const uint64_t *)sgl;
1302 struct work_request_hdr *wp = wrp;
1303
1304 wrp->wrh_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
1305 V_WR_SGLSFLT(flits)) | wr_hi;
1306
1307 while (sgl_flits) {
1308 unsigned int avail = WR_FLITS - flits;
1309
1310 if (avail > sgl_flits)
1311 avail = sgl_flits;
1312 memcpy(&txd->flit[flits], fp, avail * sizeof(*fp));
1313 sgl_flits -= avail;
1314 ndesc--;
1315 if (!sgl_flits)
1316 break;
1317
1318 fp += avail;
1319 txd++;
1320 if (++txqs->pidx == txq->size) {
1321 txqs->pidx = 0;
1322 txqs->gen ^= 1;
1323 txd = txq->desc;
1324 }
1325
1326 /*
1327 * when the head of the mbuf chain
1328 * is freed all clusters will be freed
1329 * with it
1330 */
1331 wrp = (struct work_request_hdr *)txd;
1332 wrp->wrh_hi = htonl(V_WR_DATATYPE(1) |
1333 V_WR_SGLSFLT(1)) | wr_hi;
1334 wrp->wrh_lo = htonl(V_WR_LEN(min(WR_FLITS,
1335 sgl_flits + 1)) |
1336 V_WR_GEN(txqs->gen)) | wr_lo;
1337 wr_gen2(txd, txqs->gen);
1338 flits = 1;
1339 }
1340 wrp->wrh_hi |= htonl(F_WR_EOP);
1341 wmb();
1342 wp->wrh_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
1343 wr_gen2((struct tx_desc *)wp, ogen);
1344 }
1345 }
1346
1347 /* sizeof(*eh) + sizeof(*ip) + sizeof(*tcp) */
1348 #define TCPPKTHDRSIZE (ETHER_HDR_LEN + 20 + 20)
1349
1350 #define GET_VTAG(cntrl, m) \
1351 do { \
1352 if ((m)->m_flags & M_VLANTAG) \
1353 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \
1354 } while (0)
1355
1356 static int
1357 t3_encap(struct sge_qset *qs, struct mbuf **m)
1358 {
1359 adapter_t *sc;
1360 struct mbuf *m0;
1361 struct sge_txq *txq;
1362 struct txq_state txqs;
1363 struct port_info *pi;
1364 unsigned int ndesc, flits, cntrl, mlen;
1365 int err, nsegs, tso_info = 0;
1366
1367 struct work_request_hdr *wrp;
1368 struct tx_sw_desc *txsd;
1369 struct sg_ent *sgp, *sgl;
1370 uint32_t wr_hi, wr_lo, sgl_flits;
1371 bus_dma_segment_t segs[TX_MAX_SEGS];
1372
1373 struct tx_desc *txd;
1374
1375 pi = qs->port;
1376 sc = pi->adapter;
1377 txq = &qs->txq[TXQ_ETH];
1378 txd = &txq->desc[txq->pidx];
1379 txsd = &txq->sdesc[txq->pidx];
1380 sgl = txq->txq_sgl;
1381
1382 prefetch(txd);
1383 m0 = *m;
1384
1385 mtx_assert(&qs->lock, MA_OWNED);
1386 cntrl = V_TXPKT_INTF(pi->txpkt_intf);
1387 KASSERT(m0->m_flags & M_PKTHDR, ("not packet header\n"));
1388
1389 if (m0->m_nextpkt == NULL && m0->m_next != NULL &&
1390 m0->m_pkthdr.csum_flags & (CSUM_TSO))
1391 tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz);
1392
1393 if (m0->m_nextpkt != NULL) {
1394 busdma_map_sg_vec(txq->entry_tag, txsd->map, m0, segs, &nsegs);
1395 ndesc = 1;
1396 mlen = 0;
1397 } else {
1398 if ((err = busdma_map_sg_collapse(txq->entry_tag, txsd->map,
1399 &m0, segs, &nsegs))) {
1400 if (cxgb_debug)
1401 printf("failed ... err=%d\n", err);
1402 return (err);
1403 }
1404 mlen = m0->m_pkthdr.len;
1405 ndesc = calc_tx_descs(m0, nsegs);
1406 }
1407 txq_prod(txq, ndesc, &txqs);
1408
1409 KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d", nsegs));
1410 txsd->m = m0;
1411
1412 if (m0->m_nextpkt != NULL) {
1413 struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd;
1414 int i, fidx;
1415
1416 if (nsegs > 7)
1417 panic("trying to coalesce %d packets in to one WR", nsegs);
1418 txq->txq_coalesced += nsegs;
1419 wrp = (struct work_request_hdr *)txd;
1420 flits = nsegs*2 + 1;
1421
1422 for (fidx = 1, i = 0; i < nsegs; i++, fidx += 2) {
1423 struct cpl_tx_pkt_batch_entry *cbe;
1424 uint64_t flit;
1425 uint32_t *hflit = (uint32_t *)&flit;
1426 int cflags = m0->m_pkthdr.csum_flags;
1427
1428 cntrl = V_TXPKT_INTF(pi->txpkt_intf);
1429 GET_VTAG(cntrl, m0);
1430 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1431 if (__predict_false(!(cflags & CSUM_IP)))
1432 cntrl |= F_TXPKT_IPCSUM_DIS;
1433 if (__predict_false(!(cflags & (CSUM_TCP | CSUM_UDP |
1434 CSUM_UDP_IPV6 | CSUM_TCP_IPV6))))
1435 cntrl |= F_TXPKT_L4CSUM_DIS;
1436
1437 hflit[0] = htonl(cntrl);
1438 hflit[1] = htonl(segs[i].ds_len | 0x80000000);
1439 flit |= htobe64(1 << 24);
1440 cbe = &cpl_batch->pkt_entry[i];
1441 cbe->cntrl = hflit[0];
1442 cbe->len = hflit[1];
1443 cbe->addr = htobe64(segs[i].ds_addr);
1444 }
1445
1446 wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1447 V_WR_SGLSFLT(flits)) |
1448 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl);
1449 wr_lo = htonl(V_WR_LEN(flits) |
1450 V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token));
1451 set_wr_hdr(wrp, wr_hi, wr_lo);
1452 wmb();
1453 ETHER_BPF_MTAP(pi->ifp, m0);
1454 wr_gen2(txd, txqs.gen);
1455 check_ring_tx_db(sc, txq, 0);
1456 return (0);
1457 } else if (tso_info) {
1458 uint16_t eth_type;
1459 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd;
1460 struct ether_header *eh;
1461 void *l3hdr;
1462 struct tcphdr *tcp;
1463
1464 txd->flit[2] = 0;
1465 GET_VTAG(cntrl, m0);
1466 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1467 hdr->cntrl = htonl(cntrl);
1468 hdr->len = htonl(mlen | 0x80000000);
1469
1470 if (__predict_false(mlen < TCPPKTHDRSIZE)) {
1471 printf("mbuf=%p,len=%d,tso_segsz=%d,csum_flags=%b,flags=%#x",
1472 m0, mlen, m0->m_pkthdr.tso_segsz,
1473 (int)m0->m_pkthdr.csum_flags, CSUM_BITS, m0->m_flags);
1474 panic("tx tso packet too small");
1475 }
1476
1477 /* Make sure that ether, ip, tcp headers are all in m0 */
1478 if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) {
1479 m0 = m_pullup(m0, TCPPKTHDRSIZE);
1480 if (__predict_false(m0 == NULL)) {
1481 /* XXX panic probably an overreaction */
1482 panic("couldn't fit header into mbuf");
1483 }
1484 }
1485
1486 eh = mtod(m0, struct ether_header *);
1487 eth_type = eh->ether_type;
1488 if (eth_type == htons(ETHERTYPE_VLAN)) {
1489 struct ether_vlan_header *evh = (void *)eh;
1490
1491 tso_info |= V_LSO_ETH_TYPE(CPL_ETH_II_VLAN);
1492 l3hdr = evh + 1;
1493 eth_type = evh->evl_proto;
1494 } else {
1495 tso_info |= V_LSO_ETH_TYPE(CPL_ETH_II);
1496 l3hdr = eh + 1;
1497 }
1498
1499 if (eth_type == htons(ETHERTYPE_IP)) {
1500 struct ip *ip = l3hdr;
1501
1502 tso_info |= V_LSO_IPHDR_WORDS(ip->ip_hl);
1503 tcp = (struct tcphdr *)(ip + 1);
1504 } else if (eth_type == htons(ETHERTYPE_IPV6)) {
1505 struct ip6_hdr *ip6 = l3hdr;
1506
1507 KASSERT(ip6->ip6_nxt == IPPROTO_TCP,
1508 ("%s: CSUM_TSO with ip6_nxt %d",
1509 __func__, ip6->ip6_nxt));
1510
1511 tso_info |= F_LSO_IPV6;
1512 tso_info |= V_LSO_IPHDR_WORDS(sizeof(*ip6) >> 2);
1513 tcp = (struct tcphdr *)(ip6 + 1);
1514 } else
1515 panic("%s: CSUM_TSO but neither ip nor ip6", __func__);
1516
1517 tso_info |= V_LSO_TCPHDR_WORDS(tcp->th_off);
1518 hdr->lso_info = htonl(tso_info);
1519
1520 if (__predict_false(mlen <= PIO_LEN)) {
1521 /*
1522 * pkt not undersized but fits in PIO_LEN
1523 * Indicates a TSO bug at the higher levels.
1524 */
1525 txsd->m = NULL;
1526 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[3]);
1527 flits = (mlen + 7) / 8 + 3;
1528 wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
1529 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) |
1530 F_WR_SOP | F_WR_EOP | txqs.compl);
1531 wr_lo = htonl(V_WR_LEN(flits) |
1532 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token));
1533 set_wr_hdr(&hdr->wr, wr_hi, wr_lo);
1534 wmb();
1535 ETHER_BPF_MTAP(pi->ifp, m0);
1536 wr_gen2(txd, txqs.gen);
1537 check_ring_tx_db(sc, txq, 0);
1538 m_freem(m0);
1539 return (0);
1540 }
1541 flits = 3;
1542 } else {
1543 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd;
1544
1545 GET_VTAG(cntrl, m0);
1546 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1547 if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP)))
1548 cntrl |= F_TXPKT_IPCSUM_DIS;
1549 if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP |
1550 CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6))))
1551 cntrl |= F_TXPKT_L4CSUM_DIS;
1552 cpl->cntrl = htonl(cntrl);
1553 cpl->len = htonl(mlen | 0x80000000);
1554
1555 if (mlen <= PIO_LEN) {
1556 txsd->m = NULL;
1557 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
1558 flits = (mlen + 7) / 8 + 2;
1559
1560 wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
1561 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) |
1562 F_WR_SOP | F_WR_EOP | txqs.compl);
1563 wr_lo = htonl(V_WR_LEN(flits) |
1564 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token));
1565 set_wr_hdr(&cpl->wr, wr_hi, wr_lo);
1566 wmb();
1567 ETHER_BPF_MTAP(pi->ifp, m0);
1568 wr_gen2(txd, txqs.gen);
1569 check_ring_tx_db(sc, txq, 0);
1570 m_freem(m0);
1571 return (0);
1572 }
1573 flits = 2;
1574 }
1575 wrp = (struct work_request_hdr *)txd;
1576 sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl;
1577 make_sgl(sgp, segs, nsegs);
1578
1579 sgl_flits = sgl_len(nsegs);
1580
1581 ETHER_BPF_MTAP(pi->ifp, m0);
1582
1583 KASSERT(ndesc <= 4, ("ndesc too large %d", ndesc));
1584 wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl);
1585 wr_lo = htonl(V_WR_TID(txq->token));
1586 write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits,
1587 sgl_flits, wr_hi, wr_lo);
1588 check_ring_tx_db(sc, txq, 0);
1589
1590 return (0);
1591 }
1592
1593 #ifdef DEBUGNET
1594 int
1595 cxgb_debugnet_encap(struct sge_qset *qs, struct mbuf **m)
1596 {
1597 int error;
1598
1599 error = t3_encap(qs, m);
1600 if (error == 0)
1601 check_ring_tx_db(qs->port->adapter, &qs->txq[TXQ_ETH], 1);
1602 else if (*m != NULL) {
1603 m_freem(*m);
1604 *m = NULL;
1605 }
1606 return (error);
1607 }
1608 #endif
1609
1610 void
1611 cxgb_tx_watchdog(void *arg)
1612 {
1613 struct sge_qset *qs = arg;
1614 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1615
1616 if (qs->coalescing != 0 &&
1617 (txq->in_use <= cxgb_tx_coalesce_enable_stop) &&
1618 TXQ_RING_EMPTY(qs))
1619 qs->coalescing = 0;
1620 else if (qs->coalescing == 0 &&
1621 (txq->in_use >= cxgb_tx_coalesce_enable_start))
1622 qs->coalescing = 1;
1623 if (TXQ_TRYLOCK(qs)) {
1624 qs->qs_flags |= QS_FLUSHING;
1625 cxgb_start_locked(qs);
1626 qs->qs_flags &= ~QS_FLUSHING;
1627 TXQ_UNLOCK(qs);
1628 }
1629 if (qs->port->ifp->if_drv_flags & IFF_DRV_RUNNING)
1630 callout_reset_on(&txq->txq_watchdog, hz/4, cxgb_tx_watchdog,
1631 qs, txq->txq_watchdog.c_cpu);
1632 }
1633
1634 static void
1635 cxgb_tx_timeout(void *arg)
1636 {
1637 struct sge_qset *qs = arg;
1638 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1639
1640 if (qs->coalescing == 0 && (txq->in_use >= (txq->size>>3)))
1641 qs->coalescing = 1;
1642 if (TXQ_TRYLOCK(qs)) {
1643 qs->qs_flags |= QS_TIMEOUT;
1644 cxgb_start_locked(qs);
1645 qs->qs_flags &= ~QS_TIMEOUT;
1646 TXQ_UNLOCK(qs);
1647 }
1648 }
1649
1650 static void
1651 cxgb_start_locked(struct sge_qset *qs)
1652 {
1653 struct mbuf *m_head = NULL;
1654 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1655 struct port_info *pi = qs->port;
1656 struct ifnet *ifp = pi->ifp;
1657
1658 if (qs->qs_flags & (QS_FLUSHING|QS_TIMEOUT))
1659 reclaim_completed_tx(qs, 0, TXQ_ETH);
1660
1661 if (!pi->link_config.link_ok) {
1662 TXQ_RING_FLUSH(qs);
1663 return;
1664 }
1665 TXQ_LOCK_ASSERT(qs);
1666 while (!TXQ_RING_EMPTY(qs) && (ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1667 pi->link_config.link_ok) {
1668 reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH);
1669
1670 if (txq->size - txq->in_use <= TX_MAX_DESC)
1671 break;
1672
1673 if ((m_head = cxgb_dequeue(qs)) == NULL)
1674 break;
1675 /*
1676 * Encapsulation can modify our pointer, and or make it
1677 * NULL on failure. In that event, we can't requeue.
1678 */
1679 if (t3_encap(qs, &m_head) || m_head == NULL)
1680 break;
1681
1682 m_head = NULL;
1683 }
1684
1685 if (txq->db_pending)
1686 check_ring_tx_db(pi->adapter, txq, 1);
1687
1688 if (!TXQ_RING_EMPTY(qs) && callout_pending(&txq->txq_timer) == 0 &&
1689 pi->link_config.link_ok)
1690 callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout,
1691 qs, txq->txq_timer.c_cpu);
1692 if (m_head != NULL)
1693 m_freem(m_head);
1694 }
1695
1696 static int
1697 cxgb_transmit_locked(struct ifnet *ifp, struct sge_qset *qs, struct mbuf *m)
1698 {
1699 struct port_info *pi = qs->port;
1700 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1701 struct buf_ring *br = txq->txq_mr;
1702 int error, avail;
1703
1704 avail = txq->size - txq->in_use;
1705 TXQ_LOCK_ASSERT(qs);
1706
1707 /*
1708 * We can only do a direct transmit if the following are true:
1709 * - we aren't coalescing (ring < 3/4 full)
1710 * - the link is up -- checked in caller
1711 * - there are no packets enqueued already
1712 * - there is space in hardware transmit queue
1713 */
1714 if (check_pkt_coalesce(qs) == 0 &&
1715 !TXQ_RING_NEEDS_ENQUEUE(qs) && avail > TX_MAX_DESC) {
1716 if (t3_encap(qs, &m)) {
1717 if (m != NULL &&
1718 (error = drbr_enqueue(ifp, br, m)) != 0)
1719 return (error);
1720 } else {
1721 if (txq->db_pending)
1722 check_ring_tx_db(pi->adapter, txq, 1);
1723
1724 /*
1725 * We've bypassed the buf ring so we need to update
1726 * the stats directly
1727 */
1728 txq->txq_direct_packets++;
1729 txq->txq_direct_bytes += m->m_pkthdr.len;
1730 }
1731 } else if ((error = drbr_enqueue(ifp, br, m)) != 0)
1732 return (error);
1733
1734 reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH);
1735 if (!TXQ_RING_EMPTY(qs) && pi->link_config.link_ok &&
1736 (!check_pkt_coalesce(qs) || (drbr_inuse(ifp, br) >= 7)))
1737 cxgb_start_locked(qs);
1738 else if (!TXQ_RING_EMPTY(qs) && !callout_pending(&txq->txq_timer))
1739 callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout,
1740 qs, txq->txq_timer.c_cpu);
1741 return (0);
1742 }
1743
1744 int
1745 cxgb_transmit(struct ifnet *ifp, struct mbuf *m)
1746 {
1747 struct sge_qset *qs;
1748 struct port_info *pi = ifp->if_softc;
1749 int error, qidx = pi->first_qset;
1750
1751 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0
1752 ||(!pi->link_config.link_ok)) {
1753 m_freem(m);
1754 return (0);
1755 }
1756
1757 /* check if flowid is set */
1758 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1759 qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset;
1760
1761 qs = &pi->adapter->sge.qs[qidx];
1762
1763 if (TXQ_TRYLOCK(qs)) {
1764 /* XXX running */
1765 error = cxgb_transmit_locked(ifp, qs, m);
1766 TXQ_UNLOCK(qs);
1767 } else
1768 error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m);
1769 return (error);
1770 }
1771
1772 void
1773 cxgb_qflush(struct ifnet *ifp)
1774 {
1775 /*
1776 * flush any enqueued mbufs in the buf_rings
1777 * and in the transmit queues
1778 * no-op for now
1779 */
1780 return;
1781 }
1782
1783 /**
1784 * write_imm - write a packet into a Tx descriptor as immediate data
1785 * @d: the Tx descriptor to write
1786 * @m: the packet
1787 * @len: the length of packet data to write as immediate data
1788 * @gen: the generation bit value to write
1789 *
1790 * Writes a packet as immediate data into a Tx descriptor. The packet
1791 * contains a work request at its beginning. We must write the packet
1792 * carefully so the SGE doesn't read accidentally before it's written in
1793 * its entirety.
1794 */
1795 static __inline void
1796 write_imm(struct tx_desc *d, caddr_t src,
1797 unsigned int len, unsigned int gen)
1798 {
1799 struct work_request_hdr *from = (struct work_request_hdr *)src;
1800 struct work_request_hdr *to = (struct work_request_hdr *)d;
1801 uint32_t wr_hi, wr_lo;
1802
1803 KASSERT(len <= WR_LEN && len >= sizeof(*from),
1804 ("%s: invalid len %d", __func__, len));
1805
1806 memcpy(&to[1], &from[1], len - sizeof(*from));
1807 wr_hi = from->wrh_hi | htonl(F_WR_SOP | F_WR_EOP |
1808 V_WR_BCNTLFLT(len & 7));
1809 wr_lo = from->wrh_lo | htonl(V_WR_GEN(gen) | V_WR_LEN((len + 7) / 8));
1810 set_wr_hdr(to, wr_hi, wr_lo);
1811 wmb();
1812 wr_gen2(d, gen);
1813 }
1814
1815 /**
1816 * check_desc_avail - check descriptor availability on a send queue
1817 * @adap: the adapter
1818 * @q: the TX queue
1819 * @m: the packet needing the descriptors
1820 * @ndesc: the number of Tx descriptors needed
1821 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1822 *
1823 * Checks if the requested number of Tx descriptors is available on an
1824 * SGE send queue. If the queue is already suspended or not enough
1825 * descriptors are available the packet is queued for later transmission.
1826 * Must be called with the Tx queue locked.
1827 *
1828 * Returns 0 if enough descriptors are available, 1 if there aren't
1829 * enough descriptors and the packet has been queued, and 2 if the caller
1830 * needs to retry because there weren't enough descriptors at the
1831 * beginning of the call but some freed up in the mean time.
1832 */
1833 static __inline int
1834 check_desc_avail(adapter_t *adap, struct sge_txq *q,
1835 struct mbuf *m, unsigned int ndesc,
1836 unsigned int qid)
1837 {
1838 /*
1839 * XXX We currently only use this for checking the control queue
1840 * the control queue is only used for binding qsets which happens
1841 * at init time so we are guaranteed enough descriptors
1842 */
1843 if (__predict_false(mbufq_len(&q->sendq))) {
1844 addq_exit: (void )mbufq_enqueue(&q->sendq, m);
1845 return 1;
1846 }
1847 if (__predict_false(q->size - q->in_use < ndesc)) {
1848
1849 struct sge_qset *qs = txq_to_qset(q, qid);
1850
1851 setbit(&qs->txq_stopped, qid);
1852 if (should_restart_tx(q) &&
1853 test_and_clear_bit(qid, &qs->txq_stopped))
1854 return 2;
1855
1856 q->stops++;
1857 goto addq_exit;
1858 }
1859 return 0;
1860 }
1861
1862
1863 /**
1864 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1865 * @q: the SGE control Tx queue
1866 *
1867 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1868 * that send only immediate data (presently just the control queues) and
1869 * thus do not have any mbufs
1870 */
1871 static __inline void
1872 reclaim_completed_tx_imm(struct sge_txq *q)
1873 {
1874 unsigned int reclaim = q->processed - q->cleaned;
1875
1876 q->in_use -= reclaim;
1877 q->cleaned += reclaim;
1878 }
1879
1880 /**
1881 * ctrl_xmit - send a packet through an SGE control Tx queue
1882 * @adap: the adapter
1883 * @q: the control queue
1884 * @m: the packet
1885 *
1886 * Send a packet through an SGE control Tx queue. Packets sent through
1887 * a control queue must fit entirely as immediate data in a single Tx
1888 * descriptor and have no page fragments.
1889 */
1890 static int
1891 ctrl_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m)
1892 {
1893 int ret;
1894 struct work_request_hdr *wrp = mtod(m, struct work_request_hdr *);
1895 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1896
1897 KASSERT(m->m_len <= WR_LEN, ("%s: bad tx data", __func__));
1898
1899 wrp->wrh_hi |= htonl(F_WR_SOP | F_WR_EOP);
1900 wrp->wrh_lo = htonl(V_WR_TID(q->token));
1901
1902 TXQ_LOCK(qs);
1903 again: reclaim_completed_tx_imm(q);
1904
1905 ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL);
1906 if (__predict_false(ret)) {
1907 if (ret == 1) {
1908 TXQ_UNLOCK(qs);
1909 return (ENOSPC);
1910 }
1911 goto again;
1912 }
1913 write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen);
1914
1915 q->in_use++;
1916 if (++q->pidx >= q->size) {
1917 q->pidx = 0;
1918 q->gen ^= 1;
1919 }
1920 TXQ_UNLOCK(qs);
1921 wmb();
1922 t3_write_reg(adap, A_SG_KDOORBELL,
1923 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1924
1925 m_free(m);
1926 return (0);
1927 }
1928
1929
1930 /**
1931 * restart_ctrlq - restart a suspended control queue
1932 * @qs: the queue set cotaining the control queue
1933 *
1934 * Resumes transmission on a suspended Tx control queue.
1935 */
1936 static void
1937 restart_ctrlq(void *data, int npending)
1938 {
1939 struct mbuf *m;
1940 struct sge_qset *qs = (struct sge_qset *)data;
1941 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1942 adapter_t *adap = qs->port->adapter;
1943
1944 TXQ_LOCK(qs);
1945 again: reclaim_completed_tx_imm(q);
1946
1947 while (q->in_use < q->size &&
1948 (m = mbufq_dequeue(&q->sendq)) != NULL) {
1949
1950 write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen);
1951 m_free(m);
1952
1953 if (++q->pidx >= q->size) {
1954 q->pidx = 0;
1955 q->gen ^= 1;
1956 }
1957 q->in_use++;
1958 }
1959 if (mbufq_len(&q->sendq)) {
1960 setbit(&qs->txq_stopped, TXQ_CTRL);
1961
1962 if (should_restart_tx(q) &&
1963 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1964 goto again;
1965 q->stops++;
1966 }
1967 TXQ_UNLOCK(qs);
1968 t3_write_reg(adap, A_SG_KDOORBELL,
1969 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1970 }
1971
1972
1973 /*
1974 * Send a management message through control queue 0
1975 */
1976 int
1977 t3_mgmt_tx(struct adapter *adap, struct mbuf *m)
1978 {
1979 return ctrl_xmit(adap, &adap->sge.qs[0], m);
1980 }
1981
1982 /**
1983 * free_qset - free the resources of an SGE queue set
1984 * @sc: the controller owning the queue set
1985 * @q: the queue set
1986 *
1987 * Release the HW and SW resources associated with an SGE queue set, such
1988 * as HW contexts, packet buffers, and descriptor rings. Traffic to the
1989 * queue set must be quiesced prior to calling this.
1990 */
1991 static void
1992 t3_free_qset(adapter_t *sc, struct sge_qset *q)
1993 {
1994 int i;
1995
1996 reclaim_completed_tx(q, 0, TXQ_ETH);
1997 if (q->txq[TXQ_ETH].txq_mr != NULL)
1998 buf_ring_free(q->txq[TXQ_ETH].txq_mr, M_DEVBUF);
1999 if (q->txq[TXQ_ETH].txq_ifq != NULL) {
2000 ifq_delete(q->txq[TXQ_ETH].txq_ifq);
2001 free(q->txq[TXQ_ETH].txq_ifq, M_DEVBUF);
2002 }
2003
2004 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
2005 if (q->fl[i].desc) {
2006 mtx_lock_spin(&sc->sge.reg_lock);
2007 t3_sge_disable_fl(sc, q->fl[i].cntxt_id);
2008 mtx_unlock_spin(&sc->sge.reg_lock);
2009 bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map);
2010 bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc,
2011 q->fl[i].desc_map);
2012 bus_dma_tag_destroy(q->fl[i].desc_tag);
2013 bus_dma_tag_destroy(q->fl[i].entry_tag);
2014 }
2015 if (q->fl[i].sdesc) {
2016 free_rx_bufs(sc, &q->fl[i]);
2017 free(q->fl[i].sdesc, M_DEVBUF);
2018 }
2019 }
2020
2021 mtx_unlock(&q->lock);
2022 MTX_DESTROY(&q->lock);
2023 for (i = 0; i < SGE_TXQ_PER_SET; i++) {
2024 if (q->txq[i].desc) {
2025 mtx_lock_spin(&sc->sge.reg_lock);
2026 t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0);
2027 mtx_unlock_spin(&sc->sge.reg_lock);
2028 bus_dmamap_unload(q->txq[i].desc_tag,
2029 q->txq[i].desc_map);
2030 bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc,
2031 q->txq[i].desc_map);
2032 bus_dma_tag_destroy(q->txq[i].desc_tag);
2033 bus_dma_tag_destroy(q->txq[i].entry_tag);
2034 }
2035 if (q->txq[i].sdesc) {
2036 free(q->txq[i].sdesc, M_DEVBUF);
2037 }
2038 }
2039
2040 if (q->rspq.desc) {
2041 mtx_lock_spin(&sc->sge.reg_lock);
2042 t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id);
2043 mtx_unlock_spin(&sc->sge.reg_lock);
2044
2045 bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map);
2046 bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc,
2047 q->rspq.desc_map);
2048 bus_dma_tag_destroy(q->rspq.desc_tag);
2049 MTX_DESTROY(&q->rspq.lock);
2050 }
2051
2052 #if defined(INET6) || defined(INET)
2053 tcp_lro_free(&q->lro.ctrl);
2054 #endif
2055
2056 bzero(q, sizeof(*q));
2057 }
2058
2059 /**
2060 * t3_free_sge_resources - free SGE resources
2061 * @sc: the adapter softc
2062 *
2063 * Frees resources used by the SGE queue sets.
2064 */
2065 void
2066 t3_free_sge_resources(adapter_t *sc, int nqsets)
2067 {
2068 int i;
2069
2070 for (i = 0; i < nqsets; ++i) {
2071 TXQ_LOCK(&sc->sge.qs[i]);
2072 t3_free_qset(sc, &sc->sge.qs[i]);
2073 }
2074 }
2075
2076 /**
2077 * t3_sge_start - enable SGE
2078 * @sc: the controller softc
2079 *
2080 * Enables the SGE for DMAs. This is the last step in starting packet
2081 * transfers.
2082 */
2083 void
2084 t3_sge_start(adapter_t *sc)
2085 {
2086 t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
2087 }
2088
2089 /**
2090 * t3_sge_stop - disable SGE operation
2091 * @sc: the adapter
2092 *
2093 * Disables the DMA engine. This can be called in emeregencies (e.g.,
2094 * from error interrupts) or from normal process context. In the latter
2095 * case it also disables any pending queue restart tasklets. Note that
2096 * if it is called in interrupt context it cannot disable the restart
2097 * tasklets as it cannot wait, however the tasklets will have no effect
2098 * since the doorbells are disabled and the driver will call this again
2099 * later from process context, at which time the tasklets will be stopped
2100 * if they are still running.
2101 */
2102 void
2103 t3_sge_stop(adapter_t *sc)
2104 {
2105
2106 t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0);
2107 }
2108
2109 /**
2110 * t3_free_tx_desc - reclaims Tx descriptors and their buffers
2111 * @adapter: the adapter
2112 * @q: the Tx queue to reclaim descriptors from
2113 * @reclaimable: the number of descriptors to reclaim
2114 * @m_vec_size: maximum number of buffers to reclaim
2115 * @desc_reclaimed: returns the number of descriptors reclaimed
2116 *
2117 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
2118 * Tx buffers. Called with the Tx queue lock held.
2119 *
2120 * Returns number of buffers of reclaimed
2121 */
2122 void
2123 t3_free_tx_desc(struct sge_qset *qs, int reclaimable, int queue)
2124 {
2125 struct tx_sw_desc *txsd;
2126 unsigned int cidx, mask;
2127 struct sge_txq *q = &qs->txq[queue];
2128
2129 #ifdef T3_TRACE
2130 T3_TRACE2(sc->tb[q->cntxt_id & 7],
2131 "reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx);
2132 #endif
2133 cidx = q->cidx;
2134 mask = q->size - 1;
2135 txsd = &q->sdesc[cidx];
2136
2137 mtx_assert(&qs->lock, MA_OWNED);
2138 while (reclaimable--) {
2139 prefetch(q->sdesc[(cidx + 1) & mask].m);
2140 prefetch(q->sdesc[(cidx + 2) & mask].m);
2141
2142 if (txsd->m != NULL) {
2143 if (txsd->flags & TX_SW_DESC_MAPPED) {
2144 bus_dmamap_unload(q->entry_tag, txsd->map);
2145 txsd->flags &= ~TX_SW_DESC_MAPPED;
2146 }
2147 m_freem_list(txsd->m);
2148 txsd->m = NULL;
2149 } else
2150 q->txq_skipped++;
2151
2152 ++txsd;
2153 if (++cidx == q->size) {
2154 cidx = 0;
2155 txsd = q->sdesc;
2156 }
2157 }
2158 q->cidx = cidx;
2159
2160 }
2161
2162 /**
2163 * is_new_response - check if a response is newly written
2164 * @r: the response descriptor
2165 * @q: the response queue
2166 *
2167 * Returns true if a response descriptor contains a yet unprocessed
2168 * response.
2169 */
2170 static __inline int
2171 is_new_response(const struct rsp_desc *r,
2172 const struct sge_rspq *q)
2173 {
2174 return (r->intr_gen & F_RSPD_GEN2) == q->gen;
2175 }
2176
2177 #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
2178 #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
2179 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
2180 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
2181 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
2182
2183 /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
2184 #define NOMEM_INTR_DELAY 2500
2185
2186 #ifdef TCP_OFFLOAD
2187 /**
2188 * write_ofld_wr - write an offload work request
2189 * @adap: the adapter
2190 * @m: the packet to send
2191 * @q: the Tx queue
2192 * @pidx: index of the first Tx descriptor to write
2193 * @gen: the generation value to use
2194 * @ndesc: number of descriptors the packet will occupy
2195 *
2196 * Write an offload work request to send the supplied packet. The packet
2197 * data already carry the work request with most fields populated.
2198 */
2199 static void
2200 write_ofld_wr(adapter_t *adap, struct mbuf *m, struct sge_txq *q,
2201 unsigned int pidx, unsigned int gen, unsigned int ndesc)
2202 {
2203 unsigned int sgl_flits, flits;
2204 int i, idx, nsegs, wrlen;
2205 struct work_request_hdr *from;
2206 struct sg_ent *sgp, t3sgl[TX_MAX_SEGS / 2 + 1];
2207 struct tx_desc *d = &q->desc[pidx];
2208 struct txq_state txqs;
2209 struct sglist_seg *segs;
2210 struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
2211 struct sglist *sgl;
2212
2213 from = (void *)(oh + 1); /* Start of WR within mbuf */
2214 wrlen = m->m_len - sizeof(*oh);
2215
2216 if (!(oh->flags & F_HDR_SGL)) {
2217 write_imm(d, (caddr_t)from, wrlen, gen);
2218
2219 /*
2220 * mbuf with "real" immediate tx data will be enqueue_wr'd by
2221 * t3_push_frames and freed in wr_ack. Others, like those sent
2222 * down by close_conn, t3_send_reset, etc. should be freed here.
2223 */
2224 if (!(oh->flags & F_HDR_DF))
2225 m_free(m);
2226 return;
2227 }
2228
2229 memcpy(&d->flit[1], &from[1], wrlen - sizeof(*from));
2230
2231 sgl = oh->sgl;
2232 flits = wrlen / 8;
2233 sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : t3sgl;
2234
2235 nsegs = sgl->sg_nseg;
2236 segs = sgl->sg_segs;
2237 for (idx = 0, i = 0; i < nsegs; i++) {
2238 KASSERT(segs[i].ss_len, ("%s: 0 len in sgl", __func__));
2239 if (i && idx == 0)
2240 ++sgp;
2241 sgp->len[idx] = htobe32(segs[i].ss_len);
2242 sgp->addr[idx] = htobe64(segs[i].ss_paddr);
2243 idx ^= 1;
2244 }
2245 if (idx) {
2246 sgp->len[idx] = 0;
2247 sgp->addr[idx] = 0;
2248 }
2249
2250 sgl_flits = sgl_len(nsegs);
2251 txqs.gen = gen;
2252 txqs.pidx = pidx;
2253 txqs.compl = 0;
2254
2255 write_wr_hdr_sgl(ndesc, d, &txqs, q, t3sgl, flits, sgl_flits,
2256 from->wrh_hi, from->wrh_lo);
2257 }
2258
2259 /**
2260 * ofld_xmit - send a packet through an offload queue
2261 * @adap: the adapter
2262 * @q: the Tx offload queue
2263 * @m: the packet
2264 *
2265 * Send an offload packet through an SGE offload queue.
2266 */
2267 static int
2268 ofld_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m)
2269 {
2270 int ret;
2271 unsigned int ndesc;
2272 unsigned int pidx, gen;
2273 struct sge_txq *q = &qs->txq[TXQ_OFLD];
2274 struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
2275
2276 ndesc = G_HDR_NDESC(oh->flags);
2277
2278 TXQ_LOCK(qs);
2279 again: reclaim_completed_tx(qs, 16, TXQ_OFLD);
2280 ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD);
2281 if (__predict_false(ret)) {
2282 if (ret == 1) {
2283 TXQ_UNLOCK(qs);
2284 return (EINTR);
2285 }
2286 goto again;
2287 }
2288
2289 gen = q->gen;
2290 q->in_use += ndesc;
2291 pidx = q->pidx;
2292 q->pidx += ndesc;
2293 if (q->pidx >= q->size) {
2294 q->pidx -= q->size;
2295 q->gen ^= 1;
2296 }
2297
2298 write_ofld_wr(adap, m, q, pidx, gen, ndesc);
2299 check_ring_tx_db(adap, q, 1);
2300 TXQ_UNLOCK(qs);
2301
2302 return (0);
2303 }
2304
2305 /**
2306 * restart_offloadq - restart a suspended offload queue
2307 * @qs: the queue set cotaining the offload queue
2308 *
2309 * Resumes transmission on a suspended Tx offload queue.
2310 */
2311 static void
2312 restart_offloadq(void *data, int npending)
2313 {
2314 struct mbuf *m;
2315 struct sge_qset *qs = data;
2316 struct sge_txq *q = &qs->txq[TXQ_OFLD];
2317 adapter_t *adap = qs->port->adapter;
2318
2319 TXQ_LOCK(qs);
2320 again:
2321 while ((m = mbufq_first(&q->sendq)) != NULL) {
2322 unsigned int gen, pidx;
2323 struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
2324 unsigned int ndesc = G_HDR_NDESC(oh->flags);
2325
2326 if (__predict_false(q->size - q->in_use < ndesc)) {
2327 setbit(&qs->txq_stopped, TXQ_OFLD);
2328 if (should_restart_tx(q) &&
2329 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
2330 goto again;
2331 q->stops++;
2332 break;
2333 }
2334
2335 gen = q->gen;
2336 q->in_use += ndesc;
2337 pidx = q->pidx;
2338 q->pidx += ndesc;
2339 if (q->pidx >= q->size) {
2340 q->pidx -= q->size;
2341 q->gen ^= 1;
2342 }
2343
2344 (void)mbufq_dequeue(&q->sendq);
2345 TXQ_UNLOCK(qs);
2346 write_ofld_wr(adap, m, q, pidx, gen, ndesc);
2347 TXQ_LOCK(qs);
2348 }
2349 #if USE_GTS
2350 set_bit(TXQ_RUNNING, &q->flags);
2351 set_bit(TXQ_LAST_PKT_DB, &q->flags);
2352 #endif
2353 TXQ_UNLOCK(qs);
2354 wmb();
2355 t3_write_reg(adap, A_SG_KDOORBELL,
2356 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
2357 }
2358
2359 /**
2360 * t3_offload_tx - send an offload packet
2361 * @m: the packet
2362 *
2363 * Sends an offload packet. We use the packet priority to select the
2364 * appropriate Tx queue as follows: bit 0 indicates whether the packet
2365 * should be sent as regular or control, bits 1-3 select the queue set.
2366 */
2367 int
2368 t3_offload_tx(struct adapter *sc, struct mbuf *m)
2369 {
2370 struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
2371 struct sge_qset *qs = &sc->sge.qs[G_HDR_QSET(oh->flags)];
2372
2373 if (oh->flags & F_HDR_CTRL) {
2374 m_adj(m, sizeof (*oh)); /* trim ofld_hdr off */
2375 return (ctrl_xmit(sc, qs, m));
2376 } else
2377 return (ofld_xmit(sc, qs, m));
2378 }
2379 #endif
2380
2381 static void
2382 restart_tx(struct sge_qset *qs)
2383 {
2384 struct adapter *sc = qs->port->adapter;
2385
2386 if (isset(&qs->txq_stopped, TXQ_OFLD) &&
2387 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
2388 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
2389 qs->txq[TXQ_OFLD].restarts++;
2390 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
2391 }
2392
2393 if (isset(&qs->txq_stopped, TXQ_CTRL) &&
2394 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
2395 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
2396 qs->txq[TXQ_CTRL].restarts++;
2397 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
2398 }
2399 }
2400
2401 /**
2402 * t3_sge_alloc_qset - initialize an SGE queue set
2403 * @sc: the controller softc
2404 * @id: the queue set id
2405 * @nports: how many Ethernet ports will be using this queue set
2406 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2407 * @p: configuration parameters for this queue set
2408 * @ntxq: number of Tx queues for the queue set
2409 * @pi: port info for queue set
2410 *
2411 * Allocate resources and initialize an SGE queue set. A queue set
2412 * comprises a response queue, two Rx free-buffer queues, and up to 3
2413 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2414 * queue, offload queue, and control queue.
2415 */
2416 int
2417 t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
2418 const struct qset_params *p, int ntxq, struct port_info *pi)
2419 {
2420 struct sge_qset *q = &sc->sge.qs[id];
2421 int i, ret = 0;
2422
2423 MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF);
2424 q->port = pi;
2425 q->adap = sc;
2426
2427 if ((q->txq[TXQ_ETH].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size,
2428 M_DEVBUF, M_WAITOK, &q->lock)) == NULL) {
2429 device_printf(sc->dev, "failed to allocate mbuf ring\n");
2430 goto err;
2431 }
2432 if ((q->txq[TXQ_ETH].txq_ifq = malloc(sizeof(struct ifaltq), M_DEVBUF,
2433 M_NOWAIT | M_ZERO)) == NULL) {
2434 device_printf(sc->dev, "failed to allocate ifq\n");
2435 goto err;
2436 }
2437 ifq_init(q->txq[TXQ_ETH].txq_ifq, pi->ifp);
2438 callout_init(&q->txq[TXQ_ETH].txq_timer, 1);
2439 callout_init(&q->txq[TXQ_ETH].txq_watchdog, 1);
2440 q->txq[TXQ_ETH].txq_timer.c_cpu = id % mp_ncpus;
2441 q->txq[TXQ_ETH].txq_watchdog.c_cpu = id % mp_ncpus;
2442
2443 init_qset_cntxt(q, id);
2444 q->idx = id;
2445 if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc),
2446 sizeof(struct rx_sw_desc), &q->fl[0].phys_addr,
2447 &q->fl[0].desc, &q->fl[0].sdesc,
2448 &q->fl[0].desc_tag, &q->fl[0].desc_map,
2449 sc->rx_dmat, &q->fl[0].entry_tag)) != 0) {
2450 printf("error %d from alloc ring fl0\n", ret);
2451 goto err;
2452 }
2453
2454 if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc),
2455 sizeof(struct rx_sw_desc), &q->fl[1].phys_addr,
2456 &q->fl[1].desc, &q->fl[1].sdesc,
2457 &q->fl[1].desc_tag, &q->fl[1].desc_map,
2458 sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) {
2459 printf("error %d from alloc ring fl1\n", ret);
2460 goto err;
2461 }
2462
2463 if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0,
2464 &q->rspq.phys_addr, &q->rspq.desc, NULL,
2465 &q->rspq.desc_tag, &q->rspq.desc_map,
2466 NULL, NULL)) != 0) {
2467 printf("error %d from alloc ring rspq\n", ret);
2468 goto err;
2469 }
2470
2471 snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d",
2472 device_get_unit(sc->dev), irq_vec_idx);
2473 MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF);
2474
2475 for (i = 0; i < ntxq; ++i) {
2476 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
2477
2478 if ((ret = alloc_ring(sc, p->txq_size[i],
2479 sizeof(struct tx_desc), sz,
2480 &q->txq[i].phys_addr, &q->txq[i].desc,
2481 &q->txq[i].sdesc, &q->txq[i].desc_tag,
2482 &q->txq[i].desc_map,
2483 sc->tx_dmat, &q->txq[i].entry_tag)) != 0) {
2484 printf("error %d from alloc ring tx %i\n", ret, i);
2485 goto err;
2486 }
2487 mbufq_init(&q->txq[i].sendq, INT_MAX);
2488 q->txq[i].gen = 1;
2489 q->txq[i].size = p->txq_size[i];
2490 }
2491
2492 #ifdef TCP_OFFLOAD
2493 TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q);
2494 #endif
2495 TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q);
2496 TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, q);
2497 TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, q);
2498
2499 q->fl[0].gen = q->fl[1].gen = 1;
2500 q->fl[0].size = p->fl_size;
2501 q->fl[1].size = p->jumbo_size;
2502
2503 q->rspq.gen = 1;
2504 q->rspq.cidx = 0;
2505 q->rspq.size = p->rspq_size;
2506
2507 q->txq[TXQ_ETH].stop_thres = nports *
2508 flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
2509
2510 q->fl[0].buf_size = MCLBYTES;
2511 q->fl[0].zone = zone_pack;
2512 q->fl[0].type = EXT_PACKET;
2513
2514 if (p->jumbo_buf_size == MJUM16BYTES) {
2515 q->fl[1].zone = zone_jumbo16;
2516 q->fl[1].type = EXT_JUMBO16;
2517 } else if (p->jumbo_buf_size == MJUM9BYTES) {
2518 q->fl[1].zone = zone_jumbo9;
2519 q->fl[1].type = EXT_JUMBO9;
2520 } else if (p->jumbo_buf_size == MJUMPAGESIZE) {
2521 q->fl[1].zone = zone_jumbop;
2522 q->fl[1].type = EXT_JUMBOP;
2523 } else {
2524 KASSERT(0, ("can't deal with jumbo_buf_size %d.", p->jumbo_buf_size));
2525 ret = EDOOFUS;
2526 goto err;
2527 }
2528 q->fl[1].buf_size = p->jumbo_buf_size;
2529
2530 /* Allocate and setup the lro_ctrl structure */
2531 q->lro.enabled = !!(pi->ifp->if_capenable & IFCAP_LRO);
2532 #if defined(INET6) || defined(INET)
2533 ret = tcp_lro_init(&q->lro.ctrl);
2534 if (ret) {
2535 printf("error %d from tcp_lro_init\n", ret);
2536 goto err;
2537 }
2538 #endif
2539 q->lro.ctrl.ifp = pi->ifp;
2540
2541 mtx_lock_spin(&sc->sge.reg_lock);
2542 ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
2543 q->rspq.phys_addr, q->rspq.size,
2544 q->fl[0].buf_size, 1, 0);
2545 if (ret) {
2546 printf("error %d from t3_sge_init_rspcntxt\n", ret);
2547 goto err_unlock;
2548 }
2549
2550 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
2551 ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0,
2552 q->fl[i].phys_addr, q->fl[i].size,
2553 q->fl[i].buf_size, p->cong_thres, 1,
2554 0);
2555 if (ret) {
2556 printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i);
2557 goto err_unlock;
2558 }
2559 }
2560
2561 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
2562 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
2563 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
2564 1, 0);
2565 if (ret) {
2566 printf("error %d from t3_sge_init_ecntxt\n", ret);
2567 goto err_unlock;
2568 }
2569
2570 if (ntxq > 1) {
2571 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id,
2572 USE_GTS, SGE_CNTXT_OFLD, id,
2573 q->txq[TXQ_OFLD].phys_addr,
2574 q->txq[TXQ_OFLD].size, 0, 1, 0);
2575 if (ret) {
2576 printf("error %d from t3_sge_init_ecntxt\n", ret);
2577 goto err_unlock;
2578 }
2579 }
2580
2581 if (ntxq > 2) {
2582 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0,
2583 SGE_CNTXT_CTRL, id,
2584 q->txq[TXQ_CTRL].phys_addr,
2585 q->txq[TXQ_CTRL].size,
2586 q->txq[TXQ_CTRL].token, 1, 0);
2587 if (ret) {
2588 printf("error %d from t3_sge_init_ecntxt\n", ret);
2589 goto err_unlock;
2590 }
2591 }
2592
2593 mtx_unlock_spin(&sc->sge.reg_lock);
2594 t3_update_qset_coalesce(q, p);
2595
2596 refill_fl(sc, &q->fl[0], q->fl[0].size);
2597 refill_fl(sc, &q->fl[1], q->fl[1].size);
2598 refill_rspq(sc, &q->rspq, q->rspq.size - 1);
2599
2600 t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
2601 V_NEWTIMER(q->rspq.holdoff_tmr));
2602
2603 return (0);
2604
2605 err_unlock:
2606 mtx_unlock_spin(&sc->sge.reg_lock);
2607 err:
2608 TXQ_LOCK(q);
2609 t3_free_qset(sc, q);
2610
2611 return (ret);
2612 }
2613
2614 /*
2615 * Remove CPL_RX_PKT headers from the mbuf and reduce it to a regular mbuf with
2616 * ethernet data. Hardware assistance with various checksums and any vlan tag
2617 * will also be taken into account here.
2618 */
2619 void
2620 t3_rx_eth(struct adapter *adap, struct mbuf *m, int ethpad)
2621 {
2622 struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad);
2623 struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]];
2624 struct ifnet *ifp = pi->ifp;
2625
2626 if (cpl->vlan_valid) {
2627 m->m_pkthdr.ether_vtag = ntohs(cpl->vlan);
2628 m->m_flags |= M_VLANTAG;
2629 }
2630
2631 m->m_pkthdr.rcvif = ifp;
2632 /*
2633 * adjust after conversion to mbuf chain
2634 */
2635 m->m_pkthdr.len -= (sizeof(*cpl) + ethpad);
2636 m->m_len -= (sizeof(*cpl) + ethpad);
2637 m->m_data += (sizeof(*cpl) + ethpad);
2638
2639 if (!cpl->fragment && cpl->csum_valid && cpl->csum == 0xffff) {
2640 struct ether_header *eh = mtod(m, void *);
2641 uint16_t eh_type;
2642
2643 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2644 struct ether_vlan_header *evh = mtod(m, void *);
2645
2646 eh_type = evh->evl_proto;
2647 } else
2648 eh_type = eh->ether_type;
2649
2650 if (ifp->if_capenable & IFCAP_RXCSUM &&
2651 eh_type == htons(ETHERTYPE_IP)) {
2652 m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED |
2653 CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2654 m->m_pkthdr.csum_data = 0xffff;
2655 } else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 &&
2656 eh_type == htons(ETHERTYPE_IPV6)) {
2657 m->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 |
2658 CSUM_PSEUDO_HDR);
2659 m->m_pkthdr.csum_data = 0xffff;
2660 }
2661 }
2662 }
2663
2664 /**
2665 * get_packet - return the next ingress packet buffer from a free list
2666 * @adap: the adapter that received the packet
2667 * @drop_thres: # of remaining buffers before we start dropping packets
2668 * @qs: the qset that the SGE free list holding the packet belongs to
2669 * @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain
2670 * @r: response descriptor
2671 *
2672 * Get the next packet from a free list and complete setup of the
2673 * sk_buff. If the packet is small we make a copy and recycle the
2674 * original buffer, otherwise we use the original buffer itself. If a
2675 * positive drop threshold is supplied packets are dropped and their
2676 * buffers recycled if (a) the number of remaining buffers is under the
2677 * threshold and the packet is too big to copy, or (b) the packet should
2678 * be copied but there is no memory for the copy.
2679 */
2680 static int
2681 get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
2682 struct t3_mbuf_hdr *mh, struct rsp_desc *r)
2683 {
2684
2685 unsigned int len_cq = ntohl(r->len_cq);
2686 struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
2687 int mask, cidx = fl->cidx;
2688 struct rx_sw_desc *sd = &fl->sdesc[cidx];
2689 uint32_t len = G_RSPD_LEN(len_cq);
2690 uint32_t flags = M_EXT;
2691 uint8_t sopeop = G_RSPD_SOP_EOP(ntohl(r->flags));
2692 caddr_t cl;
2693 struct mbuf *m;
2694 int ret = 0;
2695
2696 mask = fl->size - 1;
2697 prefetch(fl->sdesc[(cidx + 1) & mask].m);
2698 prefetch(fl->sdesc[(cidx + 2) & mask].m);
2699 prefetch(fl->sdesc[(cidx + 1) & mask].rxsd_cl);
2700 prefetch(fl->sdesc[(cidx + 2) & mask].rxsd_cl);
2701
2702 fl->credits--;
2703 bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
2704
2705 if (recycle_enable && len <= SGE_RX_COPY_THRES &&
2706 sopeop == RSPQ_SOP_EOP) {
2707 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
2708 goto skip_recycle;
2709 cl = mtod(m, void *);
2710 memcpy(cl, sd->rxsd_cl, len);
2711 recycle_rx_buf(adap, fl, fl->cidx);
2712 m->m_pkthdr.len = m->m_len = len;
2713 m->m_flags = 0;
2714 mh->mh_head = mh->mh_tail = m;
2715 ret = 1;
2716 goto done;
2717 } else {
2718 skip_recycle:
2719 bus_dmamap_unload(fl->entry_tag, sd->map);
2720 cl = sd->rxsd_cl;
2721 m = sd->m;
2722
2723 if ((sopeop == RSPQ_SOP_EOP) ||
2724 (sopeop == RSPQ_SOP))
2725 flags |= M_PKTHDR;
2726 m_init(m, M_NOWAIT, MT_DATA, flags);
2727 if (fl->zone == zone_pack) {
2728 /*
2729 * restore clobbered data pointer
2730 */
2731 m->m_data = m->m_ext.ext_buf;
2732 } else {
2733 m_cljset(m, cl, fl->type);
2734 }
2735 m->m_len = len;
2736 }
2737 switch(sopeop) {
2738 case RSPQ_SOP_EOP:
2739 ret = 1;
2740 /* FALLTHROUGH */
2741 case RSPQ_SOP:
2742 mh->mh_head = mh->mh_tail = m;
2743 m->m_pkthdr.len = len;
2744 break;
2745 case RSPQ_EOP:
2746 ret = 1;
2747 /* FALLTHROUGH */
2748 case RSPQ_NSOP_NEOP:
2749 if (mh->mh_tail == NULL) {
2750 log(LOG_ERR, "discarding intermediate descriptor entry\n");
2751 m_freem(m);
2752 m = NULL;
2753 break;
2754 }
2755 mh->mh_tail->m_next = m;
2756 mh->mh_tail = m;
2757 mh->mh_head->m_pkthdr.len += len;
2758 break;
2759 }
2760 if (cxgb_debug && m != NULL)
2761 printf("len=%d pktlen=%d\n", m->m_len, m->m_pkthdr.len);
2762 done:
2763 if (++fl->cidx == fl->size)
2764 fl->cidx = 0;
2765
2766 return (ret);
2767 }
2768
2769 /**
2770 * handle_rsp_cntrl_info - handles control information in a response
2771 * @qs: the queue set corresponding to the response
2772 * @flags: the response control flags
2773 *
2774 * Handles the control information of an SGE response, such as GTS
2775 * indications and completion credits for the queue set's Tx queues.
2776 * HW coalesces credits, we don't do any extra SW coalescing.
2777 */
2778 static __inline void
2779 handle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags)
2780 {
2781 unsigned int credits;
2782
2783 #if USE_GTS
2784 if (flags & F_RSPD_TXQ0_GTS)
2785 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
2786 #endif
2787 credits = G_RSPD_TXQ0_CR(flags);
2788 if (credits)
2789 qs->txq[TXQ_ETH].processed += credits;
2790
2791 credits = G_RSPD_TXQ2_CR(flags);
2792 if (credits)
2793 qs->txq[TXQ_CTRL].processed += credits;
2794
2795 # if USE_GTS
2796 if (flags & F_RSPD_TXQ1_GTS)
2797 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
2798 # endif
2799 credits = G_RSPD_TXQ1_CR(flags);
2800 if (credits)
2801 qs->txq[TXQ_OFLD].processed += credits;
2802
2803 }
2804
2805 static void
2806 check_ring_db(adapter_t *adap, struct sge_qset *qs,
2807 unsigned int sleeping)
2808 {
2809 ;
2810 }
2811
2812 /**
2813 * process_responses - process responses from an SGE response queue
2814 * @adap: the adapter
2815 * @qs: the queue set to which the response queue belongs
2816 * @budget: how many responses can be processed in this round
2817 *
2818 * Process responses from an SGE response queue up to the supplied budget.
2819 * Responses include received packets as well as credits and other events
2820 * for the queues that belong to the response queue's queue set.
2821 * A negative budget is effectively unlimited.
2822 *
2823 * Additionally choose the interrupt holdoff time for the next interrupt
2824 * on this queue. If the system is under memory shortage use a fairly
2825 * long delay to help recovery.
2826 */
2827 static int
2828 process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
2829 {
2830 struct sge_rspq *rspq = &qs->rspq;
2831 struct rsp_desc *r = &rspq->desc[rspq->cidx];
2832 int budget_left = budget;
2833 unsigned int sleeping = 0;
2834 #if defined(INET6) || defined(INET)
2835 int lro_enabled = qs->lro.enabled;
2836 int skip_lro;
2837 struct lro_ctrl *lro_ctrl = &qs->lro.ctrl;
2838 #endif
2839 struct t3_mbuf_hdr *mh = &rspq->rspq_mh;
2840 #ifdef DEBUG
2841 static int last_holdoff = 0;
2842 if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) {
2843 printf("next_holdoff=%d\n", rspq->holdoff_tmr);
2844 last_holdoff = rspq->holdoff_tmr;
2845 }
2846 #endif
2847 rspq->next_holdoff = rspq->holdoff_tmr;
2848
2849 while (__predict_true(budget_left && is_new_response(r, rspq))) {
2850 int eth, eop = 0, ethpad = 0;
2851 uint32_t flags = ntohl(r->flags);
2852 uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val);
2853 uint8_t opcode = r->rss_hdr.opcode;
2854
2855 eth = (opcode == CPL_RX_PKT);
2856
2857 if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) {
2858 struct mbuf *m;
2859
2860 if (cxgb_debug)
2861 printf("async notification\n");
2862
2863 if (mh->mh_head == NULL) {
2864 mh->mh_head = m_gethdr(M_NOWAIT, MT_DATA);
2865 m = mh->mh_head;
2866 } else {
2867 m = m_gethdr(M_NOWAIT, MT_DATA);
2868 }
2869 if (m == NULL)
2870 goto no_mem;
2871
2872 memcpy(mtod(m, char *), r, AN_PKT_SIZE);
2873 m->m_len = m->m_pkthdr.len = AN_PKT_SIZE;
2874 *mtod(m, uint8_t *) = CPL_ASYNC_NOTIF;
2875 opcode = CPL_ASYNC_NOTIF;
2876 eop = 1;
2877 rspq->async_notif++;
2878 goto skip;
2879 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2880 struct mbuf *m = m_gethdr(M_NOWAIT, MT_DATA);
2881
2882 if (m == NULL) {
2883 no_mem:
2884 rspq->next_holdoff = NOMEM_INTR_DELAY;
2885 budget_left--;
2886 break;
2887 }
2888 if (mh->mh_head == NULL)
2889 mh->mh_head = m;
2890 else
2891 mh->mh_tail->m_next = m;
2892 mh->mh_tail = m;
2893
2894 get_imm_packet(adap, r, m);
2895 mh->mh_head->m_pkthdr.len += m->m_len;
2896 eop = 1;
2897 rspq->imm_data++;
2898 } else if (r->len_cq) {
2899 int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
2900
2901 eop = get_packet(adap, drop_thresh, qs, mh, r);
2902 if (eop) {
2903 if (r->rss_hdr.hash_type && !adap->timestamp) {
2904 M_HASHTYPE_SET(mh->mh_head,
2905 M_HASHTYPE_OPAQUE_HASH);
2906 mh->mh_head->m_pkthdr.flowid = rss_hash;
2907 }
2908 }
2909
2910 ethpad = 2;
2911 } else {
2912 rspq->pure_rsps++;
2913 }
2914 skip:
2915 if (flags & RSPD_CTRL_MASK) {
2916 sleeping |= flags & RSPD_GTS_MASK;
2917 handle_rsp_cntrl_info(qs, flags);
2918 }
2919
2920 if (!eth && eop) {
2921 rspq->offload_pkts++;
2922 #ifdef TCP_OFFLOAD
2923 adap->cpl_handler[opcode](qs, r, mh->mh_head);
2924 #else
2925 m_freem(mh->mh_head);
2926 #endif
2927 mh->mh_head = NULL;
2928 } else if (eth && eop) {
2929 struct mbuf *m = mh->mh_head;
2930
2931 t3_rx_eth(adap, m, ethpad);
2932
2933 /*
2934 * The T304 sends incoming packets on any qset. If LRO
2935 * is also enabled, we could end up sending packet up
2936 * lro_ctrl->ifp's input. That is incorrect.
2937 *
2938 * The mbuf's rcvif was derived from the cpl header and
2939 * is accurate. Skip LRO and just use that.
2940 */
2941 #if defined(INET6) || defined(INET)
2942 skip_lro = __predict_false(qs->port->ifp != m->m_pkthdr.rcvif);
2943
2944 if (lro_enabled && lro_ctrl->lro_cnt && !skip_lro
2945 && (tcp_lro_rx(lro_ctrl, m, 0) == 0)
2946 ) {
2947 /* successfully queue'd for LRO */
2948 } else
2949 #endif
2950 {
2951 /*
2952 * LRO not enabled, packet unsuitable for LRO,
2953 * or unable to queue. Pass it up right now in
2954 * either case.
2955 */
2956 struct ifnet *ifp = m->m_pkthdr.rcvif;
2957 (*ifp->if_input)(ifp, m);
2958 }
2959 mh->mh_head = NULL;
2960
2961 }
2962
2963 r++;
2964 if (__predict_false(++rspq->cidx == rspq->size)) {
2965 rspq->cidx = 0;
2966 rspq->gen ^= 1;
2967 r = rspq->desc;
2968 }
2969
2970 if (++rspq->credits >= 64) {
2971 refill_rspq(adap, rspq, rspq->credits);
2972 rspq->credits = 0;
2973 }
2974 __refill_fl_lt(adap, &qs->fl[0], 32);
2975 __refill_fl_lt(adap, &qs->fl[1], 32);
2976 --budget_left;
2977 }
2978
2979 #if defined(INET6) || defined(INET)
2980 /* Flush LRO */
2981 tcp_lro_flush_all(lro_ctrl);
2982 #endif
2983
2984 if (sleeping)
2985 check_ring_db(adap, qs, sleeping);
2986
2987 mb(); /* commit Tx queue processed updates */
2988 if (__predict_false(qs->txq_stopped > 1))
2989 restart_tx(qs);
2990
2991 __refill_fl_lt(adap, &qs->fl[0], 512);
2992 __refill_fl_lt(adap, &qs->fl[1], 512);
2993 budget -= budget_left;
2994 return (budget);
2995 }
2996
2997 /*
2998 * A helper function that processes responses and issues GTS.
2999 */
3000 static __inline int
3001 process_responses_gts(adapter_t *adap, struct sge_rspq *rq)
3002 {
3003 int work;
3004 static int last_holdoff = 0;
3005
3006 work = process_responses(adap, rspq_to_qset(rq), -1);
3007
3008 if (cxgb_debug && (rq->next_holdoff != last_holdoff)) {
3009 printf("next_holdoff=%d\n", rq->next_holdoff);
3010 last_holdoff = rq->next_holdoff;
3011 }
3012 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
3013 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
3014
3015 return (work);
3016 }
3017
3018 #ifdef DEBUGNET
3019 int
3020 cxgb_debugnet_poll_rx(adapter_t *adap, struct sge_qset *qs)
3021 {
3022
3023 return (process_responses_gts(adap, &qs->rspq));
3024 }
3025 #endif
3026
3027 /*
3028 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
3029 * Handles data events from SGE response queues as well as error and other
3030 * async events as they all use the same interrupt pin. We use one SGE
3031 * response queue per port in this mode and protect all response queues with
3032 * queue 0's lock.
3033 */
3034 void
3035 t3b_intr(void *data)
3036 {
3037 uint32_t i, map;
3038 adapter_t *adap = data;
3039 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
3040
3041 t3_write_reg(adap, A_PL_CLI, 0);
3042 map = t3_read_reg(adap, A_SG_DATA_INTR);
3043
3044 if (!map)
3045 return;
3046
3047 if (__predict_false(map & F_ERRINTR)) {
3048 t3_write_reg(adap, A_PL_INT_ENABLE0, 0);
3049 (void) t3_read_reg(adap, A_PL_INT_ENABLE0);
3050 taskqueue_enqueue(adap->tq, &adap->slow_intr_task);
3051 }
3052
3053 mtx_lock(&q0->lock);
3054 for_each_port(adap, i)
3055 if (map & (1 << i))
3056 process_responses_gts(adap, &adap->sge.qs[i].rspq);
3057 mtx_unlock(&q0->lock);
3058 }
3059
3060 /*
3061 * The MSI interrupt handler. This needs to handle data events from SGE
3062 * response queues as well as error and other async events as they all use
3063 * the same MSI vector. We use one SGE response queue per port in this mode
3064 * and protect all response queues with queue 0's lock.
3065 */
3066 void
3067 t3_intr_msi(void *data)
3068 {
3069 adapter_t *adap = data;
3070 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
3071 int i, new_packets = 0;
3072
3073 mtx_lock(&q0->lock);
3074
3075 for_each_port(adap, i)
3076 if (process_responses_gts(adap, &adap->sge.qs[i].rspq))
3077 new_packets = 1;
3078 mtx_unlock(&q0->lock);
3079 if (new_packets == 0) {
3080 t3_write_reg(adap, A_PL_INT_ENABLE0, 0);
3081 (void) t3_read_reg(adap, A_PL_INT_ENABLE0);
3082 taskqueue_enqueue(adap->tq, &adap->slow_intr_task);
3083 }
3084 }
3085
3086 void
3087 t3_intr_msix(void *data)
3088 {
3089 struct sge_qset *qs = data;
3090 adapter_t *adap = qs->port->adapter;
3091 struct sge_rspq *rspq = &qs->rspq;
3092
3093 if (process_responses_gts(adap, rspq) == 0)
3094 rspq->unhandled_irqs++;
3095 }
3096
3097 #define QDUMP_SBUF_SIZE 32 * 400
3098 static int
3099 t3_dump_rspq(SYSCTL_HANDLER_ARGS)
3100 {
3101 struct sge_rspq *rspq;
3102 struct sge_qset *qs;
3103 int i, err, dump_end, idx;
3104 struct sbuf *sb;
3105 struct rsp_desc *rspd;
3106 uint32_t data[4];
3107
3108 rspq = arg1;
3109 qs = rspq_to_qset(rspq);
3110 if (rspq->rspq_dump_count == 0)
3111 return (0);
3112 if (rspq->rspq_dump_count > RSPQ_Q_SIZE) {
3113 log(LOG_WARNING,
3114 "dump count is too large %d\n", rspq->rspq_dump_count);
3115 rspq->rspq_dump_count = 0;
3116 return (EINVAL);
3117 }
3118 if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) {
3119 log(LOG_WARNING,
3120 "dump start of %d is greater than queue size\n",
3121 rspq->rspq_dump_start);
3122 rspq->rspq_dump_start = 0;
3123 return (EINVAL);
3124 }
3125 err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data);
3126 if (err)
3127 return (err);
3128 err = sysctl_wire_old_buffer(req, 0);
3129 if (err)
3130 return (err);
3131 sb = sbuf_new_for_sysctl(NULL, NULL, QDUMP_SBUF_SIZE, req);
3132
3133 sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n",
3134 (data[0] & 0xffff), data[0] >> 16, ((data[2] >> 20) & 0x3f),
3135 ((data[2] >> 26) & 1), ((data[2] >> 27) & 1));
3136 sbuf_printf(sb, " generation=%u CQ mode=%u FL threshold=%u\n",
3137 ((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]);
3138
3139 sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start,
3140 (rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1));
3141
3142 dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count;
3143 for (i = rspq->rspq_dump_start; i < dump_end; i++) {
3144 idx = i & (RSPQ_Q_SIZE-1);
3145
3146 rspd = &rspq->desc[idx];
3147 sbuf_printf(sb, "\tidx=%04d opcode=%02x cpu_idx=%x hash_type=%x cq_idx=%x\n",
3148 idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx,
3149 rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx));
3150 sbuf_printf(sb, "\trss_hash_val=%x flags=%08x len_cq=%x intr_gen=%x\n",
3151 rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags),
3152 be32toh(rspd->len_cq), rspd->intr_gen);
3153 }
3154
3155 err = sbuf_finish(sb);
3156 sbuf_delete(sb);
3157 return (err);
3158 }
3159
3160 static int
3161 t3_dump_txq_eth(SYSCTL_HANDLER_ARGS)
3162 {
3163 struct sge_txq *txq;
3164 struct sge_qset *qs;
3165 int i, j, err, dump_end;
3166 struct sbuf *sb;
3167 struct tx_desc *txd;
3168 uint32_t *WR, wr_hi, wr_lo, gen;
3169 uint32_t data[4];
3170
3171 txq = arg1;
3172 qs = txq_to_qset(txq, TXQ_ETH);
3173 if (txq->txq_dump_count == 0) {
3174 return (0);
3175 }
3176 if (txq->txq_dump_count > TX_ETH_Q_SIZE) {
3177 log(LOG_WARNING,
3178 "dump count is too large %d\n", txq->txq_dump_count);
3179 txq->txq_dump_count = 1;
3180 return (EINVAL);
3181 }
3182 if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) {
3183 log(LOG_WARNING,
3184 "dump start of %d is greater than queue size\n",
3185 txq->txq_dump_start);
3186 txq->txq_dump_start = 0;
3187 return (EINVAL);
3188 }
3189 err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data);
3190 if (err)
3191 return (err);
3192 err = sysctl_wire_old_buffer(req, 0);
3193 if (err)
3194 return (err);
3195 sb = sbuf_new_for_sysctl(NULL, NULL, QDUMP_SBUF_SIZE, req);
3196
3197 sbuf_printf(sb, " \n credits=%u GTS=%u index=%u size=%u rspq#=%u cmdq#=%u\n",
3198 (data[0] & 0x7fff), ((data[0] >> 15) & 1), (data[0] >> 16),
3199 (data[1] & 0xffff), ((data[3] >> 4) & 7), ((data[3] >> 7) & 1));
3200 sbuf_printf(sb, " TUN=%u TOE=%u generation%u uP token=%u valid=%u\n",
3201 ((data[3] >> 8) & 1), ((data[3] >> 9) & 1), ((data[3] >> 10) & 1),
3202 ((data[3] >> 11) & 0xfffff), ((data[3] >> 31) & 1));
3203 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx,
3204 txq->txq_dump_start,
3205 (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1));
3206
3207 dump_end = txq->txq_dump_start + txq->txq_dump_count;
3208 for (i = txq->txq_dump_start; i < dump_end; i++) {
3209 txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)];
3210 WR = (uint32_t *)txd->flit;
3211 wr_hi = ntohl(WR[0]);
3212 wr_lo = ntohl(WR[1]);
3213 gen = G_WR_GEN(wr_lo);
3214
3215 sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n",
3216 wr_hi, wr_lo, gen);
3217 for (j = 2; j < 30; j += 4)
3218 sbuf_printf(sb, "\t%08x %08x %08x %08x \n",
3219 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]);
3220
3221 }
3222 err = sbuf_finish(sb);
3223 sbuf_delete(sb);
3224 return (err);
3225 }
3226
3227 static int
3228 t3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS)
3229 {
3230 struct sge_txq *txq;
3231 struct sge_qset *qs;
3232 int i, j, err, dump_end;
3233 struct sbuf *sb;
3234 struct tx_desc *txd;
3235 uint32_t *WR, wr_hi, wr_lo, gen;
3236
3237 txq = arg1;
3238 qs = txq_to_qset(txq, TXQ_CTRL);
3239 if (txq->txq_dump_count == 0) {
3240 return (0);
3241 }
3242 if (txq->txq_dump_count > 256) {
3243 log(LOG_WARNING,
3244 "dump count is too large %d\n", txq->txq_dump_count);
3245 txq->txq_dump_count = 1;
3246 return (EINVAL);
3247 }
3248 if (txq->txq_dump_start > 255) {
3249 log(LOG_WARNING,
3250 "dump start of %d is greater than queue size\n",
3251 txq->txq_dump_start);
3252 txq->txq_dump_start = 0;
3253 return (EINVAL);
3254 }
3255
3256 err = sysctl_wire_old_buffer(req, 0);
3257 if (err != 0)
3258 return (err);
3259 sb = sbuf_new_for_sysctl(NULL, NULL, QDUMP_SBUF_SIZE, req);
3260 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx,
3261 txq->txq_dump_start,
3262 (txq->txq_dump_start + txq->txq_dump_count) & 255);
3263
3264 dump_end = txq->txq_dump_start + txq->txq_dump_count;
3265 for (i = txq->txq_dump_start; i < dump_end; i++) {
3266 txd = &txq->desc[i & (255)];
3267 WR = (uint32_t *)txd->flit;
3268 wr_hi = ntohl(WR[0]);
3269 wr_lo = ntohl(WR[1]);
3270 gen = G_WR_GEN(wr_lo);
3271
3272 sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n",
3273 wr_hi, wr_lo, gen);
3274 for (j = 2; j < 30; j += 4)
3275 sbuf_printf(sb, "\t%08x %08x %08x %08x \n",
3276 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]);
3277
3278 }
3279 err = sbuf_finish(sb);
3280 sbuf_delete(sb);
3281 return (err);
3282 }
3283
3284 static int
3285 t3_set_coalesce_usecs(SYSCTL_HANDLER_ARGS)
3286 {
3287 adapter_t *sc = arg1;
3288 struct qset_params *qsp = &sc->params.sge.qset[0];
3289 int coalesce_usecs;
3290 struct sge_qset *qs;
3291 int i, j, err, nqsets = 0;
3292 struct mtx *lock;
3293
3294 if ((sc->flags & FULL_INIT_DONE) == 0)
3295 return (ENXIO);
3296
3297 coalesce_usecs = qsp->coalesce_usecs;
3298 err = sysctl_handle_int(oidp, &coalesce_usecs, arg2, req);
3299
3300 if (err != 0) {
3301 return (err);
3302 }
3303 if (coalesce_usecs == qsp->coalesce_usecs)
3304 return (0);
3305
3306 for (i = 0; i < sc->params.nports; i++)
3307 for (j = 0; j < sc->port[i].nqsets; j++)
3308 nqsets++;
3309
3310 coalesce_usecs = max(1, coalesce_usecs);
3311
3312 for (i = 0; i < nqsets; i++) {
3313 qs = &sc->sge.qs[i];
3314 qsp = &sc->params.sge.qset[i];
3315 qsp->coalesce_usecs = coalesce_usecs;
3316
3317 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock :
3318 &sc->sge.qs[0].rspq.lock;
3319
3320 mtx_lock(lock);
3321 t3_update_qset_coalesce(qs, qsp);
3322 t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
3323 V_NEWTIMER(qs->rspq.holdoff_tmr));
3324 mtx_unlock(lock);
3325 }
3326
3327 return (0);
3328 }
3329
3330 static int
3331 t3_pkt_timestamp(SYSCTL_HANDLER_ARGS)
3332 {
3333 adapter_t *sc = arg1;
3334 int rc, timestamp;
3335
3336 if ((sc->flags & FULL_INIT_DONE) == 0)
3337 return (ENXIO);
3338
3339 timestamp = sc->timestamp;
3340 rc = sysctl_handle_int(oidp, ×tamp, arg2, req);
3341
3342 if (rc != 0)
3343 return (rc);
3344
3345 if (timestamp != sc->timestamp) {
3346 t3_set_reg_field(sc, A_TP_PC_CONFIG2, F_ENABLERXPKTTMSTPRSS,
3347 timestamp ? F_ENABLERXPKTTMSTPRSS : 0);
3348 sc->timestamp = timestamp;
3349 }
3350
3351 return (0);
3352 }
3353
3354 void
3355 t3_add_attach_sysctls(adapter_t *sc)
3356 {
3357 struct sysctl_ctx_list *ctx;
3358 struct sysctl_oid_list *children;
3359
3360 ctx = device_get_sysctl_ctx(sc->dev);
3361 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
3362
3363 /* random information */
3364 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
3365 "firmware_version",
3366 CTLFLAG_RD, sc->fw_version,
3367 0, "firmware version");
3368 SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
3369 "hw_revision",
3370 CTLFLAG_RD, &sc->params.rev,
3371 0, "chip model");
3372 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
3373 "port_types",
3374 CTLFLAG_RD, sc->port_types,
3375 0, "type of ports");
3376 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
3377 "enable_debug",
3378 CTLFLAG_RW, &cxgb_debug,
3379 0, "enable verbose debugging output");
3380 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tunq_coalesce",
3381 CTLFLAG_RD, &sc->tunq_coalesce,
3382 "#tunneled packets freed");
3383 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
3384 "txq_overrun",
3385 CTLFLAG_RD, &txq_fills,
3386 0, "#times txq overrun");
3387 SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
3388 "core_clock",
3389 CTLFLAG_RD, &sc->params.vpd.cclk,
3390 0, "core clock frequency (in KHz)");
3391 }
3392
3393
3394 static const char *rspq_name = "rspq";
3395 static const char *txq_names[] =
3396 {
3397 "txq_eth",
3398 "txq_ofld",
3399 "txq_ctrl"
3400 };
3401
3402 static int
3403 sysctl_handle_macstat(SYSCTL_HANDLER_ARGS)
3404 {
3405 struct port_info *p = arg1;
3406 uint64_t *parg;
3407
3408 if (!p)
3409 return (EINVAL);
3410
3411 cxgb_refresh_stats(p);
3412 parg = (uint64_t *) ((uint8_t *)&p->mac.stats + arg2);
3413
3414 return (sysctl_handle_64(oidp, parg, 0, req));
3415 }
3416
3417 void
3418 t3_add_configured_sysctls(adapter_t *sc)
3419 {
3420 struct sysctl_ctx_list *ctx;
3421 struct sysctl_oid_list *children;
3422 int i, j;
3423
3424 ctx = device_get_sysctl_ctx(sc->dev);
3425 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
3426
3427 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
3428 "intr_coal",
3429 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc,
3430 0, t3_set_coalesce_usecs,
3431 "I", "interrupt coalescing timer (us)");
3432
3433 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
3434 "pkt_timestamp",
3435 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc,
3436 0, t3_pkt_timestamp,
3437 "I", "provide packet timestamp instead of connection hash");
3438
3439 for (i = 0; i < sc->params.nports; i++) {
3440 struct port_info *pi = &sc->port[i];
3441 struct sysctl_oid *poid;
3442 struct sysctl_oid_list *poidlist;
3443 struct mac_stats *mstats = &pi->mac.stats;
3444
3445 snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i);
3446 poid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO,
3447 pi->namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3448 "port statistics");
3449 poidlist = SYSCTL_CHILDREN(poid);
3450 SYSCTL_ADD_UINT(ctx, poidlist, OID_AUTO,
3451 "nqsets", CTLFLAG_RD, &pi->nqsets,
3452 0, "#queue sets");
3453
3454 for (j = 0; j < pi->nqsets; j++) {
3455 struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j];
3456 struct sysctl_oid *qspoid, *rspqpoid, *txqpoid,
3457 *ctrlqpoid, *lropoid;
3458 struct sysctl_oid_list *qspoidlist, *rspqpoidlist,
3459 *txqpoidlist, *ctrlqpoidlist,
3460 *lropoidlist;
3461 struct sge_txq *txq = &qs->txq[TXQ_ETH];
3462
3463 snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j);
3464
3465 qspoid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO,
3466 qs->namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3467 "qset statistics");
3468 qspoidlist = SYSCTL_CHILDREN(qspoid);
3469
3470 SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl0_empty",
3471 CTLFLAG_RD, &qs->fl[0].empty, 0,
3472 "freelist #0 empty");
3473 SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl1_empty",
3474 CTLFLAG_RD, &qs->fl[1].empty, 0,
3475 "freelist #1 empty");
3476
3477 rspqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
3478 rspq_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3479 "rspq statistics");
3480 rspqpoidlist = SYSCTL_CHILDREN(rspqpoid);
3481
3482 txqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
3483 txq_names[0], CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3484 "txq statistics");
3485 txqpoidlist = SYSCTL_CHILDREN(txqpoid);
3486
3487 ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
3488 txq_names[2], CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3489 "ctrlq statistics");
3490 ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid);
3491
3492 lropoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
3493 "lro_stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3494 "LRO statistics");
3495 lropoidlist = SYSCTL_CHILDREN(lropoid);
3496
3497 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size",
3498 CTLFLAG_RD, &qs->rspq.size,
3499 0, "#entries in response queue");
3500 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "cidx",
3501 CTLFLAG_RD, &qs->rspq.cidx,
3502 0, "consumer index");
3503 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "credits",
3504 CTLFLAG_RD, &qs->rspq.credits,
3505 0, "#credits");
3506 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "starved",
3507 CTLFLAG_RD, &qs->rspq.starved,
3508 0, "#times starved");
3509 SYSCTL_ADD_UAUTO(ctx, rspqpoidlist, OID_AUTO, "phys_addr",
3510 CTLFLAG_RD, &qs->rspq.phys_addr,
3511 "physical_address_of the queue");
3512 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_start",
3513 CTLFLAG_RW, &qs->rspq.rspq_dump_start,
3514 0, "start rspq dump entry");
3515 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_count",
3516 CTLFLAG_RW, &qs->rspq.rspq_dump_count,
3517 0, "#rspq entries to dump");
3518 SYSCTL_ADD_PROC(ctx, rspqpoidlist, OID_AUTO, "qdump",
3519 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
3520 &qs->rspq, 0, t3_dump_rspq, "A",
3521 "dump of the response queue");
3522
3523 SYSCTL_ADD_UQUAD(ctx, txqpoidlist, OID_AUTO, "dropped",
3524 CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_mr->br_drops,
3525 "#tunneled packets dropped");
3526 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "sendqlen",
3527 CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.mq_len,
3528 0, "#tunneled packets waiting to be sent");
3529 #if 0
3530 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_pidx",
3531 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod,
3532 0, "#tunneled packets queue producer index");
3533 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_cidx",
3534 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons,
3535 0, "#tunneled packets queue consumer index");
3536 #endif
3537 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "processed",
3538 CTLFLAG_RD, &qs->txq[TXQ_ETH].processed,
3539 0, "#tunneled packets processed by the card");
3540 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "cleaned",
3541 CTLFLAG_RD, &txq->cleaned,
3542 0, "#tunneled packets cleaned");
3543 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "in_use",
3544 CTLFLAG_RD, &txq->in_use,
3545 0, "#tunneled packet slots in use");
3546 SYSCTL_ADD_UQUAD(ctx, txqpoidlist, OID_AUTO, "frees",
3547 CTLFLAG_RD, &txq->txq_frees,
3548 "#tunneled packets freed");
3549 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "skipped",
3550 CTLFLAG_RD, &txq->txq_skipped,
3551 0, "#tunneled packet descriptors skipped");
3552 SYSCTL_ADD_UQUAD(ctx, txqpoidlist, OID_AUTO, "coalesced",
3553 CTLFLAG_RD, &txq->txq_coalesced,
3554 "#tunneled packets coalesced");
3555 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "enqueued",
3556 CTLFLAG_RD, &txq->txq_enqueued,
3557 0, "#tunneled packets enqueued to hardware");
3558 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "stopped_flags",
3559 CTLFLAG_RD, &qs->txq_stopped,
3560 0, "tx queues stopped");
3561 SYSCTL_ADD_UAUTO(ctx, txqpoidlist, OID_AUTO, "phys_addr",
3562 CTLFLAG_RD, &txq->phys_addr,
3563 "physical_address_of the queue");
3564 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "qgen",
3565 CTLFLAG_RW, &qs->txq[TXQ_ETH].gen,
3566 0, "txq generation");
3567 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_cidx",
3568 CTLFLAG_RD, &txq->cidx,
3569 0, "hardware queue cidx");
3570 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_pidx",
3571 CTLFLAG_RD, &txq->pidx,
3572 0, "hardware queue pidx");
3573 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_start",
3574 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start,
3575 0, "txq start idx for dump");
3576 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_count",
3577 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count,
3578 0, "txq #entries to dump");
3579 SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump",
3580 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
3581 &qs->txq[TXQ_ETH], 0, t3_dump_txq_eth, "A",
3582 "dump of the transmit queue");
3583
3584 SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start",
3585 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start,
3586 0, "ctrlq start idx for dump");
3587 SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count",
3588 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count,
3589 0, "ctrl #entries to dump");
3590 SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump",
3591 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
3592 &qs->txq[TXQ_CTRL], 0, t3_dump_txq_ctrl, "A",
3593 "dump of the transmit queue");
3594
3595 SYSCTL_ADD_U64(ctx, lropoidlist, OID_AUTO, "lro_queued",
3596 CTLFLAG_RD, &qs->lro.ctrl.lro_queued, 0, NULL);
3597 SYSCTL_ADD_U64(ctx, lropoidlist, OID_AUTO, "lro_flushed",
3598 CTLFLAG_RD, &qs->lro.ctrl.lro_flushed, 0, NULL);
3599 SYSCTL_ADD_U64(ctx, lropoidlist, OID_AUTO, "lro_bad_csum",
3600 CTLFLAG_RD, &qs->lro.ctrl.lro_bad_csum, 0, NULL);
3601 SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_cnt",
3602 CTLFLAG_RD, &qs->lro.ctrl.lro_cnt, 0, NULL);
3603 }
3604
3605 /* Now add a node for mac stats. */
3606 poid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, "mac_stats",
3607 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "MAC statistics");
3608 poidlist = SYSCTL_CHILDREN(poid);
3609
3610 /*
3611 * We (ab)use the length argument (arg2) to pass on the offset
3612 * of the data that we are interested in. This is only required
3613 * for the quad counters that are updated from the hardware (we
3614 * make sure that we return the latest value).
3615 * sysctl_handle_macstat first updates *all* the counters from
3616 * the hardware, and then returns the latest value of the
3617 * requested counter. Best would be to update only the
3618 * requested counter from hardware, but t3_mac_update_stats()
3619 * hides all the register details and we don't want to dive into
3620 * all that here.
3621 */
3622 #define CXGB_SYSCTL_ADD_QUAD(a) SYSCTL_ADD_OID(ctx, poidlist, OID_AUTO, #a, \
3623 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_NEEDGIANT, pi, \
3624 offsetof(struct mac_stats, a), sysctl_handle_macstat, "QU", 0)
3625 CXGB_SYSCTL_ADD_QUAD(tx_octets);
3626 CXGB_SYSCTL_ADD_QUAD(tx_octets_bad);
3627 CXGB_SYSCTL_ADD_QUAD(tx_frames);
3628 CXGB_SYSCTL_ADD_QUAD(tx_mcast_frames);
3629 CXGB_SYSCTL_ADD_QUAD(tx_bcast_frames);
3630 CXGB_SYSCTL_ADD_QUAD(tx_pause);
3631 CXGB_SYSCTL_ADD_QUAD(tx_deferred);
3632 CXGB_SYSCTL_ADD_QUAD(tx_late_collisions);
3633 CXGB_SYSCTL_ADD_QUAD(tx_total_collisions);
3634 CXGB_SYSCTL_ADD_QUAD(tx_excess_collisions);
3635 CXGB_SYSCTL_ADD_QUAD(tx_underrun);
3636 CXGB_SYSCTL_ADD_QUAD(tx_len_errs);
3637 CXGB_SYSCTL_ADD_QUAD(tx_mac_internal_errs);
3638 CXGB_SYSCTL_ADD_QUAD(tx_excess_deferral);
3639 CXGB_SYSCTL_ADD_QUAD(tx_fcs_errs);
3640 CXGB_SYSCTL_ADD_QUAD(tx_frames_64);
3641 CXGB_SYSCTL_ADD_QUAD(tx_frames_65_127);
3642 CXGB_SYSCTL_ADD_QUAD(tx_frames_128_255);
3643 CXGB_SYSCTL_ADD_QUAD(tx_frames_256_511);
3644 CXGB_SYSCTL_ADD_QUAD(tx_frames_512_1023);
3645 CXGB_SYSCTL_ADD_QUAD(tx_frames_1024_1518);
3646 CXGB_SYSCTL_ADD_QUAD(tx_frames_1519_max);
3647 CXGB_SYSCTL_ADD_QUAD(rx_octets);
3648 CXGB_SYSCTL_ADD_QUAD(rx_octets_bad);
3649 CXGB_SYSCTL_ADD_QUAD(rx_frames);
3650 CXGB_SYSCTL_ADD_QUAD(rx_mcast_frames);
3651 CXGB_SYSCTL_ADD_QUAD(rx_bcast_frames);
3652 CXGB_SYSCTL_ADD_QUAD(rx_pause);
3653 CXGB_SYSCTL_ADD_QUAD(rx_fcs_errs);
3654 CXGB_SYSCTL_ADD_QUAD(rx_align_errs);
3655 CXGB_SYSCTL_ADD_QUAD(rx_symbol_errs);
3656 CXGB_SYSCTL_ADD_QUAD(rx_data_errs);
3657 CXGB_SYSCTL_ADD_QUAD(rx_sequence_errs);
3658 CXGB_SYSCTL_ADD_QUAD(rx_runt);
3659 CXGB_SYSCTL_ADD_QUAD(rx_jabber);
3660 CXGB_SYSCTL_ADD_QUAD(rx_short);
3661 CXGB_SYSCTL_ADD_QUAD(rx_too_long);
3662 CXGB_SYSCTL_ADD_QUAD(rx_mac_internal_errs);
3663 CXGB_SYSCTL_ADD_QUAD(rx_cong_drops);
3664 CXGB_SYSCTL_ADD_QUAD(rx_frames_64);
3665 CXGB_SYSCTL_ADD_QUAD(rx_frames_65_127);
3666 CXGB_SYSCTL_ADD_QUAD(rx_frames_128_255);
3667 CXGB_SYSCTL_ADD_QUAD(rx_frames_256_511);
3668 CXGB_SYSCTL_ADD_QUAD(rx_frames_512_1023);
3669 CXGB_SYSCTL_ADD_QUAD(rx_frames_1024_1518);
3670 CXGB_SYSCTL_ADD_QUAD(rx_frames_1519_max);
3671 #undef CXGB_SYSCTL_ADD_QUAD
3672
3673 #define CXGB_SYSCTL_ADD_ULONG(a) SYSCTL_ADD_ULONG(ctx, poidlist, OID_AUTO, #a, \
3674 CTLFLAG_RD, &mstats->a, 0)
3675 CXGB_SYSCTL_ADD_ULONG(tx_fifo_parity_err);
3676 CXGB_SYSCTL_ADD_ULONG(rx_fifo_parity_err);
3677 CXGB_SYSCTL_ADD_ULONG(tx_fifo_urun);
3678 CXGB_SYSCTL_ADD_ULONG(rx_fifo_ovfl);
3679 CXGB_SYSCTL_ADD_ULONG(serdes_signal_loss);
3680 CXGB_SYSCTL_ADD_ULONG(xaui_pcs_ctc_err);
3681 CXGB_SYSCTL_ADD_ULONG(xaui_pcs_align_change);
3682 CXGB_SYSCTL_ADD_ULONG(num_toggled);
3683 CXGB_SYSCTL_ADD_ULONG(num_resets);
3684 CXGB_SYSCTL_ADD_ULONG(link_faults);
3685 #undef CXGB_SYSCTL_ADD_ULONG
3686 }
3687 }
3688
3689 /**
3690 * t3_get_desc - dump an SGE descriptor for debugging purposes
3691 * @qs: the queue set
3692 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
3693 * @idx: the descriptor index in the queue
3694 * @data: where to dump the descriptor contents
3695 *
3696 * Dumps the contents of a HW descriptor of an SGE queue. Returns the
3697 * size of the descriptor.
3698 */
3699 int
3700 t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
3701 unsigned char *data)
3702 {
3703 if (qnum >= 6)
3704 return (EINVAL);
3705
3706 if (qnum < 3) {
3707 if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size)
3708 return -EINVAL;
3709 memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc));
3710 return sizeof(struct tx_desc);
3711 }
3712
3713 if (qnum == 3) {
3714 if (!qs->rspq.desc || idx >= qs->rspq.size)
3715 return (EINVAL);
3716 memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc));
3717 return sizeof(struct rsp_desc);
3718 }
3719
3720 qnum -= 4;
3721 if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size)
3722 return (EINVAL);
3723 memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc));
3724 return sizeof(struct rx_desc);
3725 }
Cache object: c62edcb4a7003a1a5c851e29f30286e6
|