1 /**************************************************************************
2
3 Copyright (c) 2007-2008, Chelsio Inc.
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
27
28
29 $FreeBSD$
30
31
32 ***************************************************************************/
33
34
35 #ifndef _CXGB_ADAPTER_H_
36 #define _CXGB_ADAPTER_H_
37
38 #include <sys/lock.h>
39 #include <sys/mutex.h>
40 #include <sys/sx.h>
41 #include <sys/rman.h>
42 #include <sys/mbuf.h>
43 #include <sys/socket.h>
44 #include <sys/sockio.h>
45 #include <sys/condvar.h>
46
47 #include <net/ethernet.h>
48 #include <net/if.h>
49 #include <net/if_media.h>
50 #include <net/if_dl.h>
51 #include <netinet/tcp_lro.h>
52
53 #include <machine/bus.h>
54 #include <machine/resource.h>
55
56 #include <sys/bus_dma.h>
57 #include <dev/pci/pcireg.h>
58 #include <dev/pci/pcivar.h>
59
60
61 #ifdef CONFIG_DEFINED
62 #include <cxgb_osdep.h>
63 #include <t3cdev.h>
64 #include <ulp/toecore/cxgb_toedev.h>
65 #include <sys/mbufq.h>
66 #else
67 #include <dev/cxgb/cxgb_osdep.h>
68 #include <dev/cxgb/t3cdev.h>
69 #include <dev/cxgb/sys/mbufq.h>
70 #include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
71 #endif
72
73 #define USE_SX
74
75 struct adapter;
76 struct sge_qset;
77 extern int cxgb_debug;
78
79 #ifdef DEBUG_LOCKING
80 #define MTX_INIT(lock, lockname, class, flags) \
81 do { \
82 printf("initializing %s at %s:%d\n", lockname, __FILE__, __LINE__); \
83 mtx_init((lock), lockname, class, flags); \
84 } while (0)
85
86 #define MTX_DESTROY(lock) \
87 do { \
88 printf("destroying %s at %s:%d\n", (lock)->lock_object.lo_name, __FILE__, __LINE__); \
89 mtx_destroy((lock)); \
90 } while (0)
91
92 #define SX_INIT(lock, lockname) \
93 do { \
94 printf("initializing %s at %s:%d\n", lockname, __FILE__, __LINE__); \
95 sx_init((lock), lockname); \
96 } while (0)
97
98 #define SX_DESTROY(lock) \
99 do { \
100 printf("destroying %s at %s:%d\n", (lock)->lock_object.lo_name, __FILE__, __LINE__); \
101 sx_destroy((lock)); \
102 } while (0)
103 #else
104 #define MTX_INIT mtx_init
105 #define MTX_DESTROY mtx_destroy
106 #define SX_INIT sx_init
107 #define SX_DESTROY sx_destroy
108 #endif
109
110 struct port_info {
111 struct adapter *adapter;
112 struct ifnet *ifp;
113 int if_flags;
114 const struct port_type_info *port_type;
115 struct cphy phy;
116 struct cmac mac;
117 struct link_config link_config;
118 struct ifmedia media;
119 #ifdef USE_SX
120 struct sx lock;
121 #else
122 struct mtx lock;
123 #endif
124 uint8_t port_id;
125 uint8_t tx_chan;
126 uint8_t txpkt_intf;
127 uint8_t first_qset;
128 uint32_t nqsets;
129
130 uint8_t hw_addr[ETHER_ADDR_LEN];
131 struct task timer_reclaim_task;
132 struct cdev *port_cdev;
133
134 #define PORT_LOCK_NAME_LEN 32
135 #define PORT_NAME_LEN 32
136 char lockbuf[PORT_LOCK_NAME_LEN];
137 char namebuf[PORT_NAME_LEN];
138 };
139
140 enum { /* adapter flags */
141 FULL_INIT_DONE = (1 << 0),
142 USING_MSI = (1 << 1),
143 USING_MSIX = (1 << 2),
144 QUEUES_BOUND = (1 << 3),
145 FW_UPTODATE = (1 << 4),
146 TPS_UPTODATE = (1 << 5),
147 CXGB_SHUTDOWN = (1 << 6),
148 CXGB_OFLD_INIT = (1 << 7),
149 TP_PARITY_INIT = (1 << 8),
150 };
151
152 #define FL_Q_SIZE 4096
153 #define JUMBO_Q_SIZE 1024
154 #define RSPQ_Q_SIZE 1024
155 #define TX_ETH_Q_SIZE 1024
156
157 enum { TXQ_ETH = 0,
158 TXQ_OFLD = 1,
159 TXQ_CTRL = 2, };
160
161
162 /*
163 * work request size in bytes
164 */
165 #define WR_LEN (WR_FLITS * 8)
166 #define PIO_LEN (WR_LEN - sizeof(struct cpl_tx_pkt_lso))
167
168 struct lro_state {
169 unsigned short enabled;
170 struct lro_ctrl ctrl;
171 };
172
173 #define RX_BUNDLE_SIZE 8
174
175 struct rsp_desc;
176
177 struct sge_rspq {
178 uint32_t credits;
179 uint32_t size;
180 uint32_t cidx;
181 uint32_t gen;
182 uint32_t polling;
183 uint32_t holdoff_tmr;
184 uint32_t next_holdoff;
185 uint32_t imm_data;
186 uint32_t async_notif;
187 uint32_t cntxt_id;
188 uint32_t offload_pkts;
189 uint32_t offload_bundles;
190 uint32_t pure_rsps;
191 uint32_t unhandled_irqs;
192
193 bus_addr_t phys_addr;
194 bus_dma_tag_t desc_tag;
195 bus_dmamap_t desc_map;
196
197 struct t3_mbuf_hdr rspq_mh;
198 struct rsp_desc *desc;
199 struct mtx lock;
200 #define RSPQ_NAME_LEN 32
201 char lockbuf[RSPQ_NAME_LEN];
202 uint32_t rspq_dump_start;
203 uint32_t rspq_dump_count;
204 };
205
206 #ifndef DISABLE_MBUF_IOVEC
207 #define rspq_mbuf rspq_mh.mh_head
208 #endif
209
210 struct rx_desc;
211 struct rx_sw_desc;
212
213 struct sge_fl {
214 uint32_t buf_size;
215 uint32_t credits;
216 uint32_t size;
217 uint32_t cidx;
218 uint32_t pidx;
219 uint32_t gen;
220 bus_addr_t phys_addr;
221 uint32_t cntxt_id;
222 uint64_t empty;
223 bus_dma_tag_t desc_tag;
224 bus_dmamap_t desc_map;
225 bus_dma_tag_t entry_tag;
226 uma_zone_t zone;
227 struct rx_desc *desc;
228 struct rx_sw_desc *sdesc;
229 int type;
230 };
231
232 struct tx_desc;
233 struct tx_sw_desc;
234
235 #define TXQ_TRANSMITTING 0x1
236
237 struct sge_txq {
238 uint64_t flags;
239 uint32_t in_use;
240 uint32_t size;
241 uint32_t processed;
242 uint32_t cleaned;
243 uint32_t stop_thres;
244 uint32_t cidx;
245 uint32_t pidx;
246 uint32_t gen;
247 uint32_t unacked;
248 struct tx_desc *desc;
249 struct tx_sw_desc *sdesc;
250 uint32_t token;
251 bus_addr_t phys_addr;
252 struct task qresume_task;
253 struct task qreclaim_task;
254 struct port_info *port;
255 uint32_t cntxt_id;
256 uint64_t stops;
257 uint64_t restarts;
258 bus_dma_tag_t desc_tag;
259 bus_dmamap_t desc_map;
260 bus_dma_tag_t entry_tag;
261 struct mbuf_head sendq;
262 /*
263 * cleanq should really be an buf_ring to avoid extra
264 * mbuf touches
265 */
266 struct mbuf_head cleanq;
267 struct buf_ring txq_mr;
268 struct mbuf *immpkt;
269 uint32_t txq_drops;
270 uint32_t txq_skipped;
271 uint32_t txq_coalesced;
272 uint32_t txq_enqueued;
273 uint32_t txq_dump_start;
274 uint32_t txq_dump_count;
275 unsigned long txq_frees;
276 struct mtx lock;
277 struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1];
278 #define TXQ_NAME_LEN 32
279 char lockbuf[TXQ_NAME_LEN];
280 };
281
282
283 enum {
284 SGE_PSTAT_TSO, /* # of TSO requests */
285 SGE_PSTAT_RX_CSUM_GOOD, /* # of successful RX csum offloads */
286 SGE_PSTAT_TX_CSUM, /* # of TX checksum offloads */
287 SGE_PSTAT_VLANEX, /* # of VLAN tag extractions */
288 SGE_PSTAT_VLANINS, /* # of VLAN tag insertions */
289 };
290
291 #define SGE_PSTAT_MAX (SGE_PSTAT_VLANINS+1)
292
293 #define QS_EXITING 0x1
294 #define QS_RUNNING 0x2
295 #define QS_BOUND 0x4
296
297 struct sge_qset {
298 struct sge_rspq rspq;
299 struct sge_fl fl[SGE_RXQ_PER_SET];
300 struct lro_state lro;
301 struct sge_txq txq[SGE_TXQ_PER_SET];
302 uint32_t txq_stopped; /* which Tx queues are stopped */
303 uint64_t port_stats[SGE_PSTAT_MAX];
304 struct port_info *port;
305 int idx; /* qset # */
306 int qs_cpuid;
307 int qs_flags;
308 struct cv qs_cv;
309 struct mtx qs_mtx;
310 #define QS_NAME_LEN 32
311 char namebuf[QS_NAME_LEN];
312 };
313
314 struct sge {
315 struct sge_qset qs[SGE_QSETS];
316 struct mtx reg_lock;
317 };
318
319 struct filter_info;
320
321 struct adapter {
322 device_t dev;
323 int flags;
324 TAILQ_ENTRY(adapter) adapter_entry;
325
326 /* PCI register resources */
327 int regs_rid;
328 struct resource *regs_res;
329 int udbs_rid;
330 struct resource *udbs_res;
331 bus_space_handle_t bh;
332 bus_space_tag_t bt;
333 bus_size_t mmio_len;
334 uint32_t link_width;
335
336 /* DMA resources */
337 bus_dma_tag_t parent_dmat;
338 bus_dma_tag_t rx_dmat;
339 bus_dma_tag_t rx_jumbo_dmat;
340 bus_dma_tag_t tx_dmat;
341
342 /* Interrupt resources */
343 struct resource *irq_res;
344 int irq_rid;
345 void *intr_tag;
346
347 uint32_t msix_regs_rid;
348 struct resource *msix_regs_res;
349
350 struct resource *msix_irq_res[SGE_QSETS];
351 int msix_irq_rid[SGE_QSETS];
352 void *msix_intr_tag[SGE_QSETS];
353 uint8_t rxpkt_map[8]; /* maps RX_PKT interface values to port ids */
354 uint8_t rrss_map[SGE_QSETS]; /* revers RSS map table */
355 uint16_t rspq_map[RSS_TABLE_SIZE]; /* maps 7-bit cookie to qidx */
356 union {
357 uint8_t fill[SGE_QSETS];
358 uint64_t coalesce;
359 } u;
360
361 #define tunq_fill u.fill
362 #define tunq_coalesce u.coalesce
363
364 struct filter_info *filters;
365
366 /* Tasks */
367 struct task ext_intr_task;
368 struct task slow_intr_task;
369 struct task tick_task;
370 struct taskqueue *tq;
371 struct callout cxgb_tick_ch;
372 struct callout sge_timer_ch;
373
374 unsigned int check_task_cnt;
375
376 /* Register lock for use by the hardware layer */
377 struct mtx mdio_lock;
378 struct mtx elmer_lock;
379
380 /* Bookkeeping for the hardware layer */
381 struct adapter_params params;
382 unsigned int slow_intr_mask;
383 unsigned long irq_stats[IRQ_NUM_STATS];
384
385 struct sge sge;
386 struct mc7 pmrx;
387 struct mc7 pmtx;
388 struct mc7 cm;
389 struct mc5 mc5;
390
391 struct port_info port[MAX_NPORTS];
392 device_t portdev[MAX_NPORTS];
393 struct t3cdev tdev;
394 char fw_version[64];
395 uint32_t open_device_map;
396 uint32_t registered_device_map;
397 #ifdef USE_SX
398 struct sx lock;
399 #else
400 struct mtx lock;
401 #endif
402 driver_intr_t *cxgb_intr;
403 int msi_count;
404
405 #define ADAPTER_LOCK_NAME_LEN 32
406 char lockbuf[ADAPTER_LOCK_NAME_LEN];
407 char reglockbuf[ADAPTER_LOCK_NAME_LEN];
408 char mdiolockbuf[ADAPTER_LOCK_NAME_LEN];
409 char elmerlockbuf[ADAPTER_LOCK_NAME_LEN];
410 };
411
412 struct t3_rx_mode {
413
414 uint32_t idx;
415 struct port_info *port;
416 };
417
418
419 #define MDIO_LOCK(adapter) mtx_lock(&(adapter)->mdio_lock)
420 #define MDIO_UNLOCK(adapter) mtx_unlock(&(adapter)->mdio_lock)
421 #define ELMR_LOCK(adapter) mtx_lock(&(adapter)->elmer_lock)
422 #define ELMR_UNLOCK(adapter) mtx_unlock(&(adapter)->elmer_lock)
423
424
425 #ifdef USE_SX
426 #define PORT_LOCK(port) sx_xlock(&(port)->lock);
427 #define PORT_UNLOCK(port) sx_xunlock(&(port)->lock);
428 #define PORT_LOCK_INIT(port, name) SX_INIT(&(port)->lock, name)
429 #define PORT_LOCK_DEINIT(port) SX_DESTROY(&(port)->lock)
430 #define PORT_LOCK_ASSERT_OWNED(port) sx_assert(&(port)->lock, SA_LOCKED)
431
432 #define ADAPTER_LOCK(adap) sx_xlock(&(adap)->lock);
433 #define ADAPTER_UNLOCK(adap) sx_xunlock(&(adap)->lock);
434 #define ADAPTER_LOCK_INIT(adap, name) SX_INIT(&(adap)->lock, name)
435 #define ADAPTER_LOCK_DEINIT(adap) SX_DESTROY(&(adap)->lock)
436 #define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) sx_assert(&(adap)->lock, SA_UNLOCKED)
437 #else
438 #define PORT_LOCK(port) mtx_lock(&(port)->lock);
439 #define PORT_UNLOCK(port) mtx_unlock(&(port)->lock);
440 #define PORT_LOCK_INIT(port, name) mtx_init(&(port)->lock, name, 0, MTX_DEF)
441 #define PORT_LOCK_DEINIT(port) mtx_destroy(&(port)->lock)
442 #define PORT_LOCK_ASSERT_OWNED(port) mtx_assert(&(port)->lock, MA_OWNED)
443
444 #define ADAPTER_LOCK(adap) mtx_lock(&(adap)->lock);
445 #define ADAPTER_UNLOCK(adap) mtx_unlock(&(adap)->lock);
446 #define ADAPTER_LOCK_INIT(adap, name) mtx_init(&(adap)->lock, name, 0, MTX_DEF)
447 #define ADAPTER_LOCK_DEINIT(adap) mtx_destroy(&(adap)->lock)
448 #define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) mtx_assert(&(adap)->lock, MO_NOTOWNED)
449 #endif
450
451
452 static __inline uint32_t
453 t3_read_reg(adapter_t *adapter, uint32_t reg_addr)
454 {
455 return (bus_space_read_4(adapter->bt, adapter->bh, reg_addr));
456 }
457
458 static __inline void
459 t3_write_reg(adapter_t *adapter, uint32_t reg_addr, uint32_t val)
460 {
461 bus_space_write_4(adapter->bt, adapter->bh, reg_addr, val);
462 }
463
464 static __inline void
465 t3_os_pci_read_config_4(adapter_t *adapter, int reg, uint32_t *val)
466 {
467 *val = pci_read_config(adapter->dev, reg, 4);
468 }
469
470 static __inline void
471 t3_os_pci_write_config_4(adapter_t *adapter, int reg, uint32_t val)
472 {
473 pci_write_config(adapter->dev, reg, val, 4);
474 }
475
476 static __inline void
477 t3_os_pci_read_config_2(adapter_t *adapter, int reg, uint16_t *val)
478 {
479 *val = pci_read_config(adapter->dev, reg, 2);
480 }
481
482 static __inline void
483 t3_os_pci_write_config_2(adapter_t *adapter, int reg, uint16_t val)
484 {
485 pci_write_config(adapter->dev, reg, val, 2);
486 }
487
488 static __inline uint8_t *
489 t3_get_next_mcaddr(struct t3_rx_mode *rm)
490 {
491 uint8_t *macaddr = NULL;
492 struct ifnet *ifp = rm->port->ifp;
493 struct ifmultiaddr *ifma;
494 int i = 0;
495
496 IF_ADDR_LOCK(ifp);
497 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
498 if (ifma->ifma_addr->sa_family != AF_LINK)
499 continue;
500 if (i == rm->idx) {
501 macaddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
502 break;
503 }
504 i++;
505 }
506 IF_ADDR_UNLOCK(ifp);
507
508
509 rm->idx++;
510 return (macaddr);
511 }
512
513 static __inline void
514 t3_init_rx_mode(struct t3_rx_mode *rm, struct port_info *port)
515 {
516 rm->idx = 0;
517 rm->port = port;
518 }
519
520 static __inline struct port_info *
521 adap2pinfo(struct adapter *adap, int idx)
522 {
523 return &adap->port[idx];
524 }
525
526 int t3_os_find_pci_capability(adapter_t *adapter, int cap);
527 int t3_os_pci_save_state(struct adapter *adapter);
528 int t3_os_pci_restore_state(struct adapter *adapter);
529 void t3_os_link_changed(adapter_t *adapter, int port_id, int link_status,
530 int speed, int duplex, int fc);
531 void t3_os_phymod_changed(struct adapter *adap, int port_id);
532 void t3_sge_err_intr_handler(adapter_t *adapter);
533 int t3_offload_tx(struct t3cdev *, struct mbuf *);
534 void t3_os_ext_intr_handler(adapter_t *adapter);
535 void t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]);
536 int t3_mgmt_tx(adapter_t *adap, struct mbuf *m);
537
538
539 int t3_sge_alloc(struct adapter *);
540 int t3_sge_free(struct adapter *);
541 int t3_sge_alloc_qset(adapter_t *, uint32_t, int, int, const struct qset_params *,
542 int, struct port_info *);
543 void t3_free_sge_resources(adapter_t *);
544 void t3_sge_start(adapter_t *);
545 void t3_sge_stop(adapter_t *);
546 void t3b_intr(void *data);
547 void t3_intr_msi(void *data);
548 void t3_intr_msix(void *data);
549 int t3_encap(struct sge_qset *, struct mbuf **, int);
550
551 int t3_sge_init_adapter(adapter_t *);
552 int t3_sge_reset_adapter(adapter_t *);
553 int t3_sge_init_port(struct port_info *);
554 void t3_sge_deinit_sw(adapter_t *);
555 void t3_free_tx_desc(struct sge_txq *q, int n);
556 void t3_free_tx_desc_all(struct sge_txq *q);
557
558 void t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad);
559
560 void t3_add_attach_sysctls(adapter_t *sc);
561 void t3_add_configured_sysctls(adapter_t *sc);
562 int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
563 unsigned char *data);
564 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p);
565
566 #define CXGB_TICKS(a) ((a)->params.linkpoll_period ? \
567 (hz * (a)->params.linkpoll_period) / 10 : \
568 (a)->params.stats_update_period * hz)
569
570 /*
571 * XXX figure out how we can return this to being private to sge
572 */
573 #define desc_reclaimable(q) ((int)((q)->processed - (q)->cleaned - TX_MAX_DESC))
574
575 #define container_of(p, stype, field) ((stype *)(((uint8_t *)(p)) - offsetof(stype, field)))
576
577 static __inline struct sge_qset *
578 fl_to_qset(struct sge_fl *q, int qidx)
579 {
580 return container_of(q, struct sge_qset, fl[qidx]);
581 }
582
583 static __inline struct sge_qset *
584 rspq_to_qset(struct sge_rspq *q)
585 {
586 return container_of(q, struct sge_qset, rspq);
587 }
588
589 static __inline struct sge_qset *
590 txq_to_qset(struct sge_txq *q, int qidx)
591 {
592 return container_of(q, struct sge_qset, txq[qidx]);
593 }
594
595 static __inline struct adapter *
596 tdev2adap(struct t3cdev *d)
597 {
598 return container_of(d, struct adapter, tdev);
599 }
600
601 #undef container_of
602
603 #define OFFLOAD_DEVMAP_BIT 15
604 static inline int offload_running(adapter_t *adapter)
605 {
606 return isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
607 }
608
609 int cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m);
610 int cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *m);
611 void cxgb_pcpu_shutdown_threads(struct adapter *sc);
612 void cxgb_pcpu_startup_threads(struct adapter *sc);
613
614 int process_responses(adapter_t *adap, struct sge_qset *qs, int budget);
615 void t3_free_qset(adapter_t *sc, struct sge_qset *q);
616 void cxgb_start(struct ifnet *ifp);
617 void refill_fl_service(adapter_t *adap, struct sge_fl *fl);
618 #endif
Cache object: de3b73d5840d75df04aebaadd174ec24
|