The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/cxgb/cxgb_main.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /**************************************************************************
    2 
    3 Copyright (c) 2007-2009, Chelsio Inc.
    4 All rights reserved.
    5 
    6 Redistribution and use in source and binary forms, with or without
    7 modification, are permitted provided that the following conditions are met:
    8 
    9  1. Redistributions of source code must retain the above copyright notice,
   10     this list of conditions and the following disclaimer.
   11 
   12  2. Neither the name of the Chelsio Corporation nor the names of its
   13     contributors may be used to endorse or promote products derived from
   14     this software without specific prior written permission.
   15 
   16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   26 POSSIBILITY OF SUCH DAMAGE.
   27 
   28 ***************************************************************************/
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/8.0/sys/dev/cxgb/cxgb_main.c 194921 2009-06-24 21:56:05Z np $");
   32 
   33 #include <sys/param.h>
   34 #include <sys/systm.h>
   35 #include <sys/kernel.h>
   36 #include <sys/bus.h>
   37 #include <sys/module.h>
   38 #include <sys/pciio.h>
   39 #include <sys/conf.h>
   40 #include <machine/bus.h>
   41 #include <machine/resource.h>
   42 #include <sys/bus_dma.h>
   43 #include <sys/ktr.h>
   44 #include <sys/rman.h>
   45 #include <sys/ioccom.h>
   46 #include <sys/mbuf.h>
   47 #include <sys/linker.h>
   48 #include <sys/firmware.h>
   49 #include <sys/socket.h>
   50 #include <sys/sockio.h>
   51 #include <sys/smp.h>
   52 #include <sys/sysctl.h>
   53 #include <sys/syslog.h>
   54 #include <sys/queue.h>
   55 #include <sys/taskqueue.h>
   56 #include <sys/proc.h>
   57 
   58 #include <net/bpf.h>
   59 #include <net/ethernet.h>
   60 #include <net/if.h>
   61 #include <net/if_arp.h>
   62 #include <net/if_dl.h>
   63 #include <net/if_media.h>
   64 #include <net/if_types.h>
   65 #include <net/if_vlan_var.h>
   66 
   67 #include <netinet/in_systm.h>
   68 #include <netinet/in.h>
   69 #include <netinet/if_ether.h>
   70 #include <netinet/ip.h>
   71 #include <netinet/ip.h>
   72 #include <netinet/tcp.h>
   73 #include <netinet/udp.h>
   74 
   75 #include <dev/pci/pcireg.h>
   76 #include <dev/pci/pcivar.h>
   77 #include <dev/pci/pci_private.h>
   78 
   79 #include <cxgb_include.h>
   80 
   81 #ifdef PRIV_SUPPORTED
   82 #include <sys/priv.h>
   83 #endif
   84 
   85 static int cxgb_setup_interrupts(adapter_t *);
   86 static void cxgb_teardown_interrupts(adapter_t *);
   87 static int cxgb_begin_op(struct port_info *, const char *);
   88 static int cxgb_begin_detach(struct port_info *);
   89 static int cxgb_end_op(struct port_info *);
   90 static void cxgb_init(void *);
   91 static int cxgb_init_synchronized(struct port_info *);
   92 static int cxgb_uninit_synchronized(struct port_info *);
   93 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
   94 static int cxgb_media_change(struct ifnet *);
   95 static int cxgb_ifm_type(int);
   96 static void cxgb_build_medialist(struct port_info *);
   97 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
   98 static int setup_sge_qsets(adapter_t *);
   99 static void cxgb_async_intr(void *);
  100 static void cxgb_ext_intr_handler(void *, int);
  101 static void cxgb_tick_handler(void *, int);
  102 static void cxgb_tick(void *);
  103 static void setup_rss(adapter_t *sc);
  104 
  105 /* Attachment glue for the PCI controller end of the device.  Each port of
  106  * the device is attached separately, as defined later.
  107  */
  108 static int cxgb_controller_probe(device_t);
  109 static int cxgb_controller_attach(device_t);
  110 static int cxgb_controller_detach(device_t);
  111 static void cxgb_free(struct adapter *);
  112 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
  113     unsigned int end);
  114 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
  115 static int cxgb_get_regs_len(void);
  116 static int offload_open(struct port_info *pi);
  117 static void touch_bars(device_t dev);
  118 static int offload_close(struct t3cdev *tdev);
  119 int t3_detect_link_fault(adapter_t *adapter, int port_id);
  120 
  121 static device_method_t cxgb_controller_methods[] = {
  122         DEVMETHOD(device_probe,         cxgb_controller_probe),
  123         DEVMETHOD(device_attach,        cxgb_controller_attach),
  124         DEVMETHOD(device_detach,        cxgb_controller_detach),
  125 
  126         /* bus interface */
  127         DEVMETHOD(bus_print_child,      bus_generic_print_child),
  128         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
  129 
  130         { 0, 0 }
  131 };
  132 
  133 static driver_t cxgb_controller_driver = {
  134         "cxgbc",
  135         cxgb_controller_methods,
  136         sizeof(struct adapter)
  137 };
  138 
  139 static devclass_t       cxgb_controller_devclass;
  140 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
  141 
  142 /*
  143  * Attachment glue for the ports.  Attachment is done directly to the
  144  * controller device.
  145  */
  146 static int cxgb_port_probe(device_t);
  147 static int cxgb_port_attach(device_t);
  148 static int cxgb_port_detach(device_t);
  149 
  150 static device_method_t cxgb_port_methods[] = {
  151         DEVMETHOD(device_probe,         cxgb_port_probe),
  152         DEVMETHOD(device_attach,        cxgb_port_attach),
  153         DEVMETHOD(device_detach,        cxgb_port_detach),
  154         { 0, 0 }
  155 };
  156 
  157 static driver_t cxgb_port_driver = {
  158         "cxgb",
  159         cxgb_port_methods,
  160         0
  161 };
  162 
  163 static d_ioctl_t cxgb_extension_ioctl;
  164 static d_open_t cxgb_extension_open;
  165 static d_close_t cxgb_extension_close;
  166 
  167 static struct cdevsw cxgb_cdevsw = {
  168        .d_version =    D_VERSION,
  169        .d_flags =      0,
  170        .d_open =       cxgb_extension_open,
  171        .d_close =      cxgb_extension_close,
  172        .d_ioctl =      cxgb_extension_ioctl,
  173        .d_name =       "cxgb",
  174 };
  175 
  176 static devclass_t       cxgb_port_devclass;
  177 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
  178 
  179 /*
  180  * The driver uses the best interrupt scheme available on a platform in the
  181  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
  182  * of these schemes the driver may consider as follows:
  183  *
  184  * msi = 2: choose from among all three options
  185  * msi = 1 : only consider MSI and pin interrupts
  186  * msi = 0: force pin interrupts
  187  */
  188 static int msi_allowed = 2;
  189 
  190 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
  191 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
  192 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
  193     "MSI-X, MSI, INTx selector");
  194 
  195 /*
  196  * The driver enables offload as a default.
  197  * To disable it, use ofld_disable = 1.
  198  */
  199 static int ofld_disable = 0;
  200 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
  201 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
  202     "disable ULP offload");
  203 
  204 /*
  205  * The driver uses an auto-queue algorithm by default.
  206  * To disable it and force a single queue-set per port, use multiq = 0
  207  */
  208 static int multiq = 1;
  209 TUNABLE_INT("hw.cxgb.multiq", &multiq);
  210 SYSCTL_UINT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
  211     "use min(ncpus/ports, 8) queue-sets per port");
  212 
  213 /*
  214  * By default the driver will not update the firmware unless
  215  * it was compiled against a newer version
  216  * 
  217  */
  218 static int force_fw_update = 0;
  219 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
  220 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
  221     "update firmware even if up to date");
  222 
  223 int cxgb_use_16k_clusters = 1;
  224 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
  225 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
  226     &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
  227 
  228 /*
  229  * Tune the size of the output queue.
  230  */
  231 int cxgb_snd_queue_len = IFQ_MAXLEN;
  232 TUNABLE_INT("hw.cxgb.snd_queue_len", &cxgb_snd_queue_len);
  233 SYSCTL_UINT(_hw_cxgb, OID_AUTO, snd_queue_len, CTLFLAG_RDTUN,
  234     &cxgb_snd_queue_len, 0, "send queue size ");
  235 
  236 
  237 enum {
  238         MAX_TXQ_ENTRIES      = 16384,
  239         MAX_CTRL_TXQ_ENTRIES = 1024,
  240         MAX_RSPQ_ENTRIES     = 16384,
  241         MAX_RX_BUFFERS       = 16384,
  242         MAX_RX_JUMBO_BUFFERS = 16384,
  243         MIN_TXQ_ENTRIES      = 4,
  244         MIN_CTRL_TXQ_ENTRIES = 4,
  245         MIN_RSPQ_ENTRIES     = 32,
  246         MIN_FL_ENTRIES       = 32,
  247         MIN_FL_JUMBO_ENTRIES = 32
  248 };
  249 
  250 struct filter_info {
  251         u32 sip;
  252         u32 sip_mask;
  253         u32 dip;
  254         u16 sport;
  255         u16 dport;
  256         u32 vlan:12;
  257         u32 vlan_prio:3;
  258         u32 mac_hit:1;
  259         u32 mac_idx:4;
  260         u32 mac_vld:1;
  261         u32 pkt_type:2;
  262         u32 report_filter_id:1;
  263         u32 pass:1;
  264         u32 rss:1;
  265         u32 qset:3;
  266         u32 locked:1;
  267         u32 valid:1;
  268 };
  269 
  270 enum { FILTER_NO_VLAN_PRI = 7 };
  271 
  272 #define EEPROM_MAGIC 0x38E2F10C
  273 
  274 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  275 
  276 /* Table for probing the cards.  The desc field isn't actually used */
  277 struct cxgb_ident {
  278         uint16_t        vendor;
  279         uint16_t        device;
  280         int             index;
  281         char            *desc;
  282 } cxgb_identifiers[] = {
  283         {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
  284         {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
  285         {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
  286         {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
  287         {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
  288         {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
  289         {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
  290         {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
  291         {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
  292         {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
  293         {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
  294         {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "N310E"},
  295         {0, 0, 0, NULL}
  296 };
  297 
  298 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
  299 
  300 
  301 static __inline char
  302 t3rev2char(struct adapter *adapter)
  303 {
  304         char rev = 'z';
  305 
  306         switch(adapter->params.rev) {
  307         case T3_REV_A:
  308                 rev = 'a';
  309                 break;
  310         case T3_REV_B:
  311         case T3_REV_B2:
  312                 rev = 'b';
  313                 break;
  314         case T3_REV_C:
  315                 rev = 'c';
  316                 break;
  317         }
  318         return rev;
  319 }
  320 
  321 static struct cxgb_ident *
  322 cxgb_get_ident(device_t dev)
  323 {
  324         struct cxgb_ident *id;
  325 
  326         for (id = cxgb_identifiers; id->desc != NULL; id++) {
  327                 if ((id->vendor == pci_get_vendor(dev)) &&
  328                     (id->device == pci_get_device(dev))) {
  329                         return (id);
  330                 }
  331         }
  332         return (NULL);
  333 }
  334 
  335 static const struct adapter_info *
  336 cxgb_get_adapter_info(device_t dev)
  337 {
  338         struct cxgb_ident *id;
  339         const struct adapter_info *ai;
  340 
  341         id = cxgb_get_ident(dev);
  342         if (id == NULL)
  343                 return (NULL);
  344 
  345         ai = t3_get_adapter_info(id->index);
  346 
  347         return (ai);
  348 }
  349 
  350 static int
  351 cxgb_controller_probe(device_t dev)
  352 {
  353         const struct adapter_info *ai;
  354         char *ports, buf[80];
  355         int nports;
  356         struct adapter *sc = device_get_softc(dev);
  357 
  358         ai = cxgb_get_adapter_info(dev);
  359         if (ai == NULL)
  360                 return (ENXIO);
  361 
  362         nports = ai->nports0 + ai->nports1;
  363         if (nports == 1)
  364                 ports = "port";
  365         else
  366                 ports = "ports";
  367 
  368         snprintf(buf, sizeof(buf), "%s %sNIC, rev: %d nports: %d %s",
  369             ai->desc, is_offload(sc) ? "R" : "",
  370             sc->params.rev, nports, ports);
  371         device_set_desc_copy(dev, buf);
  372         return (BUS_PROBE_DEFAULT);
  373 }
  374 
  375 #define FW_FNAME "cxgb_t3fw"
  376 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
  377 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
  378 
  379 static int
  380 upgrade_fw(adapter_t *sc)
  381 {
  382 #ifdef FIRMWARE_LATEST
  383         const struct firmware *fw;
  384 #else
  385         struct firmware *fw;
  386 #endif  
  387         int status;
  388         
  389         if ((fw = firmware_get(FW_FNAME)) == NULL)  {
  390                 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
  391                 return (ENOENT);
  392         } else
  393                 device_printf(sc->dev, "updating firmware on card\n");
  394         status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
  395 
  396         device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status);
  397         
  398         firmware_put(fw, FIRMWARE_UNLOAD);
  399 
  400         return (status);        
  401 }
  402 
  403 /*
  404  * The cxgb_controller_attach function is responsible for the initial
  405  * bringup of the device.  Its responsibilities include:
  406  *
  407  *  1. Determine if the device supports MSI or MSI-X.
  408  *  2. Allocate bus resources so that we can access the Base Address Register
  409  *  3. Create and initialize mutexes for the controller and its control
  410  *     logic such as SGE and MDIO.
  411  *  4. Call hardware specific setup routine for the adapter as a whole.
  412  *  5. Allocate the BAR for doing MSI-X.
  413  *  6. Setup the line interrupt iff MSI-X is not supported.
  414  *  7. Create the driver's taskq.
  415  *  8. Start one task queue service thread.
  416  *  9. Check if the firmware and SRAM are up-to-date.  They will be
  417  *     auto-updated later (before FULL_INIT_DONE), if required.
  418  * 10. Create a child device for each MAC (port)
  419  * 11. Initialize T3 private state.
  420  * 12. Trigger the LED
  421  * 13. Setup offload iff supported.
  422  * 14. Reset/restart the tick callout.
  423  * 15. Attach sysctls
  424  *
  425  * NOTE: Any modification or deviation from this list MUST be reflected in
  426  * the above comment.  Failure to do so will result in problems on various
  427  * error conditions including link flapping.
  428  */
  429 static int
  430 cxgb_controller_attach(device_t dev)
  431 {
  432         device_t child;
  433         const struct adapter_info *ai;
  434         struct adapter *sc;
  435         int i, error = 0;
  436         uint32_t vers;
  437         int port_qsets = 1;
  438 #ifdef MSI_SUPPORTED
  439         int msi_needed, reg;
  440 #endif
  441         char buf[80];
  442 
  443         sc = device_get_softc(dev);
  444         sc->dev = dev;
  445         sc->msi_count = 0;
  446         ai = cxgb_get_adapter_info(dev);
  447 
  448         /*
  449          * XXX not really related but a recent addition
  450          */
  451 #ifdef MSI_SUPPORTED    
  452         /* find the PCIe link width and set max read request to 4KB*/
  453         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
  454                 uint16_t lnk, pectl;
  455                 lnk = pci_read_config(dev, reg + 0x12, 2);
  456                 sc->link_width = (lnk >> 4) & 0x3f;
  457                 
  458                 pectl = pci_read_config(dev, reg + 0x8, 2);
  459                 pectl = (pectl & ~0x7000) | (5 << 12);
  460                 pci_write_config(dev, reg + 0x8, pectl, 2);
  461         }
  462 
  463         if (sc->link_width != 0 && sc->link_width <= 4 &&
  464             (ai->nports0 + ai->nports1) <= 2) {
  465                 device_printf(sc->dev,
  466                     "PCIe x%d Link, expect reduced performance\n",
  467                     sc->link_width);
  468         }
  469 #endif
  470         touch_bars(dev);
  471         pci_enable_busmaster(dev);
  472         /*
  473          * Allocate the registers and make them available to the driver.
  474          * The registers that we care about for NIC mode are in BAR 0
  475          */
  476         sc->regs_rid = PCIR_BAR(0);
  477         if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  478             &sc->regs_rid, RF_ACTIVE)) == NULL) {
  479                 device_printf(dev, "Cannot allocate BAR region 0\n");
  480                 return (ENXIO);
  481         }
  482         sc->udbs_rid = PCIR_BAR(2);
  483         sc->udbs_res = NULL;
  484         if (is_offload(sc) &&
  485             ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  486                    &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
  487                 device_printf(dev, "Cannot allocate BAR region 1\n");
  488                 error = ENXIO;
  489                 goto out;
  490         }
  491 
  492         snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
  493             device_get_unit(dev));
  494         ADAPTER_LOCK_INIT(sc, sc->lockbuf);
  495 
  496         snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
  497             device_get_unit(dev));
  498         snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
  499             device_get_unit(dev));
  500         snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
  501             device_get_unit(dev));
  502         
  503         MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
  504         MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
  505         MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
  506         
  507         sc->bt = rman_get_bustag(sc->regs_res);
  508         sc->bh = rman_get_bushandle(sc->regs_res);
  509         sc->mmio_len = rman_get_size(sc->regs_res);
  510 
  511         if (t3_prep_adapter(sc, ai, 1) < 0) {
  512                 printf("prep adapter failed\n");
  513                 error = ENODEV;
  514                 goto out;
  515         }
  516         /* Allocate the BAR for doing MSI-X.  If it succeeds, try to allocate
  517          * enough messages for the queue sets.  If that fails, try falling
  518          * back to MSI.  If that fails, then try falling back to the legacy
  519          * interrupt pin model.
  520          */
  521 #ifdef MSI_SUPPORTED
  522 
  523         sc->msix_regs_rid = 0x20;
  524         if ((msi_allowed >= 2) &&
  525             (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  526             &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
  527 
  528                 if (multiq)
  529                         port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
  530                 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
  531 
  532                 if (pci_msix_count(dev) == 0 ||
  533                     (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
  534                     sc->msi_count != msi_needed) {
  535                         device_printf(dev, "alloc msix failed - "
  536                                       "msi_count=%d, msi_needed=%d, err=%d; "
  537                                       "will try MSI\n", sc->msi_count,
  538                                       msi_needed, error);
  539                         sc->msi_count = 0;
  540                         port_qsets = 1;
  541                         pci_release_msi(dev);
  542                         bus_release_resource(dev, SYS_RES_MEMORY,
  543                             sc->msix_regs_rid, sc->msix_regs_res);
  544                         sc->msix_regs_res = NULL;
  545                 } else {
  546                         sc->flags |= USING_MSIX;
  547                         sc->cxgb_intr = cxgb_async_intr;
  548                         device_printf(dev,
  549                                       "using MSI-X interrupts (%u vectors)\n",
  550                                       sc->msi_count);
  551                 }
  552         }
  553 
  554         if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
  555                 sc->msi_count = 1;
  556                 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
  557                         device_printf(dev, "alloc msi failed - "
  558                                       "err=%d; will try INTx\n", error);
  559                         sc->msi_count = 0;
  560                         port_qsets = 1;
  561                         pci_release_msi(dev);
  562                 } else {
  563                         sc->flags |= USING_MSI;
  564                         sc->cxgb_intr = t3_intr_msi;
  565                         device_printf(dev, "using MSI interrupts\n");
  566                 }
  567         }
  568 #endif
  569         if (sc->msi_count == 0) {
  570                 device_printf(dev, "using line interrupts\n");
  571                 sc->cxgb_intr = t3b_intr;
  572         }
  573 
  574         /* Create a private taskqueue thread for handling driver events */
  575 #ifdef TASKQUEUE_CURRENT        
  576         sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
  577             taskqueue_thread_enqueue, &sc->tq);
  578 #else
  579         sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
  580             taskqueue_thread_enqueue, &sc->tq);
  581 #endif  
  582         if (sc->tq == NULL) {
  583                 device_printf(dev, "failed to allocate controller task queue\n");
  584                 goto out;
  585         }
  586 
  587         taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
  588             device_get_nameunit(dev));
  589         TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
  590         TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
  591 
  592         
  593         /* Create a periodic callout for checking adapter status */
  594         callout_init(&sc->cxgb_tick_ch, TRUE);
  595         
  596         if (t3_check_fw_version(sc) < 0 || force_fw_update) {
  597                 /*
  598                  * Warn user that a firmware update will be attempted in init.
  599                  */
  600                 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
  601                     FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
  602                 sc->flags &= ~FW_UPTODATE;
  603         } else {
  604                 sc->flags |= FW_UPTODATE;
  605         }
  606 
  607         if (t3_check_tpsram_version(sc) < 0) {
  608                 /*
  609                  * Warn user that a firmware update will be attempted in init.
  610                  */
  611                 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
  612                     t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
  613                 sc->flags &= ~TPS_UPTODATE;
  614         } else {
  615                 sc->flags |= TPS_UPTODATE;
  616         }
  617         
  618         /*
  619          * Create a child device for each MAC.  The ethernet attachment
  620          * will be done in these children.
  621          */     
  622         for (i = 0; i < (sc)->params.nports; i++) {
  623                 struct port_info *pi;
  624                 
  625                 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
  626                         device_printf(dev, "failed to add child port\n");
  627                         error = EINVAL;
  628                         goto out;
  629                 }
  630                 pi = &sc->port[i];
  631                 pi->adapter = sc;
  632                 pi->nqsets = port_qsets;
  633                 pi->first_qset = i*port_qsets;
  634                 pi->port_id = i;
  635                 pi->tx_chan = i >= ai->nports0;
  636                 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
  637                 sc->rxpkt_map[pi->txpkt_intf] = i;
  638                 sc->port[i].tx_chan = i >= ai->nports0;
  639                 sc->portdev[i] = child;
  640                 device_set_softc(child, pi);
  641         }
  642         if ((error = bus_generic_attach(dev)) != 0)
  643                 goto out;
  644 
  645         /* initialize sge private state */
  646         t3_sge_init_adapter(sc);
  647 
  648         t3_led_ready(sc);
  649         
  650         cxgb_offload_init();
  651         if (is_offload(sc)) {
  652                 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
  653                 cxgb_adapter_ofld(sc);
  654         }
  655         error = t3_get_fw_version(sc, &vers);
  656         if (error)
  657                 goto out;
  658 
  659         snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
  660             G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
  661             G_FW_VERSION_MICRO(vers));
  662 
  663         snprintf(buf, sizeof(buf), "%s\t E/C: %s S/N: %s", 
  664                  ai->desc,
  665                  sc->params.vpd.ec, sc->params.vpd.sn);
  666         device_set_desc_copy(dev, buf);
  667 
  668         snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
  669                  sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
  670                  sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
  671 
  672         device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
  673         callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
  674         t3_add_attach_sysctls(sc);
  675 out:
  676         if (error)
  677                 cxgb_free(sc);
  678 
  679         return (error);
  680 }
  681 
  682 /*
  683  * The cxgb_controller_detach routine is called with the device is
  684  * unloaded from the system.
  685  */
  686 
  687 static int
  688 cxgb_controller_detach(device_t dev)
  689 {
  690         struct adapter *sc;
  691 
  692         sc = device_get_softc(dev);
  693 
  694         cxgb_free(sc);
  695 
  696         return (0);
  697 }
  698 
  699 /*
  700  * The cxgb_free() is called by the cxgb_controller_detach() routine
  701  * to tear down the structures that were built up in
  702  * cxgb_controller_attach(), and should be the final piece of work
  703  * done when fully unloading the driver.
  704  * 
  705  *
  706  *  1. Shutting down the threads started by the cxgb_controller_attach()
  707  *     routine.
  708  *  2. Stopping the lower level device and all callouts (cxgb_down_locked()).
  709  *  3. Detaching all of the port devices created during the
  710  *     cxgb_controller_attach() routine.
  711  *  4. Removing the device children created via cxgb_controller_attach().
  712  *  5. Releasing PCI resources associated with the device.
  713  *  6. Turning off the offload support, iff it was turned on.
  714  *  7. Destroying the mutexes created in cxgb_controller_attach().
  715  *
  716  */
  717 static void
  718 cxgb_free(struct adapter *sc)
  719 {
  720         int i;
  721 
  722         ADAPTER_LOCK(sc);
  723         sc->flags |= CXGB_SHUTDOWN;
  724         ADAPTER_UNLOCK(sc);
  725 
  726         /*
  727          * Make sure all child devices are gone.
  728          */
  729         bus_generic_detach(sc->dev);
  730         for (i = 0; i < (sc)->params.nports; i++) {
  731                 if (sc->portdev[i] &&
  732                     device_delete_child(sc->dev, sc->portdev[i]) != 0)
  733                         device_printf(sc->dev, "failed to delete child port\n");
  734         }
  735 
  736         /*
  737          * At this point, it is as if cxgb_port_detach has run on all ports, and
  738          * cxgb_down has run on the adapter.  All interrupts have been silenced,
  739          * all open devices have been closed.
  740          */
  741         KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
  742                                            __func__, sc->open_device_map));
  743         for (i = 0; i < sc->params.nports; i++) {
  744                 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
  745                                                   __func__, i));
  746         }
  747 
  748         /*
  749          * Finish off the adapter's callouts.
  750          */
  751         callout_drain(&sc->cxgb_tick_ch);
  752         callout_drain(&sc->sge_timer_ch);
  753 
  754         /*
  755          * Release resources grabbed under FULL_INIT_DONE by cxgb_up.  The
  756          * sysctls are cleaned up by the kernel linker.
  757          */
  758         if (sc->flags & FULL_INIT_DONE) {
  759                 t3_free_sge_resources(sc);
  760                 sc->flags &= ~FULL_INIT_DONE;
  761         }
  762 
  763         /*
  764          * Release all interrupt resources.
  765          */
  766         cxgb_teardown_interrupts(sc);
  767 #ifdef MSI_SUPPORTED
  768         if (sc->flags & (USING_MSI | USING_MSIX)) {
  769                 device_printf(sc->dev, "releasing msi message(s)\n");
  770                 pci_release_msi(sc->dev);
  771         } else {
  772                 device_printf(sc->dev, "no msi message to release\n");
  773         }
  774 
  775         if (sc->msix_regs_res != NULL) {
  776                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
  777                     sc->msix_regs_res);
  778         }
  779 #endif
  780 
  781         /*
  782          * Free the adapter's taskqueue.
  783          */
  784         if (sc->tq != NULL) {
  785                 taskqueue_free(sc->tq);
  786                 sc->tq = NULL;
  787         }
  788         
  789         if (is_offload(sc)) {
  790                 clrbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
  791                 cxgb_adapter_unofld(sc);
  792         }
  793 
  794 #ifdef notyet
  795         if (sc->flags & CXGB_OFLD_INIT)
  796                 cxgb_offload_deactivate(sc);
  797 #endif
  798         free(sc->filters, M_DEVBUF);
  799         t3_sge_free(sc);
  800 
  801         cxgb_offload_exit();
  802 
  803         if (sc->udbs_res != NULL)
  804                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
  805                     sc->udbs_res);
  806 
  807         if (sc->regs_res != NULL)
  808                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
  809                     sc->regs_res);
  810 
  811         MTX_DESTROY(&sc->mdio_lock);
  812         MTX_DESTROY(&sc->sge.reg_lock);
  813         MTX_DESTROY(&sc->elmer_lock);
  814         ADAPTER_LOCK_DEINIT(sc);
  815 }
  816 
  817 /**
  818  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
  819  *      @sc: the controller softc
  820  *
  821  *      Determines how many sets of SGE queues to use and initializes them.
  822  *      We support multiple queue sets per port if we have MSI-X, otherwise
  823  *      just one queue set per port.
  824  */
  825 static int
  826 setup_sge_qsets(adapter_t *sc)
  827 {
  828         int i, j, err, irq_idx = 0, qset_idx = 0;
  829         u_int ntxq = SGE_TXQ_PER_SET;
  830 
  831         if ((err = t3_sge_alloc(sc)) != 0) {
  832                 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
  833                 return (err);
  834         }
  835 
  836         if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
  837                 irq_idx = -1;
  838 
  839         for (i = 0; i < (sc)->params.nports; i++) {
  840                 struct port_info *pi = &sc->port[i];
  841 
  842                 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
  843                         err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
  844                             (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
  845                             &sc->params.sge.qset[qset_idx], ntxq, pi);
  846                         if (err) {
  847                                 t3_free_sge_resources(sc);
  848                                 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n",
  849                                     err);
  850                                 return (err);
  851                         }
  852                 }
  853         }
  854 
  855         return (0);
  856 }
  857 
  858 static void
  859 cxgb_teardown_interrupts(adapter_t *sc)
  860 {
  861         int i;
  862 
  863         for (i = 0; i < SGE_QSETS; i++) {
  864                 if (sc->msix_intr_tag[i] == NULL) {
  865 
  866                         /* Should have been setup fully or not at all */
  867                         KASSERT(sc->msix_irq_res[i] == NULL &&
  868                                 sc->msix_irq_rid[i] == 0,
  869                                 ("%s: half-done interrupt (%d).", __func__, i));
  870 
  871                         continue;
  872                 }
  873 
  874                 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
  875                                   sc->msix_intr_tag[i]);
  876                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
  877                                      sc->msix_irq_res[i]);
  878 
  879                 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
  880                 sc->msix_irq_rid[i] = 0;
  881         }
  882 
  883         if (sc->intr_tag) {
  884                 KASSERT(sc->irq_res != NULL,
  885                         ("%s: half-done interrupt.", __func__));
  886 
  887                 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
  888                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  889                                      sc->irq_res);
  890 
  891                 sc->irq_res = sc->intr_tag = NULL;
  892                 sc->irq_rid = 0;
  893         }
  894 }
  895 
  896 static int
  897 cxgb_setup_interrupts(adapter_t *sc)
  898 {
  899         struct resource *res;
  900         void *tag;
  901         int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
  902 
  903         sc->irq_rid = intr_flag ? 1 : 0;
  904         sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
  905                                              RF_SHAREABLE | RF_ACTIVE);
  906         if (sc->irq_res == NULL) {
  907                 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
  908                               intr_flag, sc->irq_rid);
  909                 err = EINVAL;
  910                 sc->irq_rid = 0;
  911         } else {
  912                 err = bus_setup_intr(sc->dev, sc->irq_res,
  913                                      INTR_MPSAFE | INTR_TYPE_NET,
  914 #ifdef INTR_FILTERS
  915                                      NULL,
  916 #endif
  917                                      sc->cxgb_intr, sc, &sc->intr_tag);
  918 
  919                 if (err) {
  920                         device_printf(sc->dev,
  921                                       "Cannot set up interrupt (%x, %u, %d)\n",
  922                                       intr_flag, sc->irq_rid, err);
  923                         bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  924                                              sc->irq_res);
  925                         sc->irq_res = sc->intr_tag = NULL;
  926                         sc->irq_rid = 0;
  927                 }
  928         }
  929 
  930         /* That's all for INTx or MSI */
  931         if (!(intr_flag & USING_MSIX) || err)
  932                 return (err);
  933 
  934         for (i = 0; i < sc->msi_count - 1; i++) {
  935                 rid = i + 2;
  936                 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
  937                                              RF_SHAREABLE | RF_ACTIVE);
  938                 if (res == NULL) {
  939                         device_printf(sc->dev, "Cannot allocate interrupt "
  940                                       "for message %d\n", rid);
  941                         err = EINVAL;
  942                         break;
  943                 }
  944 
  945                 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
  946 #ifdef INTR_FILTERS
  947                                      NULL,
  948 #endif
  949                                      t3_intr_msix, &sc->sge.qs[i], &tag);
  950                 if (err) {
  951                         device_printf(sc->dev, "Cannot set up interrupt "
  952                                       "for message %d (%d)\n", rid, err);
  953                         bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
  954                         break;
  955                 }
  956 
  957                 sc->msix_irq_rid[i] = rid;
  958                 sc->msix_irq_res[i] = res;
  959                 sc->msix_intr_tag[i] = tag;
  960         }
  961 
  962         if (err)
  963                 cxgb_teardown_interrupts(sc);
  964 
  965         return (err);
  966 }
  967 
  968 
  969 static int
  970 cxgb_port_probe(device_t dev)
  971 {
  972         struct port_info *p;
  973         char buf[80];
  974         const char *desc;
  975         
  976         p = device_get_softc(dev);
  977         desc = p->phy.desc;
  978         snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
  979         device_set_desc_copy(dev, buf);
  980         return (0);
  981 }
  982 
  983 
  984 static int
  985 cxgb_makedev(struct port_info *pi)
  986 {
  987         
  988         pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
  989             UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
  990         
  991         if (pi->port_cdev == NULL)
  992                 return (ENOMEM);
  993 
  994         pi->port_cdev->si_drv1 = (void *)pi;
  995         
  996         return (0);
  997 }
  998 
  999 #ifndef LRO_SUPPORTED
 1000 #ifdef IFCAP_LRO
 1001 #undef IFCAP_LRO
 1002 #endif
 1003 #define IFCAP_LRO 0x0
 1004 #endif
 1005 
 1006 #ifdef TSO_SUPPORTED
 1007 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO)
 1008 /* Don't enable TSO6 yet */
 1009 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO)
 1010 #else
 1011 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
 1012 /* Don't enable TSO6 yet */
 1013 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM |  IFCAP_JUMBO_MTU)
 1014 #define IFCAP_TSO4 0x0
 1015 #define IFCAP_TSO6 0x0
 1016 #define CSUM_TSO   0x0
 1017 #endif
 1018 
 1019 
 1020 static int
 1021 cxgb_port_attach(device_t dev)
 1022 {
 1023         struct port_info *p;
 1024         struct ifnet *ifp;
 1025         int err;
 1026         struct adapter *sc;
 1027         
 1028         
 1029         p = device_get_softc(dev);
 1030         sc = p->adapter;
 1031         snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
 1032             device_get_unit(device_get_parent(dev)), p->port_id);
 1033         PORT_LOCK_INIT(p, p->lockbuf);
 1034 
 1035         /* Allocate an ifnet object and set it up */
 1036         ifp = p->ifp = if_alloc(IFT_ETHER);
 1037         if (ifp == NULL) {
 1038                 device_printf(dev, "Cannot allocate ifnet\n");
 1039                 return (ENOMEM);
 1040         }
 1041         
 1042         /*
 1043          * Note that there is currently no watchdog timer.
 1044          */
 1045         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 1046         ifp->if_init = cxgb_init;
 1047         ifp->if_softc = p;
 1048         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 1049         ifp->if_ioctl = cxgb_ioctl;
 1050         ifp->if_start = cxgb_start;
 1051 
 1052 
 1053         ifp->if_timer = 0;      /* Disable ifnet watchdog */
 1054         ifp->if_watchdog = NULL;
 1055 
 1056         ifp->if_snd.ifq_drv_maxlen = cxgb_snd_queue_len;
 1057         IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
 1058         IFQ_SET_READY(&ifp->if_snd);
 1059 
 1060         ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
 1061         ifp->if_capabilities |= CXGB_CAP;
 1062         ifp->if_capenable |= CXGB_CAP_ENABLE;
 1063         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
 1064         /*
 1065          * disable TSO on 4-port - it isn't supported by the firmware yet
 1066          */     
 1067         if (p->adapter->params.nports > 2) {
 1068                 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6);
 1069                 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6);
 1070                 ifp->if_hwassist &= ~CSUM_TSO;
 1071         }
 1072 
 1073         ether_ifattach(ifp, p->hw_addr);
 1074         ifp->if_transmit = cxgb_transmit;
 1075         ifp->if_qflush = cxgb_qflush;
 1076 
 1077         /*
 1078          * Only default to jumbo frames on 10GigE
 1079          */
 1080         if (p->adapter->params.nports <= 2)
 1081                 ifp->if_mtu = ETHERMTU_JUMBO;
 1082         if ((err = cxgb_makedev(p)) != 0) {
 1083                 printf("makedev failed %d\n", err);
 1084                 return (err);
 1085         }
 1086 
 1087         /* Create a list of media supported by this port */
 1088         ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
 1089             cxgb_media_status);
 1090         cxgb_build_medialist(p);
 1091       
 1092         t3_sge_init_port(p);
 1093 
 1094         return (err);
 1095 }
 1096 
 1097 /*
 1098  * cxgb_port_detach() is called via the device_detach methods when
 1099  * cxgb_free() calls the bus_generic_detach.  It is responsible for 
 1100  * removing the device from the view of the kernel, i.e. from all 
 1101  * interfaces lists etc.  This routine is only called when the driver is 
 1102  * being unloaded, not when the link goes down.
 1103  */
 1104 static int
 1105 cxgb_port_detach(device_t dev)
 1106 {
 1107         struct port_info *p;
 1108         struct adapter *sc;
 1109         int i;
 1110 
 1111         p = device_get_softc(dev);
 1112         sc = p->adapter;
 1113 
 1114         cxgb_begin_detach(p);
 1115 
 1116         if (p->port_cdev != NULL)
 1117                 destroy_dev(p->port_cdev);
 1118 
 1119         cxgb_uninit_synchronized(p);
 1120         ether_ifdetach(p->ifp);
 1121 
 1122         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
 1123                 struct sge_qset *qs = &sc->sge.qs[i];
 1124                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
 1125 
 1126                 callout_drain(&txq->txq_watchdog);
 1127                 callout_drain(&txq->txq_timer);
 1128         }
 1129 
 1130         PORT_LOCK_DEINIT(p);
 1131         if_free(p->ifp);
 1132         p->ifp = NULL;
 1133 
 1134         cxgb_end_op(p);
 1135         return (0);
 1136 }
 1137 
 1138 void
 1139 t3_fatal_err(struct adapter *sc)
 1140 {
 1141         u_int fw_status[4];
 1142 
 1143         if (sc->flags & FULL_INIT_DONE) {
 1144                 t3_sge_stop(sc);
 1145                 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
 1146                 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
 1147                 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
 1148                 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
 1149                 t3_intr_disable(sc);
 1150         }
 1151         device_printf(sc->dev,"encountered fatal error, operation suspended\n");
 1152         if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
 1153                 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
 1154                     fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
 1155 }
 1156 
 1157 int
 1158 t3_os_find_pci_capability(adapter_t *sc, int cap)
 1159 {
 1160         device_t dev;
 1161         struct pci_devinfo *dinfo;
 1162         pcicfgregs *cfg;
 1163         uint32_t status;
 1164         uint8_t ptr;
 1165 
 1166         dev = sc->dev;
 1167         dinfo = device_get_ivars(dev);
 1168         cfg = &dinfo->cfg;
 1169 
 1170         status = pci_read_config(dev, PCIR_STATUS, 2);
 1171         if (!(status & PCIM_STATUS_CAPPRESENT))
 1172                 return (0);
 1173 
 1174         switch (cfg->hdrtype & PCIM_HDRTYPE) {
 1175         case 0:
 1176         case 1:
 1177                 ptr = PCIR_CAP_PTR;
 1178                 break;
 1179         case 2:
 1180                 ptr = PCIR_CAP_PTR_2;
 1181                 break;
 1182         default:
 1183                 return (0);
 1184                 break;
 1185         }
 1186         ptr = pci_read_config(dev, ptr, 1);
 1187 
 1188         while (ptr != 0) {
 1189                 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
 1190                         return (ptr);
 1191                 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
 1192         }
 1193 
 1194         return (0);
 1195 }
 1196 
 1197 int
 1198 t3_os_pci_save_state(struct adapter *sc)
 1199 {
 1200         device_t dev;
 1201         struct pci_devinfo *dinfo;
 1202 
 1203         dev = sc->dev;
 1204         dinfo = device_get_ivars(dev);
 1205 
 1206         pci_cfg_save(dev, dinfo, 0);
 1207         return (0);
 1208 }
 1209 
 1210 int
 1211 t3_os_pci_restore_state(struct adapter *sc)
 1212 {
 1213         device_t dev;
 1214         struct pci_devinfo *dinfo;
 1215 
 1216         dev = sc->dev;
 1217         dinfo = device_get_ivars(dev);
 1218 
 1219         pci_cfg_restore(dev, dinfo);
 1220         return (0);
 1221 }
 1222 
 1223 /**
 1224  *      t3_os_link_changed - handle link status changes
 1225  *      @adapter: the adapter associated with the link change
 1226  *      @port_id: the port index whose limk status has changed
 1227  *      @link_status: the new status of the link
 1228  *      @speed: the new speed setting
 1229  *      @duplex: the new duplex setting
 1230  *      @fc: the new flow-control setting
 1231  *
 1232  *      This is the OS-dependent handler for link status changes.  The OS
 1233  *      neutral handler takes care of most of the processing for these events,
 1234  *      then calls this handler for any OS-specific processing.
 1235  */
 1236 void
 1237 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
 1238      int duplex, int fc)
 1239 {
 1240         struct port_info *pi = &adapter->port[port_id];
 1241         struct ifnet *ifp = pi->ifp;
 1242 
 1243         /* no race with detach, so ifp should always be good */
 1244         KASSERT(ifp, ("%s: if detached.", __func__));
 1245 
 1246         if (link_status) {
 1247                 ifp->if_baudrate = IF_Mbps(speed);
 1248                 if_link_state_change(ifp, LINK_STATE_UP);
 1249         } else
 1250                 if_link_state_change(ifp, LINK_STATE_DOWN);
 1251 }
 1252 
 1253 /**
 1254  *      t3_os_phymod_changed - handle PHY module changes
 1255  *      @phy: the PHY reporting the module change
 1256  *      @mod_type: new module type
 1257  *
 1258  *      This is the OS-dependent handler for PHY module changes.  It is
 1259  *      invoked when a PHY module is removed or inserted for any OS-specific
 1260  *      processing.
 1261  */
 1262 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 1263 {
 1264         static const char *mod_str[] = {
 1265                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 1266         };
 1267         struct port_info *pi = &adap->port[port_id];
 1268         int mod = pi->phy.modtype;
 1269 
 1270         if (mod != pi->media.ifm_cur->ifm_data)
 1271                 cxgb_build_medialist(pi);
 1272 
 1273         if (mod == phy_modtype_none)
 1274                 if_printf(pi->ifp, "PHY module unplugged\n");
 1275         else {
 1276                 KASSERT(mod < ARRAY_SIZE(mod_str),
 1277                         ("invalid PHY module type %d", mod));
 1278                 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
 1279         }
 1280 }
 1281 
 1282 /*
 1283  * Interrupt-context handler for external (PHY) interrupts.
 1284  */
 1285 void
 1286 t3_os_ext_intr_handler(adapter_t *sc)
 1287 {
 1288         if (cxgb_debug)
 1289                 printf("t3_os_ext_intr_handler\n");
 1290         /*
 1291          * Schedule a task to handle external interrupts as they may be slow
 1292          * and we use a mutex to protect MDIO registers.  We disable PHY
 1293          * interrupts in the meantime and let the task reenable them when
 1294          * it's done.
 1295          */
 1296         if (sc->slow_intr_mask) {
 1297                 ADAPTER_LOCK(sc);
 1298                 sc->slow_intr_mask &= ~F_T3DBG;
 1299                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
 1300                 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
 1301                 ADAPTER_UNLOCK(sc);
 1302         }
 1303 }
 1304 
 1305 void
 1306 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
 1307 {
 1308 
 1309         /*
 1310          * The ifnet might not be allocated before this gets called,
 1311          * as this is called early on in attach by t3_prep_adapter
 1312          * save the address off in the port structure
 1313          */
 1314         if (cxgb_debug)
 1315                 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
 1316         bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
 1317 }
 1318 
 1319 /*
 1320  * Programs the XGMAC based on the settings in the ifnet.  These settings
 1321  * include MTU, MAC address, mcast addresses, etc.
 1322  */
 1323 static void
 1324 cxgb_update_mac_settings(struct port_info *p)
 1325 {
 1326         struct ifnet *ifp = p->ifp;
 1327         struct t3_rx_mode rm;
 1328         struct cmac *mac = &p->mac;
 1329         int mtu, hwtagging;
 1330 
 1331         PORT_LOCK_ASSERT_OWNED(p);
 1332 
 1333         bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
 1334 
 1335         mtu = ifp->if_mtu;
 1336         if (ifp->if_capenable & IFCAP_VLAN_MTU)
 1337                 mtu += ETHER_VLAN_ENCAP_LEN;
 1338 
 1339         hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
 1340 
 1341         t3_mac_set_mtu(mac, mtu);
 1342         t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
 1343         t3_mac_set_address(mac, 0, p->hw_addr);
 1344         t3_init_rx_mode(&rm, p);
 1345         t3_mac_set_rx_mode(mac, &rm);
 1346 }
 1347 
 1348 
 1349 static int
 1350 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 1351                               unsigned long n)
 1352 {
 1353         int attempts = 5;
 1354 
 1355         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 1356                 if (!--attempts)
 1357                         return (ETIMEDOUT);
 1358                 t3_os_sleep(10);
 1359         }
 1360         return 0;
 1361 }
 1362 
 1363 static int
 1364 init_tp_parity(struct adapter *adap)
 1365 {
 1366         int i;
 1367         struct mbuf *m;
 1368         struct cpl_set_tcb_field *greq;
 1369         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 1370 
 1371         t3_tp_set_offload_mode(adap, 1);
 1372 
 1373         for (i = 0; i < 16; i++) {
 1374                 struct cpl_smt_write_req *req;
 1375 
 1376                 m = m_gethdr(M_WAITOK, MT_DATA);
 1377                 req = mtod(m, struct cpl_smt_write_req *);
 1378                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1379                 memset(req, 0, sizeof(*req));
 1380                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1381                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 1382                 req->iff = i;
 1383                 t3_mgmt_tx(adap, m);
 1384         }
 1385 
 1386         for (i = 0; i < 2048; i++) {
 1387                 struct cpl_l2t_write_req *req;
 1388 
 1389                 m = m_gethdr(M_WAITOK, MT_DATA);
 1390                 req = mtod(m, struct cpl_l2t_write_req *);
 1391                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1392                 memset(req, 0, sizeof(*req));
 1393                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1394                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 1395                 req->params = htonl(V_L2T_W_IDX(i));
 1396                 t3_mgmt_tx(adap, m);
 1397         }
 1398 
 1399         for (i = 0; i < 2048; i++) {
 1400                 struct cpl_rte_write_req *req;
 1401 
 1402                 m = m_gethdr(M_WAITOK, MT_DATA);
 1403                 req = mtod(m, struct cpl_rte_write_req *);
 1404                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1405                 memset(req, 0, sizeof(*req));
 1406                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1407                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 1408                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 1409                 t3_mgmt_tx(adap, m);
 1410         }
 1411 
 1412         m = m_gethdr(M_WAITOK, MT_DATA);
 1413         greq = mtod(m, struct cpl_set_tcb_field *);
 1414         m->m_len = m->m_pkthdr.len = sizeof(*greq);
 1415         memset(greq, 0, sizeof(*greq));
 1416         greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1417         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 1418         greq->mask = htobe64(1);
 1419         t3_mgmt_tx(adap, m);
 1420 
 1421         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 1422         t3_tp_set_offload_mode(adap, 0);
 1423         return (i);
 1424 }
 1425 
 1426 /**
 1427  *      setup_rss - configure Receive Side Steering (per-queue connection demux) 
 1428  *      @adap: the adapter
 1429  *
 1430  *      Sets up RSS to distribute packets to multiple receive queues.  We
 1431  *      configure the RSS CPU lookup table to distribute to the number of HW
 1432  *      receive queues, and the response queue lookup table to narrow that
 1433  *      down to the response queues actually configured for each port.
 1434  *      We always configure the RSS mapping for two ports since the mapping
 1435  *      table has plenty of entries.
 1436  */
 1437 static void
 1438 setup_rss(adapter_t *adap)
 1439 {
 1440         int i;
 1441         u_int nq[2]; 
 1442         uint8_t cpus[SGE_QSETS + 1];
 1443         uint16_t rspq_map[RSS_TABLE_SIZE];
 1444         
 1445         for (i = 0; i < SGE_QSETS; ++i)
 1446                 cpus[i] = i;
 1447         cpus[SGE_QSETS] = 0xff;
 1448 
 1449         nq[0] = nq[1] = 0;
 1450         for_each_port(adap, i) {
 1451                 const struct port_info *pi = adap2pinfo(adap, i);
 1452 
 1453                 nq[pi->tx_chan] += pi->nqsets;
 1454         }
 1455         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 1456                 rspq_map[i] = nq[0] ? i % nq[0] : 0;
 1457                 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
 1458         }
 1459         /* Calculate the reverse RSS map table */
 1460         for (i = 0; i < RSS_TABLE_SIZE; ++i)
 1461                 if (adap->rrss_map[rspq_map[i]] == 0xff)
 1462                         adap->rrss_map[rspq_map[i]] = i;
 1463 
 1464         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 1465                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
 1466                       F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
 1467                       cpus, rspq_map);
 1468 
 1469 }
 1470 
 1471 /*
 1472  * Sends an mbuf to an offload queue driver
 1473  * after dealing with any active network taps.
 1474  */
 1475 static inline int
 1476 offload_tx(struct t3cdev *tdev, struct mbuf *m)
 1477 {
 1478         int ret;
 1479 
 1480         ret = t3_offload_tx(tdev, m);
 1481         return (ret);
 1482 }
 1483 
 1484 static int
 1485 write_smt_entry(struct adapter *adapter, int idx)
 1486 {
 1487         struct port_info *pi = &adapter->port[idx];
 1488         struct cpl_smt_write_req *req;
 1489         struct mbuf *m;
 1490 
 1491         if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
 1492                 return (ENOMEM);
 1493 
 1494         req = mtod(m, struct cpl_smt_write_req *);
 1495         m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
 1496         
 1497         req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1498         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 1499         req->mtu_idx = NMTUS - 1;  /* should be 0 but there's a T3 bug */
 1500         req->iff = idx;
 1501         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 1502         memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
 1503 
 1504         m_set_priority(m, 1);
 1505 
 1506         offload_tx(&adapter->tdev, m);
 1507 
 1508         return (0);
 1509 }
 1510 
 1511 static int
 1512 init_smt(struct adapter *adapter)
 1513 {
 1514         int i;
 1515 
 1516         for_each_port(adapter, i)
 1517                 write_smt_entry(adapter, i);
 1518         return 0;
 1519 }
 1520 
 1521 static void
 1522 init_port_mtus(adapter_t *adapter)
 1523 {
 1524         unsigned int mtus = ETHERMTU | (ETHERMTU << 16);
 1525 
 1526         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 1527 }
 1528 
 1529 static void
 1530 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 1531                               int hi, int port)
 1532 {
 1533         struct mbuf *m;
 1534         struct mngt_pktsched_wr *req;
 1535 
 1536         m = m_gethdr(M_DONTWAIT, MT_DATA);
 1537         if (m) {        
 1538                 req = mtod(m, struct mngt_pktsched_wr *);
 1539                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 1540                 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 1541                 req->sched = sched;
 1542                 req->idx = qidx;
 1543                 req->min = lo;
 1544                 req->max = hi;
 1545                 req->binding = port;
 1546                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1547                 t3_mgmt_tx(adap, m);
 1548         }
 1549 }
 1550 
 1551 static void
 1552 bind_qsets(adapter_t *sc)
 1553 {
 1554         int i, j;
 1555 
 1556         for (i = 0; i < (sc)->params.nports; ++i) {
 1557                 const struct port_info *pi = adap2pinfo(sc, i);
 1558 
 1559                 for (j = 0; j < pi->nqsets; ++j) {
 1560                         send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
 1561                                           -1, pi->tx_chan);
 1562 
 1563                 }
 1564         }
 1565 }
 1566 
 1567 static void
 1568 update_tpeeprom(struct adapter *adap)
 1569 {
 1570 #ifdef FIRMWARE_LATEST
 1571         const struct firmware *tpeeprom;
 1572 #else
 1573         struct firmware *tpeeprom;
 1574 #endif
 1575 
 1576         uint32_t version;
 1577         unsigned int major, minor;
 1578         int ret, len;
 1579         char rev, name[32];
 1580 
 1581         t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
 1582 
 1583         major = G_TP_VERSION_MAJOR(version);
 1584         minor = G_TP_VERSION_MINOR(version);
 1585         if (major == TP_VERSION_MAJOR  && minor == TP_VERSION_MINOR)
 1586                 return; 
 1587 
 1588         rev = t3rev2char(adap);
 1589         snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
 1590 
 1591         tpeeprom = firmware_get(name);
 1592         if (tpeeprom == NULL) {
 1593                 device_printf(adap->dev,
 1594                               "could not load TP EEPROM: unable to load %s\n",
 1595                               name);
 1596                 return;
 1597         }
 1598 
 1599         len = tpeeprom->datasize - 4;
 1600         
 1601         ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
 1602         if (ret)
 1603                 goto release_tpeeprom;
 1604 
 1605         if (len != TP_SRAM_LEN) {
 1606                 device_printf(adap->dev,
 1607                               "%s length is wrong len=%d expected=%d\n", name,
 1608                               len, TP_SRAM_LEN);
 1609                 return;
 1610         }
 1611         
 1612         ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
 1613             TP_SRAM_OFFSET);
 1614         
 1615         if (!ret) {
 1616                 device_printf(adap->dev,
 1617                         "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
 1618                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 1619         } else 
 1620                 device_printf(adap->dev,
 1621                               "Protocol SRAM image update in EEPROM failed\n");
 1622 
 1623 release_tpeeprom:
 1624         firmware_put(tpeeprom, FIRMWARE_UNLOAD);
 1625         
 1626         return;
 1627 }
 1628 
 1629 static int
 1630 update_tpsram(struct adapter *adap)
 1631 {
 1632 #ifdef FIRMWARE_LATEST
 1633         const struct firmware *tpsram;
 1634 #else
 1635         struct firmware *tpsram;
 1636 #endif  
 1637         int ret;
 1638         char rev, name[32];
 1639 
 1640         rev = t3rev2char(adap);
 1641         snprintf(name, sizeof(name), TPSRAM_NAME, rev);
 1642 
 1643         update_tpeeprom(adap);
 1644 
 1645         tpsram = firmware_get(name);
 1646         if (tpsram == NULL){
 1647                 device_printf(adap->dev, "could not load TP SRAM\n");
 1648                 return (EINVAL);
 1649         } else
 1650                 device_printf(adap->dev, "updating TP SRAM\n");
 1651         
 1652         ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
 1653         if (ret)
 1654                 goto release_tpsram;    
 1655 
 1656         ret = t3_set_proto_sram(adap, tpsram->data);
 1657         if (ret)
 1658                 device_printf(adap->dev, "loading protocol SRAM failed\n");
 1659 
 1660 release_tpsram:
 1661         firmware_put(tpsram, FIRMWARE_UNLOAD);
 1662         
 1663         return ret;
 1664 }
 1665 
 1666 /**
 1667  *      cxgb_up - enable the adapter
 1668  *      @adap: adapter being enabled
 1669  *
 1670  *      Called when the first port is enabled, this function performs the
 1671  *      actions necessary to make an adapter operational, such as completing
 1672  *      the initialization of HW modules, and enabling interrupts.
 1673  */
 1674 static int
 1675 cxgb_up(struct adapter *sc)
 1676 {
 1677         int err = 0;
 1678 
 1679         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1680         KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
 1681                                            __func__, sc->open_device_map));
 1682 
 1683         if ((sc->flags & FULL_INIT_DONE) == 0) {
 1684 
 1685                 if ((sc->flags & FW_UPTODATE) == 0)
 1686                         if ((err = upgrade_fw(sc)))
 1687                                 goto out;
 1688 
 1689                 if ((sc->flags & TPS_UPTODATE) == 0)
 1690                         if ((err = update_tpsram(sc)))
 1691                                 goto out;
 1692 
 1693                 err = t3_init_hw(sc, 0);
 1694                 if (err)
 1695                         goto out;
 1696 
 1697                 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 1698                 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 1699 
 1700                 err = setup_sge_qsets(sc);
 1701                 if (err)
 1702                         goto out;
 1703 
 1704                 setup_rss(sc);
 1705 
 1706                 t3_intr_clear(sc);
 1707                 err = cxgb_setup_interrupts(sc);
 1708                 if (err)
 1709                         goto out;
 1710 
 1711                 t3_add_configured_sysctls(sc);
 1712                 sc->flags |= FULL_INIT_DONE;
 1713         }
 1714 
 1715         t3_intr_clear(sc);
 1716         t3_sge_start(sc);
 1717         t3_intr_enable(sc);
 1718 
 1719         if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
 1720             is_offload(sc) && init_tp_parity(sc) == 0)
 1721                 sc->flags |= TP_PARITY_INIT;
 1722 
 1723         if (sc->flags & TP_PARITY_INIT) {
 1724                 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
 1725                 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
 1726         }
 1727         
 1728         if (!(sc->flags & QUEUES_BOUND)) {
 1729                 bind_qsets(sc);
 1730                 sc->flags |= QUEUES_BOUND;              
 1731         }
 1732 
 1733         t3_sge_reset_adapter(sc);
 1734 out:
 1735         return (err);
 1736 }
 1737 
 1738 /*
 1739  * Called when the last open device is closed.  Does NOT undo all of cxgb_up's
 1740  * work.  Specifically, the resources grabbed under FULL_INIT_DONE are released
 1741  * during controller_detach, not here.
 1742  */
 1743 static void
 1744 cxgb_down(struct adapter *sc)
 1745 {
 1746         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1747 
 1748         t3_sge_stop(sc);
 1749         t3_intr_disable(sc);
 1750 }
 1751 
 1752 static int
 1753 offload_open(struct port_info *pi)
 1754 {
 1755         struct adapter *sc = pi->adapter;
 1756         struct t3cdev *tdev = &sc->tdev;
 1757 
 1758         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1759 
 1760         setbit(&sc->open_device_map, OFFLOAD_DEVMAP_BIT);
 1761 
 1762         t3_tp_set_offload_mode(sc, 1);
 1763         tdev->lldev = pi->ifp;
 1764         init_port_mtus(sc);
 1765         t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd,
 1766                      sc->params.rev == 0 ?  sc->port[0].ifp->if_mtu : 0xffff);
 1767         init_smt(sc);
 1768         cxgb_add_clients(tdev);
 1769 
 1770         return (0);
 1771 }
 1772 
 1773 static int
 1774 offload_close(struct t3cdev *tdev)
 1775 {
 1776         struct adapter *adapter = tdev2adap(tdev);
 1777 
 1778         if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
 1779                 return (0);
 1780 
 1781         /* Call back all registered clients */
 1782         cxgb_remove_clients(tdev);
 1783 
 1784         tdev->lldev = NULL;
 1785         cxgb_set_dummy_ops(tdev);
 1786         t3_tp_set_offload_mode(adapter, 0);
 1787 
 1788         clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
 1789 
 1790         return (0);
 1791 }
 1792 
 1793 /*
 1794  * Begin a synchronized operation.  If this call succeeds, it is guaranteed that
 1795  * no one will remove the port or its ifp from underneath the caller.  Caller is
 1796  * also granted exclusive access to open_device_map.
 1797  *
 1798  * operation here means init, uninit, detach, and ioctl service.
 1799  *
 1800  * May fail.
 1801  * EINTR (ctrl-c pressed during ifconfig for example).
 1802  * ENXIO (port is about to detach - due to kldunload for example).
 1803  */
 1804 int
 1805 cxgb_begin_op(struct port_info *p, const char *wmsg)
 1806 {
 1807         int rc = 0;
 1808         struct adapter *sc = p->adapter;
 1809 
 1810         ADAPTER_LOCK(sc);
 1811 
 1812         while (!IS_DOOMED(p) && IS_BUSY(sc)) {
 1813                 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, wmsg, 0)) {
 1814                         rc = EINTR;
 1815                         goto done;
 1816                 }
 1817         }
 1818 
 1819         if (IS_DOOMED(p))
 1820                 rc = ENXIO;
 1821         else if (!IS_BUSY(sc))
 1822                 SET_BUSY(sc);
 1823         else {
 1824                 KASSERT(0, ("%s: port %d, p->flags = %x , sc->flags = %x",
 1825                             __func__, p->port_id, p->flags, sc->flags));
 1826                 rc = EDOOFUS;
 1827         }
 1828 
 1829 done:
 1830         ADAPTER_UNLOCK(sc);
 1831         return (rc);
 1832 }
 1833 
 1834 /*
 1835  * End a synchronized operation.  Read comment block above cxgb_begin_op.
 1836  */
 1837 int
 1838 cxgb_end_op(struct port_info *p)
 1839 {
 1840         struct adapter *sc = p->adapter;
 1841 
 1842         ADAPTER_LOCK(sc);
 1843         KASSERT(IS_BUSY(sc), ("%s: not busy.", __func__));
 1844         CLR_BUSY(sc);
 1845         wakeup_one(&sc->flags);
 1846         ADAPTER_UNLOCK(sc);
 1847 
 1848         return (0);
 1849 }
 1850 
 1851 /*
 1852  * Prepare for port detachment.  Detach is a special kind of synchronized
 1853  * operation.  Also read comment before cxgb_begin_op.
 1854  */
 1855 static int
 1856 cxgb_begin_detach(struct port_info *p)
 1857 {
 1858         struct adapter *sc = p->adapter;
 1859 
 1860         /*
 1861          * Inform those waiting for this port that it is going to be destroyed
 1862          * and they should not continue further.  (They'll return with ENXIO).
 1863          */
 1864         ADAPTER_LOCK(sc);
 1865         SET_DOOMED(p);
 1866         wakeup(&sc->flags);
 1867         ADAPTER_UNLOCK(sc);
 1868 
 1869         /*
 1870          * Wait for in-progress operations.
 1871          */
 1872         ADAPTER_LOCK(sc);
 1873         while (IS_BUSY(sc)) {
 1874                 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
 1875         }
 1876         SET_BUSY(sc);
 1877         ADAPTER_UNLOCK(sc);
 1878 
 1879         return (0);
 1880 }
 1881 
 1882 /*
 1883  * if_init for cxgb ports.
 1884  */
 1885 static void
 1886 cxgb_init(void *arg)
 1887 {
 1888         struct port_info *p = arg;
 1889 
 1890         if (cxgb_begin_op(p, "cxgbinit"))
 1891                 return;
 1892 
 1893         cxgb_init_synchronized(p);
 1894         cxgb_end_op(p);
 1895 }
 1896 
 1897 static int
 1898 cxgb_init_synchronized(struct port_info *p)
 1899 {
 1900         struct adapter *sc = p->adapter;
 1901         struct ifnet *ifp = p->ifp;
 1902         struct cmac *mac = &p->mac;
 1903         int i, rc;
 1904 
 1905         if (sc->open_device_map == 0) {
 1906                 if ((rc = cxgb_up(sc)) != 0)
 1907                         return (rc);
 1908 
 1909                 if (is_offload(sc) && !ofld_disable && offload_open(p))
 1910                         log(LOG_WARNING,
 1911                             "Could not initialize offload capabilities\n");
 1912         }
 1913 
 1914         PORT_LOCK(p);
 1915         t3_port_intr_enable(sc, p->port_id);
 1916         if (!mac->multiport) 
 1917                 t3_mac_reset(mac);
 1918         cxgb_update_mac_settings(p);
 1919         t3_link_start(&p->phy, mac, &p->link_config);
 1920         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 1921         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 1922         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 1923         PORT_UNLOCK(p);
 1924 
 1925         t3_link_changed(sc, p->port_id);
 1926 
 1927         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
 1928                 struct sge_qset *qs = &sc->sge.qs[i];
 1929                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
 1930 
 1931                 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
 1932                                  txq->txq_watchdog.c_cpu);
 1933         }
 1934 
 1935         /* all ok */
 1936         setbit(&sc->open_device_map, p->port_id);
 1937 
 1938         return (0);
 1939 }
 1940 
 1941 /*
 1942  * Called on "ifconfig down", and from port_detach
 1943  */
 1944 static int
 1945 cxgb_uninit_synchronized(struct port_info *pi)
 1946 {
 1947         struct adapter *sc = pi->adapter;
 1948         struct ifnet *ifp = pi->ifp;
 1949 
 1950         /*
 1951          * Clear this port's bit from the open device map, and then drain all
 1952          * the tasks that can access/manipulate this port's port_info or ifp.
 1953          * We disable this port's interrupts here and so the the slow/ext
 1954          * interrupt tasks won't be enqueued.  The tick task will continue to
 1955          * be enqueued every second but the runs after this drain will not see
 1956          * this port in the open device map.
 1957          *
 1958          * A well behaved task must take open_device_map into account and ignore
 1959          * ports that are not open.
 1960          */
 1961         clrbit(&sc->open_device_map, pi->port_id);
 1962         t3_port_intr_disable(sc, pi->port_id);
 1963         taskqueue_drain(sc->tq, &sc->slow_intr_task);
 1964         taskqueue_drain(sc->tq, &sc->ext_intr_task);
 1965         taskqueue_drain(sc->tq, &sc->tick_task);
 1966 
 1967         PORT_LOCK(pi);
 1968         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 1969 
 1970         /* disable pause frames */
 1971         t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
 1972 
 1973         /* Reset RX FIFO HWM */
 1974         t3_set_reg_field(sc, A_XGM_RXFIFO_CFG +  pi->mac.offset,
 1975                          V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
 1976 
 1977         DELAY(100);
 1978 
 1979         /* Wait for TXFIFO empty */
 1980         t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
 1981                         F_TXFIFO_EMPTY, 1, 20, 5);
 1982 
 1983         DELAY(100);
 1984         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
 1985 
 1986 
 1987         pi->phy.ops->power_down(&pi->phy, 1);
 1988 
 1989         PORT_UNLOCK(pi);
 1990 
 1991         pi->link_config.link_ok = 0;
 1992         t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0);
 1993 
 1994         if ((sc->open_device_map & PORT_MASK) == 0)
 1995                 offload_close(&sc->tdev);
 1996 
 1997         if (sc->open_device_map == 0)
 1998                 cxgb_down(pi->adapter);
 1999 
 2000         return (0);
 2001 }
 2002 
 2003 #ifdef LRO_SUPPORTED
 2004 /*
 2005  * Mark lro enabled or disabled in all qsets for this port
 2006  */
 2007 static int
 2008 cxgb_set_lro(struct port_info *p, int enabled)
 2009 {
 2010         int i;
 2011         struct adapter *adp = p->adapter;
 2012         struct sge_qset *q;
 2013 
 2014         PORT_LOCK_ASSERT_OWNED(p);
 2015         for (i = 0; i < p->nqsets; i++) {
 2016                 q = &adp->sge.qs[p->first_qset + i];
 2017                 q->lro.enabled = (enabled != 0);
 2018         }
 2019         return (0);
 2020 }
 2021 #endif
 2022 
 2023 static int
 2024 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 2025 {
 2026         struct port_info *p = ifp->if_softc;
 2027         struct ifreq *ifr = (struct ifreq *)data;
 2028         int flags, error = 0, mtu, handle_unsynchronized = 0;
 2029         uint32_t mask;
 2030 
 2031         if ((error = cxgb_begin_op(p, "cxgbioct")) != 0)
 2032                 return (error);
 2033 
 2034         /*
 2035          * Only commands that should be handled within begin-op/end-op are
 2036          * serviced in this switch statement.  See handle_unsynchronized.
 2037          */
 2038         switch (command) {
 2039         case SIOCSIFMTU:
 2040                 mtu = ifr->ifr_mtu;
 2041                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
 2042                         error = EINVAL;
 2043                 } else {
 2044                         ifp->if_mtu = mtu;
 2045                         PORT_LOCK(p);
 2046                         cxgb_update_mac_settings(p);
 2047                         PORT_UNLOCK(p);
 2048                 }
 2049 
 2050                 break;
 2051         case SIOCSIFFLAGS:
 2052                 if (ifp->if_flags & IFF_UP) {
 2053                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2054                                 flags = p->if_flags;
 2055                                 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
 2056                                     ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
 2057                                         PORT_LOCK(p);
 2058                                         cxgb_update_mac_settings(p);
 2059                                         PORT_UNLOCK(p);
 2060                                 }
 2061                         } else
 2062                                 error = cxgb_init_synchronized(p);
 2063                         p->if_flags = ifp->if_flags;
 2064                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 2065                         error = cxgb_uninit_synchronized(p);
 2066                 
 2067                 break;
 2068         case SIOCADDMULTI:
 2069         case SIOCDELMULTI:
 2070                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2071                         PORT_LOCK(p);
 2072                         cxgb_update_mac_settings(p);
 2073                         PORT_UNLOCK(p);
 2074                 }
 2075 
 2076                 break;
 2077         case SIOCSIFCAP:
 2078                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 2079                 if (mask & IFCAP_TXCSUM) {
 2080                         if (IFCAP_TXCSUM & ifp->if_capenable) {
 2081                                 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
 2082                                 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
 2083                                     | CSUM_IP | CSUM_TSO);
 2084                         } else {
 2085                                 ifp->if_capenable |= IFCAP_TXCSUM;
 2086                                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP
 2087                                     | CSUM_IP);
 2088                         }
 2089                 }
 2090                 if (mask & IFCAP_RXCSUM) {
 2091                         ifp->if_capenable ^= IFCAP_RXCSUM;
 2092                 }
 2093                 if (mask & IFCAP_TSO4) {
 2094                         if (IFCAP_TSO4 & ifp->if_capenable) {
 2095                                 ifp->if_capenable &= ~IFCAP_TSO4;
 2096                                 ifp->if_hwassist &= ~CSUM_TSO;
 2097                         } else if (IFCAP_TXCSUM & ifp->if_capenable) {
 2098                                 ifp->if_capenable |= IFCAP_TSO4;
 2099                                 ifp->if_hwassist |= CSUM_TSO;
 2100                         } else
 2101                                 error = EINVAL;
 2102                 }
 2103 #ifdef LRO_SUPPORTED
 2104                 if (mask & IFCAP_LRO) {
 2105                         ifp->if_capenable ^= IFCAP_LRO;
 2106 
 2107                         /* Safe to do this even if cxgb_up not called yet */
 2108                         cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
 2109                 }
 2110 #endif
 2111                 if (mask & IFCAP_VLAN_HWTAGGING) {
 2112                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 2113                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2114                                 PORT_LOCK(p);
 2115                                 cxgb_update_mac_settings(p);
 2116                                 PORT_UNLOCK(p);
 2117                         }
 2118                 }
 2119                 if (mask & IFCAP_VLAN_MTU) {
 2120                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
 2121                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2122                                 PORT_LOCK(p);
 2123                                 cxgb_update_mac_settings(p);
 2124                                 PORT_UNLOCK(p);
 2125                         }
 2126                 }
 2127                 if (mask & IFCAP_VLAN_HWCSUM) {
 2128                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
 2129                 }
 2130 
 2131 #ifdef VLAN_CAPABILITIES
 2132                 VLAN_CAPABILITIES(ifp);
 2133 #endif
 2134                 break;
 2135         default:
 2136                 handle_unsynchronized = 1;
 2137                 break;
 2138         }
 2139 
 2140         /*
 2141          * We don't want to call anything outside the driver while inside a
 2142          * begin-op/end-op block.  If it calls us back (eg.  ether_ioctl may
 2143          * call cxgb_init) we may deadlock if the state is already marked busy.
 2144          *
 2145          * XXX: this probably opens a small race window with kldunload...
 2146          */
 2147         cxgb_end_op(p);
 2148 
 2149         /* The IS_DOOMED check is racy, we're clutching at straws here */
 2150         if (handle_unsynchronized && !IS_DOOMED(p)) {
 2151                 if (command == SIOCSIFMEDIA || command == SIOCGIFMEDIA)
 2152                         error = ifmedia_ioctl(ifp, ifr, &p->media, command);
 2153                 else
 2154                         error = ether_ioctl(ifp, command, data);
 2155         }
 2156 
 2157         return (error);
 2158 }
 2159 
 2160 static int
 2161 cxgb_media_change(struct ifnet *ifp)
 2162 {
 2163         return (EOPNOTSUPP);
 2164 }
 2165 
 2166 /*
 2167  * Translates phy->modtype to the correct Ethernet media subtype.
 2168  */
 2169 static int
 2170 cxgb_ifm_type(int mod)
 2171 {
 2172         switch (mod) {
 2173         case phy_modtype_sr:
 2174                 return (IFM_10G_SR);
 2175         case phy_modtype_lr:
 2176                 return (IFM_10G_LR);
 2177         case phy_modtype_lrm:
 2178                 return (IFM_10G_LRM);
 2179         case phy_modtype_twinax:
 2180                 return (IFM_10G_TWINAX);
 2181         case phy_modtype_twinax_long:
 2182                 return (IFM_10G_TWINAX_LONG);
 2183         case phy_modtype_none:
 2184                 return (IFM_NONE);
 2185         case phy_modtype_unknown:
 2186                 return (IFM_UNKNOWN);
 2187         }
 2188 
 2189         KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
 2190         return (IFM_UNKNOWN);
 2191 }
 2192 
 2193 /*
 2194  * Rebuilds the ifmedia list for this port, and sets the current media.
 2195  */
 2196 static void
 2197 cxgb_build_medialist(struct port_info *p)
 2198 {
 2199         struct cphy *phy = &p->phy;
 2200         struct ifmedia *media = &p->media;
 2201         int mod = phy->modtype;
 2202         int m = IFM_ETHER | IFM_FDX;
 2203 
 2204         PORT_LOCK(p);
 2205 
 2206         ifmedia_removeall(media);
 2207         if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
 2208                 /* Copper (RJ45) */
 2209 
 2210                 if (phy->caps & SUPPORTED_10000baseT_Full)
 2211                         ifmedia_add(media, m | IFM_10G_T, mod, NULL);
 2212 
 2213                 if (phy->caps & SUPPORTED_1000baseT_Full)
 2214                         ifmedia_add(media, m | IFM_1000_T, mod, NULL);
 2215 
 2216                 if (phy->caps & SUPPORTED_100baseT_Full)
 2217                         ifmedia_add(media, m | IFM_100_TX, mod, NULL);
 2218 
 2219                 if (phy->caps & SUPPORTED_10baseT_Full)
 2220                         ifmedia_add(media, m | IFM_10_T, mod, NULL);
 2221 
 2222                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
 2223                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
 2224 
 2225         } else if (phy->caps & SUPPORTED_TP) {
 2226                 /* Copper (CX4) */
 2227 
 2228                 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
 2229                         ("%s: unexpected cap 0x%x", __func__, phy->caps));
 2230 
 2231                 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
 2232                 ifmedia_set(media, m | IFM_10G_CX4);
 2233 
 2234         } else if (phy->caps & SUPPORTED_FIBRE &&
 2235                    phy->caps & SUPPORTED_10000baseT_Full) {
 2236                 /* 10G optical (but includes SFP+ twinax) */
 2237 
 2238                 m |= cxgb_ifm_type(mod);
 2239                 if (IFM_SUBTYPE(m) == IFM_NONE)
 2240                         m &= ~IFM_FDX;
 2241 
 2242                 ifmedia_add(media, m, mod, NULL);
 2243                 ifmedia_set(media, m);
 2244 
 2245         } else if (phy->caps & SUPPORTED_FIBRE &&
 2246                    phy->caps & SUPPORTED_1000baseT_Full) {
 2247                 /* 1G optical */
 2248 
 2249                 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
 2250                 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
 2251                 ifmedia_set(media, m | IFM_1000_SX);
 2252 
 2253         } else {
 2254                 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
 2255                             phy->caps));
 2256         }
 2257 
 2258         PORT_UNLOCK(p);
 2259 }
 2260 
 2261 static void
 2262 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 2263 {
 2264         struct port_info *p = ifp->if_softc;
 2265         struct ifmedia_entry *cur = p->media.ifm_cur;
 2266         int speed = p->link_config.speed;
 2267 
 2268         if (cur->ifm_data != p->phy.modtype) {
 2269                 cxgb_build_medialist(p);
 2270                 cur = p->media.ifm_cur;
 2271         }
 2272 
 2273         ifmr->ifm_status = IFM_AVALID;
 2274         if (!p->link_config.link_ok)
 2275                 return;
 2276 
 2277         ifmr->ifm_status |= IFM_ACTIVE;
 2278 
 2279         /*
 2280          * active and current will differ iff current media is autoselect.  That
 2281          * can happen only for copper RJ45.
 2282          */
 2283         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
 2284                 return;
 2285         KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
 2286                 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
 2287 
 2288         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
 2289         if (speed == SPEED_10000)
 2290                 ifmr->ifm_active |= IFM_10G_T;
 2291         else if (speed == SPEED_1000)
 2292                 ifmr->ifm_active |= IFM_1000_T;
 2293         else if (speed == SPEED_100)
 2294                 ifmr->ifm_active |= IFM_100_TX;
 2295         else if (speed == SPEED_10)
 2296                 ifmr->ifm_active |= IFM_10_T;
 2297         else
 2298                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
 2299                             speed));
 2300 }
 2301 
 2302 static void
 2303 cxgb_async_intr(void *data)
 2304 {
 2305         adapter_t *sc = data;
 2306 
 2307         if (cxgb_debug)
 2308                 device_printf(sc->dev, "cxgb_async_intr\n");
 2309         /*
 2310          * May need to sleep - defer to taskqueue
 2311          */
 2312         taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
 2313 }
 2314 
 2315 static void
 2316 cxgb_ext_intr_handler(void *arg, int count)
 2317 {
 2318         adapter_t *sc = (adapter_t *)arg;
 2319 
 2320         if (cxgb_debug)
 2321                 printf("cxgb_ext_intr_handler\n");
 2322 
 2323         t3_phy_intr_handler(sc);
 2324 
 2325         /* Now reenable external interrupts */
 2326         ADAPTER_LOCK(sc);
 2327         if (sc->slow_intr_mask) {
 2328                 sc->slow_intr_mask |= F_T3DBG;
 2329                 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
 2330                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
 2331         }
 2332         ADAPTER_UNLOCK(sc);
 2333 }
 2334 
 2335 static void
 2336 check_link_status(adapter_t *sc)
 2337 {
 2338         int i;
 2339 
 2340         for (i = 0; i < (sc)->params.nports; ++i) {
 2341                 struct port_info *p = &sc->port[i];
 2342 
 2343                 if (!isset(&sc->open_device_map, p->port_id))
 2344                         continue;
 2345 
 2346                 if (p->link_fault || !(p->phy.caps & SUPPORTED_IRQ))
 2347                         t3_link_changed(sc, i);
 2348         }
 2349 }
 2350 
 2351 static void
 2352 check_t3b2_mac(struct adapter *sc)
 2353 {
 2354         int i;
 2355 
 2356         if (sc->flags & CXGB_SHUTDOWN)
 2357                 return;
 2358 
 2359         for_each_port(sc, i) {
 2360                 struct port_info *p = &sc->port[i];
 2361                 int status;
 2362 #ifdef INVARIANTS
 2363                 struct ifnet *ifp = p->ifp;
 2364 #endif          
 2365 
 2366                 if (!isset(&sc->open_device_map, p->port_id))
 2367                         continue;
 2368 
 2369                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
 2370                         ("%s: state mismatch (drv_flags %x, device_map %x)",
 2371                          __func__, ifp->if_drv_flags, sc->open_device_map));
 2372 
 2373                 PORT_LOCK(p);
 2374                 status = t3b2_mac_watchdog_task(&p->mac);
 2375                 if (status == 1)
 2376                         p->mac.stats.num_toggled++;
 2377                 else if (status == 2) {
 2378                         struct cmac *mac = &p->mac;
 2379 
 2380                         cxgb_update_mac_settings(p);
 2381                         t3_link_start(&p->phy, mac, &p->link_config);
 2382                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 2383                         t3_port_intr_enable(sc, p->port_id);
 2384                         p->mac.stats.num_resets++;
 2385                 }
 2386                 PORT_UNLOCK(p);
 2387         }
 2388 }
 2389 
 2390 static void
 2391 cxgb_tick(void *arg)
 2392 {
 2393         adapter_t *sc = (adapter_t *)arg;
 2394 
 2395         if (sc->flags & CXGB_SHUTDOWN)
 2396                 return;
 2397 
 2398         taskqueue_enqueue(sc->tq, &sc->tick_task);      
 2399         callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
 2400 }
 2401 
 2402 static void
 2403 cxgb_tick_handler(void *arg, int count)
 2404 {
 2405         adapter_t *sc = (adapter_t *)arg;
 2406         const struct adapter_params *p = &sc->params;
 2407         int i;
 2408         uint32_t cause, reset;
 2409 
 2410         if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
 2411                 return;
 2412 
 2413         check_link_status(sc);
 2414         sc->check_task_cnt++;
 2415 
 2416         if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 
 2417                 check_t3b2_mac(sc);
 2418 
 2419         cause = t3_read_reg(sc, A_SG_INT_CAUSE);
 2420         reset = 0;
 2421         if (cause & F_FLEMPTY) {
 2422                 struct sge_qset *qs = &sc->sge.qs[0];
 2423 
 2424                 i = 0;
 2425                 reset |= F_FLEMPTY;
 2426 
 2427                 cause = (t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) >>
 2428                          S_FL0EMPTY) & 0xffff;
 2429                 while (cause) {
 2430                         qs->fl[i].empty += (cause & 1);
 2431                         if (i)
 2432                                 qs++;
 2433                         i ^= 1;
 2434                         cause >>= 1;
 2435                 }
 2436         }
 2437         t3_write_reg(sc, A_SG_INT_CAUSE, reset);
 2438 
 2439         for (i = 0; i < sc->params.nports; i++) {
 2440                 struct port_info *pi = &sc->port[i];
 2441                 struct ifnet *ifp = pi->ifp;
 2442                 struct cmac *mac = &pi->mac;
 2443                 struct mac_stats *mstats = &mac->stats;
 2444 
 2445                 if (!isset(&sc->open_device_map, pi->port_id))
 2446                         continue;
 2447 
 2448                 PORT_LOCK(pi);
 2449                 t3_mac_update_stats(mac);
 2450                 PORT_UNLOCK(pi);
 2451 
 2452                 ifp->if_opackets =
 2453                     mstats->tx_frames_64 +
 2454                     mstats->tx_frames_65_127 +
 2455                     mstats->tx_frames_128_255 +
 2456                     mstats->tx_frames_256_511 +
 2457                     mstats->tx_frames_512_1023 +
 2458                     mstats->tx_frames_1024_1518 +
 2459                     mstats->tx_frames_1519_max;
 2460                 
 2461                 ifp->if_ipackets =
 2462                     mstats->rx_frames_64 +
 2463                     mstats->rx_frames_65_127 +
 2464                     mstats->rx_frames_128_255 +
 2465                     mstats->rx_frames_256_511 +
 2466                     mstats->rx_frames_512_1023 +
 2467                     mstats->rx_frames_1024_1518 +
 2468                     mstats->rx_frames_1519_max;
 2469 
 2470                 ifp->if_obytes = mstats->tx_octets;
 2471                 ifp->if_ibytes = mstats->rx_octets;
 2472                 ifp->if_omcasts = mstats->tx_mcast_frames;
 2473                 ifp->if_imcasts = mstats->rx_mcast_frames;
 2474                 
 2475                 ifp->if_collisions =
 2476                     mstats->tx_total_collisions;
 2477 
 2478                 ifp->if_iqdrops = mstats->rx_cong_drops;
 2479                 
 2480                 ifp->if_oerrors =
 2481                     mstats->tx_excess_collisions +
 2482                     mstats->tx_underrun +
 2483                     mstats->tx_len_errs +
 2484                     mstats->tx_mac_internal_errs +
 2485                     mstats->tx_excess_deferral +
 2486                     mstats->tx_fcs_errs;
 2487                 ifp->if_ierrors =
 2488                     mstats->rx_jabber +
 2489                     mstats->rx_data_errs +
 2490                     mstats->rx_sequence_errs +
 2491                     mstats->rx_runt + 
 2492                     mstats->rx_too_long +
 2493                     mstats->rx_mac_internal_errs +
 2494                     mstats->rx_short +
 2495                     mstats->rx_fcs_errs;
 2496 
 2497                 if (mac->multiport)
 2498                         continue;
 2499 
 2500                 /* Count rx fifo overflows, once per second */
 2501                 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
 2502                 reset = 0;
 2503                 if (cause & F_RXFIFO_OVERFLOW) {
 2504                         mac->stats.rx_fifo_ovfl++;
 2505                         reset |= F_RXFIFO_OVERFLOW;
 2506                 }
 2507                 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
 2508         }
 2509 }
 2510 
 2511 static void
 2512 touch_bars(device_t dev)
 2513 {
 2514         /*
 2515          * Don't enable yet
 2516          */
 2517 #if !defined(__LP64__) && 0
 2518         u32 v;
 2519 
 2520         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
 2521         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
 2522         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
 2523         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
 2524         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
 2525         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
 2526 #endif
 2527 }
 2528 
 2529 static int
 2530 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
 2531 {
 2532         uint8_t *buf;
 2533         int err = 0;
 2534         u32 aligned_offset, aligned_len, *p;
 2535         struct adapter *adapter = pi->adapter;
 2536 
 2537 
 2538         aligned_offset = offset & ~3;
 2539         aligned_len = (len + (offset & 3) + 3) & ~3;
 2540 
 2541         if (aligned_offset != offset || aligned_len != len) {
 2542                 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);              
 2543                 if (!buf)
 2544                         return (ENOMEM);
 2545                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
 2546                 if (!err && aligned_len > 4)
 2547                         err = t3_seeprom_read(adapter,
 2548                                               aligned_offset + aligned_len - 4,
 2549                                               (u32 *)&buf[aligned_len - 4]);
 2550                 if (err)
 2551                         goto out;
 2552                 memcpy(buf + (offset & 3), data, len);
 2553         } else
 2554                 buf = (uint8_t *)(uintptr_t)data;
 2555 
 2556         err = t3_seeprom_wp(adapter, 0);
 2557         if (err)
 2558                 goto out;
 2559 
 2560         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
 2561                 err = t3_seeprom_write(adapter, aligned_offset, *p);
 2562                 aligned_offset += 4;
 2563         }
 2564 
 2565         if (!err)
 2566                 err = t3_seeprom_wp(adapter, 1);
 2567 out:
 2568         if (buf != data)
 2569                 free(buf, M_DEVBUF);
 2570         return err;
 2571 }
 2572 
 2573 
 2574 static int
 2575 in_range(int val, int lo, int hi)
 2576 {
 2577         return val < 0 || (val <= hi && val >= lo);
 2578 }
 2579 
 2580 static int
 2581 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
 2582 {
 2583        return (0);
 2584 }
 2585 
 2586 static int
 2587 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 2588 {
 2589        return (0);
 2590 }
 2591 
 2592 static int
 2593 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 2594     int fflag, struct thread *td)
 2595 {
 2596         int mmd, error = 0;
 2597         struct port_info *pi = dev->si_drv1;
 2598         adapter_t *sc = pi->adapter;
 2599 
 2600 #ifdef PRIV_SUPPORTED   
 2601         if (priv_check(td, PRIV_DRIVER)) {
 2602                 if (cxgb_debug) 
 2603                         printf("user does not have access to privileged ioctls\n");
 2604                 return (EPERM);
 2605         }
 2606 #else
 2607         if (suser(td)) {
 2608                 if (cxgb_debug)
 2609                         printf("user does not have access to privileged ioctls\n");
 2610                 return (EPERM);
 2611         }
 2612 #endif
 2613         
 2614         switch (cmd) {
 2615         case CHELSIO_GET_MIIREG: {
 2616                 uint32_t val;
 2617                 struct cphy *phy = &pi->phy;
 2618                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
 2619                 
 2620                 if (!phy->mdio_read)
 2621                         return (EOPNOTSUPP);
 2622                 if (is_10G(sc)) {
 2623                         mmd = mid->phy_id >> 8;
 2624                         if (!mmd)
 2625                                 mmd = MDIO_DEV_PCS;
 2626                         else if (mmd > MDIO_DEV_VEND2)
 2627                                 return (EINVAL);
 2628 
 2629                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
 2630                                              mid->reg_num, &val);
 2631                 } else
 2632                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
 2633                                              mid->reg_num & 0x1f, &val);
 2634                 if (error == 0)
 2635                         mid->val_out = val;
 2636                 break;
 2637         }
 2638         case CHELSIO_SET_MIIREG: {
 2639                 struct cphy *phy = &pi->phy;
 2640                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
 2641 
 2642                 if (!phy->mdio_write)
 2643                         return (EOPNOTSUPP);
 2644                 if (is_10G(sc)) {
 2645                         mmd = mid->phy_id >> 8;
 2646                         if (!mmd)
 2647                                 mmd = MDIO_DEV_PCS;
 2648                         else if (mmd > MDIO_DEV_VEND2)
 2649                                 return (EINVAL);
 2650                         
 2651                         error = phy->mdio_write(sc, mid->phy_id & 0x1f,
 2652                                               mmd, mid->reg_num, mid->val_in);
 2653                 } else
 2654                         error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
 2655                                               mid->reg_num & 0x1f,
 2656                                               mid->val_in);
 2657                 break;
 2658         }
 2659         case CHELSIO_SETREG: {
 2660                 struct ch_reg *edata = (struct ch_reg *)data;
 2661                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 2662                         return (EFAULT);
 2663                 t3_write_reg(sc, edata->addr, edata->val);
 2664                 break;
 2665         }
 2666         case CHELSIO_GETREG: {
 2667                 struct ch_reg *edata = (struct ch_reg *)data;
 2668                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 2669                         return (EFAULT);
 2670                 edata->val = t3_read_reg(sc, edata->addr);
 2671                 break;
 2672         }
 2673         case CHELSIO_GET_SGE_CONTEXT: {
 2674                 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
 2675                 mtx_lock_spin(&sc->sge.reg_lock);
 2676                 switch (ecntxt->cntxt_type) {
 2677                 case CNTXT_TYPE_EGRESS:
 2678                         error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
 2679                             ecntxt->data);
 2680                         break;
 2681                 case CNTXT_TYPE_FL:
 2682                         error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
 2683                             ecntxt->data);
 2684                         break;
 2685                 case CNTXT_TYPE_RSP:
 2686                         error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
 2687                             ecntxt->data);
 2688                         break;
 2689                 case CNTXT_TYPE_CQ:
 2690                         error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
 2691                             ecntxt->data);
 2692                         break;
 2693                 default:
 2694                         error = EINVAL;
 2695                         break;
 2696                 }
 2697                 mtx_unlock_spin(&sc->sge.reg_lock);
 2698                 break;
 2699         }
 2700         case CHELSIO_GET_SGE_DESC: {
 2701                 struct ch_desc *edesc = (struct ch_desc *)data;
 2702                 int ret;
 2703                 if (edesc->queue_num >= SGE_QSETS * 6)
 2704                         return (EINVAL);
 2705                 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
 2706                     edesc->queue_num % 6, edesc->idx, edesc->data);
 2707                 if (ret < 0)
 2708                         return (EINVAL);
 2709                 edesc->size = ret;
 2710                 break;
 2711         }
 2712         case CHELSIO_GET_QSET_PARAMS: {
 2713                 struct qset_params *q;
 2714                 struct ch_qset_params *t = (struct ch_qset_params *)data;
 2715                 int q1 = pi->first_qset;
 2716                 int nqsets = pi->nqsets;
 2717                 int i;
 2718 
 2719                 if (t->qset_idx >= nqsets)
 2720                         return EINVAL;
 2721 
 2722                 i = q1 + t->qset_idx;
 2723                 q = &sc->params.sge.qset[i];
 2724                 t->rspq_size   = q->rspq_size;
 2725                 t->txq_size[0] = q->txq_size[0];
 2726                 t->txq_size[1] = q->txq_size[1];
 2727                 t->txq_size[2] = q->txq_size[2];
 2728                 t->fl_size[0]  = q->fl_size;
 2729                 t->fl_size[1]  = q->jumbo_size;
 2730                 t->polling     = q->polling;
 2731                 t->lro         = q->lro;
 2732                 t->intr_lat    = q->coalesce_usecs;
 2733                 t->cong_thres  = q->cong_thres;
 2734                 t->qnum        = i;
 2735 
 2736                 if (sc->flags & USING_MSIX)
 2737                         t->vector = rman_get_start(sc->msix_irq_res[i]);
 2738                 else
 2739                         t->vector = rman_get_start(sc->irq_res);
 2740 
 2741                 break;
 2742         }
 2743         case CHELSIO_GET_QSET_NUM: {
 2744                 struct ch_reg *edata = (struct ch_reg *)data;
 2745                 edata->val = pi->nqsets;
 2746                 break;
 2747         }
 2748         case CHELSIO_LOAD_FW: {
 2749                 uint8_t *fw_data;
 2750                 uint32_t vers;
 2751                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2752 
 2753                 /*
 2754                  * You're allowed to load a firmware only before FULL_INIT_DONE
 2755                  *
 2756                  * FW_UPTODATE is also set so the rest of the initialization
 2757                  * will not overwrite what was loaded here.  This gives you the
 2758                  * flexibility to load any firmware (and maybe shoot yourself in
 2759                  * the foot).
 2760                  */
 2761 
 2762                 ADAPTER_LOCK(sc);
 2763                 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
 2764                         ADAPTER_UNLOCK(sc);
 2765                         return (EBUSY);
 2766                 }
 2767 
 2768                 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
 2769                 if (!fw_data)
 2770                         error = ENOMEM;
 2771                 else
 2772                         error = copyin(t->buf, fw_data, t->len);
 2773 
 2774                 if (!error)
 2775                         error = -t3_load_fw(sc, fw_data, t->len);
 2776 
 2777                 if (t3_get_fw_version(sc, &vers) == 0) {
 2778                         snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
 2779                             "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
 2780                             G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
 2781                 }
 2782 
 2783                 if (!error)
 2784                         sc->flags |= FW_UPTODATE;
 2785 
 2786                 free(fw_data, M_DEVBUF);
 2787                 ADAPTER_UNLOCK(sc);
 2788                 break;
 2789         }
 2790         case CHELSIO_LOAD_BOOT: {
 2791                 uint8_t *boot_data;
 2792                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2793 
 2794                 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
 2795                 if (!boot_data)
 2796                         return ENOMEM;
 2797 
 2798                 error = copyin(t->buf, boot_data, t->len);
 2799                 if (!error)
 2800                         error = -t3_load_boot(sc, boot_data, t->len);
 2801 
 2802                 free(boot_data, M_DEVBUF);
 2803                 break;
 2804         }
 2805         case CHELSIO_GET_PM: {
 2806                 struct ch_pm *m = (struct ch_pm *)data;
 2807                 struct tp_params *p = &sc->params.tp;
 2808 
 2809                 if (!is_offload(sc))
 2810                         return (EOPNOTSUPP);
 2811 
 2812                 m->tx_pg_sz = p->tx_pg_size;
 2813                 m->tx_num_pg = p->tx_num_pgs;
 2814                 m->rx_pg_sz  = p->rx_pg_size;
 2815                 m->rx_num_pg = p->rx_num_pgs;
 2816                 m->pm_total  = p->pmtx_size + p->chan_rx_size * p->nchan;
 2817 
 2818                 break;
 2819         }
 2820         case CHELSIO_SET_PM: {
 2821                 struct ch_pm *m = (struct ch_pm *)data;
 2822                 struct tp_params *p = &sc->params.tp;
 2823 
 2824                 if (!is_offload(sc))
 2825                         return (EOPNOTSUPP);
 2826                 if (sc->flags & FULL_INIT_DONE)
 2827                         return (EBUSY);
 2828 
 2829                 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
 2830                     !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
 2831                         return (EINVAL);        /* not power of 2 */
 2832                 if (!(m->rx_pg_sz & 0x14000))
 2833                         return (EINVAL);        /* not 16KB or 64KB */
 2834                 if (!(m->tx_pg_sz & 0x1554000))
 2835                         return (EINVAL);
 2836                 if (m->tx_num_pg == -1)
 2837                         m->tx_num_pg = p->tx_num_pgs;
 2838                 if (m->rx_num_pg == -1)
 2839                         m->rx_num_pg = p->rx_num_pgs;
 2840                 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
 2841                         return (EINVAL);
 2842                 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
 2843                     m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
 2844                         return (EINVAL);
 2845 
 2846                 p->rx_pg_size = m->rx_pg_sz;
 2847                 p->tx_pg_size = m->tx_pg_sz;
 2848                 p->rx_num_pgs = m->rx_num_pg;
 2849                 p->tx_num_pgs = m->tx_num_pg;
 2850                 break;
 2851         }
 2852         case CHELSIO_SETMTUTAB: {
 2853                 struct ch_mtus *m = (struct ch_mtus *)data;
 2854                 int i;
 2855                 
 2856                 if (!is_offload(sc))
 2857                         return (EOPNOTSUPP);
 2858                 if (offload_running(sc))
 2859                         return (EBUSY);
 2860                 if (m->nmtus != NMTUS)
 2861                         return (EINVAL);
 2862                 if (m->mtus[0] < 81)         /* accommodate SACK */
 2863                         return (EINVAL);
 2864                 
 2865                 /*
 2866                  * MTUs must be in ascending order
 2867                  */
 2868                 for (i = 1; i < NMTUS; ++i)
 2869                         if (m->mtus[i] < m->mtus[i - 1])
 2870                                 return (EINVAL);
 2871 
 2872                 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
 2873                 break;
 2874         }
 2875         case CHELSIO_GETMTUTAB: {
 2876                 struct ch_mtus *m = (struct ch_mtus *)data;
 2877 
 2878                 if (!is_offload(sc))
 2879                         return (EOPNOTSUPP);
 2880 
 2881                 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
 2882                 m->nmtus = NMTUS;
 2883                 break;
 2884         }
 2885         case CHELSIO_GET_MEM: {
 2886                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2887                 struct mc7 *mem;
 2888                 uint8_t *useraddr;
 2889                 u64 buf[32];
 2890 
 2891                 /*
 2892                  * Use these to avoid modifying len/addr in the the return
 2893                  * struct
 2894                  */
 2895                 uint32_t len = t->len, addr = t->addr;
 2896 
 2897                 if (!is_offload(sc))
 2898                         return (EOPNOTSUPP);
 2899                 if (!(sc->flags & FULL_INIT_DONE))
 2900                         return (EIO);         /* need the memory controllers */
 2901                 if ((addr & 0x7) || (len & 0x7))
 2902                         return (EINVAL);
 2903                 if (t->mem_id == MEM_CM)
 2904                         mem = &sc->cm;
 2905                 else if (t->mem_id == MEM_PMRX)
 2906                         mem = &sc->pmrx;
 2907                 else if (t->mem_id == MEM_PMTX)
 2908                         mem = &sc->pmtx;
 2909                 else
 2910                         return (EINVAL);
 2911 
 2912                 /*
 2913                  * Version scheme:
 2914                  * bits 0..9: chip version
 2915                  * bits 10..15: chip revision
 2916                  */
 2917                 t->version = 3 | (sc->params.rev << 10);
 2918                 
 2919                 /*
 2920                  * Read 256 bytes at a time as len can be large and we don't
 2921                  * want to use huge intermediate buffers.
 2922                  */
 2923                 useraddr = (uint8_t *)t->buf; 
 2924                 while (len) {
 2925                         unsigned int chunk = min(len, sizeof(buf));
 2926 
 2927                         error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
 2928                         if (error)
 2929                                 return (-error);
 2930                         if (copyout(buf, useraddr, chunk))
 2931                                 return (EFAULT);
 2932                         useraddr += chunk;
 2933                         addr += chunk;
 2934                         len -= chunk;
 2935                 }
 2936                 break;
 2937         }
 2938         case CHELSIO_READ_TCAM_WORD: {
 2939                 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
 2940 
 2941                 if (!is_offload(sc))
 2942                         return (EOPNOTSUPP);
 2943                 if (!(sc->flags & FULL_INIT_DONE))
 2944                         return (EIO);         /* need MC5 */            
 2945                 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
 2946                 break;
 2947         }
 2948         case CHELSIO_SET_TRACE_FILTER: {
 2949                 struct ch_trace *t = (struct ch_trace *)data;
 2950                 const struct trace_params *tp;
 2951 
 2952                 tp = (const struct trace_params *)&t->sip;
 2953                 if (t->config_tx)
 2954                         t3_config_trace_filter(sc, tp, 0, t->invert_match,
 2955                                                t->trace_tx);
 2956                 if (t->config_rx)
 2957                         t3_config_trace_filter(sc, tp, 1, t->invert_match,
 2958                                                t->trace_rx);
 2959                 break;
 2960         }
 2961         case CHELSIO_SET_PKTSCHED: {
 2962                 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
 2963                 if (sc->open_device_map == 0)
 2964                         return (EAGAIN);
 2965                 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
 2966                     p->binding);
 2967                 break;
 2968         }
 2969         case CHELSIO_IFCONF_GETREGS: {
 2970                 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
 2971                 int reglen = cxgb_get_regs_len();
 2972                 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
 2973                 if (buf == NULL) {
 2974                         return (ENOMEM);
 2975                 }
 2976                 if (regs->len > reglen)
 2977                         regs->len = reglen;
 2978                 else if (regs->len < reglen)
 2979                         error = ENOBUFS;
 2980 
 2981                 if (!error) {
 2982                         cxgb_get_regs(sc, regs, buf);
 2983                         error = copyout(buf, regs->data, reglen);
 2984                 }
 2985                 free(buf, M_DEVBUF);
 2986 
 2987                 break;
 2988         }
 2989         case CHELSIO_SET_HW_SCHED: {
 2990                 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
 2991                 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
 2992 
 2993                 if ((sc->flags & FULL_INIT_DONE) == 0)
 2994                         return (EAGAIN);       /* need TP to be initialized */
 2995                 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
 2996                     !in_range(t->channel, 0, 1) ||
 2997                     !in_range(t->kbps, 0, 10000000) ||
 2998                     !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
 2999                     !in_range(t->flow_ipg, 0,
 3000                               dack_ticks_to_usec(sc, 0x7ff)))
 3001                         return (EINVAL);
 3002 
 3003                 if (t->kbps >= 0) {
 3004                         error = t3_config_sched(sc, t->kbps, t->sched);
 3005                         if (error < 0)
 3006                                 return (-error);
 3007                 }
 3008                 if (t->class_ipg >= 0)
 3009                         t3_set_sched_ipg(sc, t->sched, t->class_ipg);
 3010                 if (t->flow_ipg >= 0) {
 3011                         t->flow_ipg *= 1000;     /* us -> ns */
 3012                         t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
 3013                 }
 3014                 if (t->mode >= 0) {
 3015                         int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
 3016 
 3017                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
 3018                                          bit, t->mode ? bit : 0);
 3019                 }
 3020                 if (t->channel >= 0)
 3021                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
 3022                                          1 << t->sched, t->channel << t->sched);
 3023                 break;
 3024         }
 3025         case CHELSIO_GET_EEPROM: {
 3026                 int i;
 3027                 struct ch_eeprom *e = (struct ch_eeprom *)data;
 3028                 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
 3029 
 3030                 if (buf == NULL) {
 3031                         return (ENOMEM);
 3032                 }
 3033                 e->magic = EEPROM_MAGIC;
 3034                 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
 3035                         error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
 3036 
 3037                 if (!error)
 3038                         error = copyout(buf + e->offset, e->data, e->len);
 3039 
 3040                 free(buf, M_DEVBUF);
 3041                 break;
 3042         }
 3043         case CHELSIO_CLEAR_STATS: {
 3044                 if (!(sc->flags & FULL_INIT_DONE))
 3045                         return EAGAIN;
 3046 
 3047                 PORT_LOCK(pi);
 3048                 t3_mac_update_stats(&pi->mac);
 3049                 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
 3050                 PORT_UNLOCK(pi);
 3051                 break;
 3052         }
 3053         case CHELSIO_GET_UP_LA: {
 3054                 struct ch_up_la *la = (struct ch_up_la *)data;
 3055                 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
 3056                 if (buf == NULL) {
 3057                         return (ENOMEM);
 3058                 }
 3059                 if (la->bufsize < LA_BUFSIZE)
 3060                         error = ENOBUFS;
 3061 
 3062                 if (!error)
 3063                         error = -t3_get_up_la(sc, &la->stopped, &la->idx,
 3064                                               &la->bufsize, buf);
 3065                 if (!error)
 3066                         error = copyout(buf, la->data, la->bufsize);
 3067 
 3068                 free(buf, M_DEVBUF);
 3069                 break;
 3070         }
 3071         case CHELSIO_GET_UP_IOQS: {
 3072                 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
 3073                 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
 3074                 uint32_t *v;
 3075 
 3076                 if (buf == NULL) {
 3077                         return (ENOMEM);
 3078                 }
 3079                 if (ioqs->bufsize < IOQS_BUFSIZE)
 3080                         error = ENOBUFS;
 3081 
 3082                 if (!error)
 3083                         error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
 3084 
 3085                 if (!error) {
 3086                         v = (uint32_t *)buf;
 3087 
 3088                         ioqs->bufsize -= 4 * sizeof(uint32_t);
 3089                         ioqs->ioq_rx_enable = *v++;
 3090                         ioqs->ioq_tx_enable = *v++;
 3091                         ioqs->ioq_rx_status = *v++;
 3092                         ioqs->ioq_tx_status = *v++;
 3093 
 3094                         error = copyout(v, ioqs->data, ioqs->bufsize);
 3095                 }
 3096 
 3097                 free(buf, M_DEVBUF);
 3098                 break;
 3099         }
 3100         default:
 3101                 return (EOPNOTSUPP);
 3102                 break;
 3103         }
 3104 
 3105         return (error);
 3106 }
 3107 
 3108 static __inline void
 3109 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
 3110     unsigned int end)
 3111 {
 3112         uint32_t *p = (uint32_t *)(buf + start);
 3113 
 3114         for ( ; start <= end; start += sizeof(uint32_t))
 3115                 *p++ = t3_read_reg(ap, start);
 3116 }
 3117 
 3118 #define T3_REGMAP_SIZE (3 * 1024)
 3119 static int
 3120 cxgb_get_regs_len(void)
 3121 {
 3122         return T3_REGMAP_SIZE;
 3123 }
 3124 
 3125 static void
 3126 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
 3127 {           
 3128         
 3129         /*
 3130          * Version scheme:
 3131          * bits 0..9: chip version
 3132          * bits 10..15: chip revision
 3133          * bit 31: set for PCIe cards
 3134          */
 3135         regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
 3136 
 3137         /*
 3138          * We skip the MAC statistics registers because they are clear-on-read.
 3139          * Also reading multi-register stats would need to synchronize with the
 3140          * periodic mac stats accumulation.  Hard to justify the complexity.
 3141          */
 3142         memset(buf, 0, cxgb_get_regs_len());
 3143         reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
 3144         reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
 3145         reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
 3146         reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
 3147         reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
 3148         reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
 3149                        XGM_REG(A_XGM_SERDES_STAT3, 1));
 3150         reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
 3151                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
 3152 }
 3153 
 3154 
 3155 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1);

Cache object: a3e9f43c949e5d13667a6e8e7a79fd41


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.