The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/cxgb/cxgb_main.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /**************************************************************************
    2 
    3 Copyright (c) 2007-2009, Chelsio Inc.
    4 All rights reserved.
    5 
    6 Redistribution and use in source and binary forms, with or without
    7 modification, are permitted provided that the following conditions are met:
    8 
    9  1. Redistributions of source code must retain the above copyright notice,
   10     this list of conditions and the following disclaimer.
   11 
   12  2. Neither the name of the Chelsio Corporation nor the names of its
   13     contributors may be used to endorse or promote products derived from
   14     this software without specific prior written permission.
   15 
   16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   26 POSSIBILITY OF SUCH DAMAGE.
   27 
   28 ***************************************************************************/
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/11.2/sys/dev/cxgb/cxgb_main.c 331722 2018-03-29 02:50:57Z eadler $");
   32 
   33 #include "opt_inet.h"
   34 
   35 #include <sys/param.h>
   36 #include <sys/systm.h>
   37 #include <sys/kernel.h>
   38 #include <sys/bus.h>
   39 #include <sys/module.h>
   40 #include <sys/pciio.h>
   41 #include <sys/conf.h>
   42 #include <machine/bus.h>
   43 #include <machine/resource.h>
   44 #include <sys/bus_dma.h>
   45 #include <sys/ktr.h>
   46 #include <sys/rman.h>
   47 #include <sys/ioccom.h>
   48 #include <sys/mbuf.h>
   49 #include <sys/linker.h>
   50 #include <sys/firmware.h>
   51 #include <sys/socket.h>
   52 #include <sys/sockio.h>
   53 #include <sys/smp.h>
   54 #include <sys/sysctl.h>
   55 #include <sys/syslog.h>
   56 #include <sys/queue.h>
   57 #include <sys/taskqueue.h>
   58 #include <sys/proc.h>
   59 
   60 #include <net/bpf.h>
   61 #include <net/ethernet.h>
   62 #include <net/if.h>
   63 #include <net/if_var.h>
   64 #include <net/if_arp.h>
   65 #include <net/if_dl.h>
   66 #include <net/if_media.h>
   67 #include <net/if_types.h>
   68 #include <net/if_vlan_var.h>
   69 
   70 #include <netinet/in_systm.h>
   71 #include <netinet/in.h>
   72 #include <netinet/if_ether.h>
   73 #include <netinet/ip.h>
   74 #include <netinet/ip.h>
   75 #include <netinet/tcp.h>
   76 #include <netinet/udp.h>
   77 
   78 #include <dev/pci/pcireg.h>
   79 #include <dev/pci/pcivar.h>
   80 #include <dev/pci/pci_private.h>
   81 
   82 #include <cxgb_include.h>
   83 
   84 #ifdef PRIV_SUPPORTED
   85 #include <sys/priv.h>
   86 #endif
   87 
   88 static int cxgb_setup_interrupts(adapter_t *);
   89 static void cxgb_teardown_interrupts(adapter_t *);
   90 static void cxgb_init(void *);
   91 static int cxgb_init_locked(struct port_info *);
   92 static int cxgb_uninit_locked(struct port_info *);
   93 static int cxgb_uninit_synchronized(struct port_info *);
   94 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
   95 static int cxgb_media_change(struct ifnet *);
   96 static int cxgb_ifm_type(int);
   97 static void cxgb_build_medialist(struct port_info *);
   98 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
   99 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
  100 static int setup_sge_qsets(adapter_t *);
  101 static void cxgb_async_intr(void *);
  102 static void cxgb_tick_handler(void *, int);
  103 static void cxgb_tick(void *);
  104 static void link_check_callout(void *);
  105 static void check_link_status(void *, int);
  106 static void setup_rss(adapter_t *sc);
  107 static int alloc_filters(struct adapter *);
  108 static int setup_hw_filters(struct adapter *);
  109 static int set_filter(struct adapter *, int, const struct filter_info *);
  110 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
  111     unsigned int, u64, u64);
  112 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
  113     unsigned int, u64, u64);
  114 #ifdef TCP_OFFLOAD
  115 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
  116 #endif
  117 
  118 /* Attachment glue for the PCI controller end of the device.  Each port of
  119  * the device is attached separately, as defined later.
  120  */
  121 static int cxgb_controller_probe(device_t);
  122 static int cxgb_controller_attach(device_t);
  123 static int cxgb_controller_detach(device_t);
  124 static void cxgb_free(struct adapter *);
  125 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
  126     unsigned int end);
  127 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
  128 static int cxgb_get_regs_len(void);
  129 static void touch_bars(device_t dev);
  130 static void cxgb_update_mac_settings(struct port_info *p);
  131 #ifdef TCP_OFFLOAD
  132 static int toe_capability(struct port_info *, int);
  133 #endif
  134 
  135 static device_method_t cxgb_controller_methods[] = {
  136         DEVMETHOD(device_probe,         cxgb_controller_probe),
  137         DEVMETHOD(device_attach,        cxgb_controller_attach),
  138         DEVMETHOD(device_detach,        cxgb_controller_detach),
  139 
  140         DEVMETHOD_END
  141 };
  142 
  143 static driver_t cxgb_controller_driver = {
  144         "cxgbc",
  145         cxgb_controller_methods,
  146         sizeof(struct adapter)
  147 };
  148 
  149 static int cxgbc_mod_event(module_t, int, void *);
  150 static devclass_t       cxgb_controller_devclass;
  151 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
  152     cxgbc_mod_event, 0);
  153 MODULE_VERSION(cxgbc, 1);
  154 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
  155 
  156 /*
  157  * Attachment glue for the ports.  Attachment is done directly to the
  158  * controller device.
  159  */
  160 static int cxgb_port_probe(device_t);
  161 static int cxgb_port_attach(device_t);
  162 static int cxgb_port_detach(device_t);
  163 
  164 static device_method_t cxgb_port_methods[] = {
  165         DEVMETHOD(device_probe,         cxgb_port_probe),
  166         DEVMETHOD(device_attach,        cxgb_port_attach),
  167         DEVMETHOD(device_detach,        cxgb_port_detach),
  168         { 0, 0 }
  169 };
  170 
  171 static driver_t cxgb_port_driver = {
  172         "cxgb",
  173         cxgb_port_methods,
  174         0
  175 };
  176 
  177 static d_ioctl_t cxgb_extension_ioctl;
  178 static d_open_t cxgb_extension_open;
  179 static d_close_t cxgb_extension_close;
  180 
  181 static struct cdevsw cxgb_cdevsw = {
  182        .d_version =    D_VERSION,
  183        .d_flags =      0,
  184        .d_open =       cxgb_extension_open,
  185        .d_close =      cxgb_extension_close,
  186        .d_ioctl =      cxgb_extension_ioctl,
  187        .d_name =       "cxgb",
  188 };
  189 
  190 static devclass_t       cxgb_port_devclass;
  191 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
  192 MODULE_VERSION(cxgb, 1);
  193 
  194 static struct mtx t3_list_lock;
  195 static SLIST_HEAD(, adapter) t3_list;
  196 #ifdef TCP_OFFLOAD
  197 static struct mtx t3_uld_list_lock;
  198 static SLIST_HEAD(, uld_info) t3_uld_list;
  199 #endif
  200 
  201 /*
  202  * The driver uses the best interrupt scheme available on a platform in the
  203  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
  204  * of these schemes the driver may consider as follows:
  205  *
  206  * msi = 2: choose from among all three options
  207  * msi = 1 : only consider MSI and pin interrupts
  208  * msi = 0: force pin interrupts
  209  */
  210 static int msi_allowed = 2;
  211 
  212 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
  213 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
  214     "MSI-X, MSI, INTx selector");
  215 
  216 /*
  217  * The driver uses an auto-queue algorithm by default.
  218  * To disable it and force a single queue-set per port, use multiq = 0
  219  */
  220 static int multiq = 1;
  221 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
  222     "use min(ncpus/ports, 8) queue-sets per port");
  223 
  224 /*
  225  * By default the driver will not update the firmware unless
  226  * it was compiled against a newer version
  227  * 
  228  */
  229 static int force_fw_update = 0;
  230 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
  231     "update firmware even if up to date");
  232 
  233 int cxgb_use_16k_clusters = -1;
  234 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
  235     &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
  236 
  237 static int nfilters = -1;
  238 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
  239     &nfilters, 0, "max number of entries in the filter table");
  240 
  241 enum {
  242         MAX_TXQ_ENTRIES      = 16384,
  243         MAX_CTRL_TXQ_ENTRIES = 1024,
  244         MAX_RSPQ_ENTRIES     = 16384,
  245         MAX_RX_BUFFERS       = 16384,
  246         MAX_RX_JUMBO_BUFFERS = 16384,
  247         MIN_TXQ_ENTRIES      = 4,
  248         MIN_CTRL_TXQ_ENTRIES = 4,
  249         MIN_RSPQ_ENTRIES     = 32,
  250         MIN_FL_ENTRIES       = 32,
  251         MIN_FL_JUMBO_ENTRIES = 32
  252 };
  253 
  254 struct filter_info {
  255         u32 sip;
  256         u32 sip_mask;
  257         u32 dip;
  258         u16 sport;
  259         u16 dport;
  260         u32 vlan:12;
  261         u32 vlan_prio:3;
  262         u32 mac_hit:1;
  263         u32 mac_idx:4;
  264         u32 mac_vld:1;
  265         u32 pkt_type:2;
  266         u32 report_filter_id:1;
  267         u32 pass:1;
  268         u32 rss:1;
  269         u32 qset:3;
  270         u32 locked:1;
  271         u32 valid:1;
  272 };
  273 
  274 enum { FILTER_NO_VLAN_PRI = 7 };
  275 
  276 #define EEPROM_MAGIC 0x38E2F10C
  277 
  278 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  279 
  280 /* Table for probing the cards.  The desc field isn't actually used */
  281 struct cxgb_ident {
  282         uint16_t        vendor;
  283         uint16_t        device;
  284         int             index;
  285         char            *desc;
  286 } cxgb_identifiers[] = {
  287         {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
  288         {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
  289         {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
  290         {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
  291         {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
  292         {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
  293         {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
  294         {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
  295         {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
  296         {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
  297         {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
  298         {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
  299         {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
  300         {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
  301         {0, 0, 0, NULL}
  302 };
  303 
  304 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
  305 
  306 
  307 static __inline char
  308 t3rev2char(struct adapter *adapter)
  309 {
  310         char rev = 'z';
  311 
  312         switch(adapter->params.rev) {
  313         case T3_REV_A:
  314                 rev = 'a';
  315                 break;
  316         case T3_REV_B:
  317         case T3_REV_B2:
  318                 rev = 'b';
  319                 break;
  320         case T3_REV_C:
  321                 rev = 'c';
  322                 break;
  323         }
  324         return rev;
  325 }
  326 
  327 static struct cxgb_ident *
  328 cxgb_get_ident(device_t dev)
  329 {
  330         struct cxgb_ident *id;
  331 
  332         for (id = cxgb_identifiers; id->desc != NULL; id++) {
  333                 if ((id->vendor == pci_get_vendor(dev)) &&
  334                     (id->device == pci_get_device(dev))) {
  335                         return (id);
  336                 }
  337         }
  338         return (NULL);
  339 }
  340 
  341 static const struct adapter_info *
  342 cxgb_get_adapter_info(device_t dev)
  343 {
  344         struct cxgb_ident *id;
  345         const struct adapter_info *ai;
  346 
  347         id = cxgb_get_ident(dev);
  348         if (id == NULL)
  349                 return (NULL);
  350 
  351         ai = t3_get_adapter_info(id->index);
  352 
  353         return (ai);
  354 }
  355 
  356 static int
  357 cxgb_controller_probe(device_t dev)
  358 {
  359         const struct adapter_info *ai;
  360         char *ports, buf[80];
  361         int nports;
  362 
  363         ai = cxgb_get_adapter_info(dev);
  364         if (ai == NULL)
  365                 return (ENXIO);
  366 
  367         nports = ai->nports0 + ai->nports1;
  368         if (nports == 1)
  369                 ports = "port";
  370         else
  371                 ports = "ports";
  372 
  373         snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
  374         device_set_desc_copy(dev, buf);
  375         return (BUS_PROBE_DEFAULT);
  376 }
  377 
  378 #define FW_FNAME "cxgb_t3fw"
  379 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
  380 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
  381 
  382 static int
  383 upgrade_fw(adapter_t *sc)
  384 {
  385         const struct firmware *fw;
  386         int status;
  387         u32 vers;
  388         
  389         if ((fw = firmware_get(FW_FNAME)) == NULL)  {
  390                 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
  391                 return (ENOENT);
  392         } else
  393                 device_printf(sc->dev, "installing firmware on card\n");
  394         status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
  395 
  396         if (status != 0) {
  397                 device_printf(sc->dev, "failed to install firmware: %d\n",
  398                     status);
  399         } else {
  400                 t3_get_fw_version(sc, &vers);
  401                 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
  402                     G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
  403                     G_FW_VERSION_MICRO(vers));
  404         }
  405 
  406         firmware_put(fw, FIRMWARE_UNLOAD);
  407 
  408         return (status);        
  409 }
  410 
  411 /*
  412  * The cxgb_controller_attach function is responsible for the initial
  413  * bringup of the device.  Its responsibilities include:
  414  *
  415  *  1. Determine if the device supports MSI or MSI-X.
  416  *  2. Allocate bus resources so that we can access the Base Address Register
  417  *  3. Create and initialize mutexes for the controller and its control
  418  *     logic such as SGE and MDIO.
  419  *  4. Call hardware specific setup routine for the adapter as a whole.
  420  *  5. Allocate the BAR for doing MSI-X.
  421  *  6. Setup the line interrupt iff MSI-X is not supported.
  422  *  7. Create the driver's taskq.
  423  *  8. Start one task queue service thread.
  424  *  9. Check if the firmware and SRAM are up-to-date.  They will be
  425  *     auto-updated later (before FULL_INIT_DONE), if required.
  426  * 10. Create a child device for each MAC (port)
  427  * 11. Initialize T3 private state.
  428  * 12. Trigger the LED
  429  * 13. Setup offload iff supported.
  430  * 14. Reset/restart the tick callout.
  431  * 15. Attach sysctls
  432  *
  433  * NOTE: Any modification or deviation from this list MUST be reflected in
  434  * the above comment.  Failure to do so will result in problems on various
  435  * error conditions including link flapping.
  436  */
  437 static int
  438 cxgb_controller_attach(device_t dev)
  439 {
  440         device_t child;
  441         const struct adapter_info *ai;
  442         struct adapter *sc;
  443         int i, error = 0;
  444         uint32_t vers;
  445         int port_qsets = 1;
  446         int msi_needed, reg;
  447         char buf[80];
  448 
  449         sc = device_get_softc(dev);
  450         sc->dev = dev;
  451         sc->msi_count = 0;
  452         ai = cxgb_get_adapter_info(dev);
  453 
  454         snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
  455             device_get_unit(dev));
  456         ADAPTER_LOCK_INIT(sc, sc->lockbuf);
  457 
  458         snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
  459             device_get_unit(dev));
  460         snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
  461             device_get_unit(dev));
  462         snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
  463             device_get_unit(dev));
  464         
  465         MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
  466         MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
  467         MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
  468 
  469         mtx_lock(&t3_list_lock);
  470         SLIST_INSERT_HEAD(&t3_list, sc, link);
  471         mtx_unlock(&t3_list_lock);
  472 
  473         /* find the PCIe link width and set max read request to 4KB*/
  474         if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
  475                 uint16_t lnk;
  476 
  477                 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
  478                 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
  479                 if (sc->link_width < 8 &&
  480                     (ai->caps & SUPPORTED_10000baseT_Full)) {
  481                         device_printf(sc->dev,
  482                             "PCIe x%d Link, expect reduced performance\n",
  483                             sc->link_width);
  484                 }
  485 
  486                 pci_set_max_read_req(dev, 4096);
  487         }
  488 
  489         touch_bars(dev);
  490         pci_enable_busmaster(dev);
  491         /*
  492          * Allocate the registers and make them available to the driver.
  493          * The registers that we care about for NIC mode are in BAR 0
  494          */
  495         sc->regs_rid = PCIR_BAR(0);
  496         if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  497             &sc->regs_rid, RF_ACTIVE)) == NULL) {
  498                 device_printf(dev, "Cannot allocate BAR region 0\n");
  499                 error = ENXIO;
  500                 goto out;
  501         }
  502 
  503         sc->bt = rman_get_bustag(sc->regs_res);
  504         sc->bh = rman_get_bushandle(sc->regs_res);
  505         sc->mmio_len = rman_get_size(sc->regs_res);
  506 
  507         for (i = 0; i < MAX_NPORTS; i++)
  508                 sc->port[i].adapter = sc;
  509 
  510         if (t3_prep_adapter(sc, ai, 1) < 0) {
  511                 printf("prep adapter failed\n");
  512                 error = ENODEV;
  513                 goto out;
  514         }
  515 
  516         sc->udbs_rid = PCIR_BAR(2);
  517         sc->udbs_res = NULL;
  518         if (is_offload(sc) &&
  519             ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  520                    &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
  521                 device_printf(dev, "Cannot allocate BAR region 1\n");
  522                 error = ENXIO;
  523                 goto out;
  524         }
  525 
  526         /* Allocate the BAR for doing MSI-X.  If it succeeds, try to allocate
  527          * enough messages for the queue sets.  If that fails, try falling
  528          * back to MSI.  If that fails, then try falling back to the legacy
  529          * interrupt pin model.
  530          */
  531         sc->msix_regs_rid = 0x20;
  532         if ((msi_allowed >= 2) &&
  533             (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  534             &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
  535 
  536                 if (multiq)
  537                         port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
  538                 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
  539 
  540                 if (pci_msix_count(dev) == 0 ||
  541                     (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
  542                     sc->msi_count != msi_needed) {
  543                         device_printf(dev, "alloc msix failed - "
  544                                       "msi_count=%d, msi_needed=%d, err=%d; "
  545                                       "will try MSI\n", sc->msi_count,
  546                                       msi_needed, error);
  547                         sc->msi_count = 0;
  548                         port_qsets = 1;
  549                         pci_release_msi(dev);
  550                         bus_release_resource(dev, SYS_RES_MEMORY,
  551                             sc->msix_regs_rid, sc->msix_regs_res);
  552                         sc->msix_regs_res = NULL;
  553                 } else {
  554                         sc->flags |= USING_MSIX;
  555                         sc->cxgb_intr = cxgb_async_intr;
  556                         device_printf(dev,
  557                                       "using MSI-X interrupts (%u vectors)\n",
  558                                       sc->msi_count);
  559                 }
  560         }
  561 
  562         if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
  563                 sc->msi_count = 1;
  564                 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
  565                         device_printf(dev, "alloc msi failed - "
  566                                       "err=%d; will try INTx\n", error);
  567                         sc->msi_count = 0;
  568                         port_qsets = 1;
  569                         pci_release_msi(dev);
  570                 } else {
  571                         sc->flags |= USING_MSI;
  572                         sc->cxgb_intr = t3_intr_msi;
  573                         device_printf(dev, "using MSI interrupts\n");
  574                 }
  575         }
  576         if (sc->msi_count == 0) {
  577                 device_printf(dev, "using line interrupts\n");
  578                 sc->cxgb_intr = t3b_intr;
  579         }
  580 
  581         /* Create a private taskqueue thread for handling driver events */
  582         sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
  583             taskqueue_thread_enqueue, &sc->tq);
  584         if (sc->tq == NULL) {
  585                 device_printf(dev, "failed to allocate controller task queue\n");
  586                 goto out;
  587         }
  588 
  589         taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
  590             device_get_nameunit(dev));
  591         TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
  592 
  593         
  594         /* Create a periodic callout for checking adapter status */
  595         callout_init(&sc->cxgb_tick_ch, 1);
  596         
  597         if (t3_check_fw_version(sc) < 0 || force_fw_update) {
  598                 /*
  599                  * Warn user that a firmware update will be attempted in init.
  600                  */
  601                 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
  602                     FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
  603                 sc->flags &= ~FW_UPTODATE;
  604         } else {
  605                 sc->flags |= FW_UPTODATE;
  606         }
  607 
  608         if (t3_check_tpsram_version(sc) < 0) {
  609                 /*
  610                  * Warn user that a firmware update will be attempted in init.
  611                  */
  612                 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
  613                     t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
  614                 sc->flags &= ~TPS_UPTODATE;
  615         } else {
  616                 sc->flags |= TPS_UPTODATE;
  617         }
  618 
  619         /*
  620          * Create a child device for each MAC.  The ethernet attachment
  621          * will be done in these children.
  622          */     
  623         for (i = 0; i < (sc)->params.nports; i++) {
  624                 struct port_info *pi;
  625                 
  626                 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
  627                         device_printf(dev, "failed to add child port\n");
  628                         error = EINVAL;
  629                         goto out;
  630                 }
  631                 pi = &sc->port[i];
  632                 pi->adapter = sc;
  633                 pi->nqsets = port_qsets;
  634                 pi->first_qset = i*port_qsets;
  635                 pi->port_id = i;
  636                 pi->tx_chan = i >= ai->nports0;
  637                 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
  638                 sc->rxpkt_map[pi->txpkt_intf] = i;
  639                 sc->port[i].tx_chan = i >= ai->nports0;
  640                 sc->portdev[i] = child;
  641                 device_set_softc(child, pi);
  642         }
  643         if ((error = bus_generic_attach(dev)) != 0)
  644                 goto out;
  645 
  646         /* initialize sge private state */
  647         t3_sge_init_adapter(sc);
  648 
  649         t3_led_ready(sc);
  650 
  651         error = t3_get_fw_version(sc, &vers);
  652         if (error)
  653                 goto out;
  654 
  655         snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
  656             G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
  657             G_FW_VERSION_MICRO(vers));
  658 
  659         snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
  660                  ai->desc, is_offload(sc) ? "R" : "",
  661                  sc->params.vpd.ec, sc->params.vpd.sn);
  662         device_set_desc_copy(dev, buf);
  663 
  664         snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
  665                  sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
  666                  sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
  667 
  668         device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
  669         callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
  670         t3_add_attach_sysctls(sc);
  671 
  672 #ifdef TCP_OFFLOAD
  673         for (i = 0; i < NUM_CPL_HANDLERS; i++)
  674                 sc->cpl_handler[i] = cpl_not_handled;
  675 #endif
  676 
  677         t3_intr_clear(sc);
  678         error = cxgb_setup_interrupts(sc);
  679 out:
  680         if (error)
  681                 cxgb_free(sc);
  682 
  683         return (error);
  684 }
  685 
  686 /*
  687  * The cxgb_controller_detach routine is called with the device is
  688  * unloaded from the system.
  689  */
  690 
  691 static int
  692 cxgb_controller_detach(device_t dev)
  693 {
  694         struct adapter *sc;
  695 
  696         sc = device_get_softc(dev);
  697 
  698         cxgb_free(sc);
  699 
  700         return (0);
  701 }
  702 
  703 /*
  704  * The cxgb_free() is called by the cxgb_controller_detach() routine
  705  * to tear down the structures that were built up in
  706  * cxgb_controller_attach(), and should be the final piece of work
  707  * done when fully unloading the driver.
  708  * 
  709  *
  710  *  1. Shutting down the threads started by the cxgb_controller_attach()
  711  *     routine.
  712  *  2. Stopping the lower level device and all callouts (cxgb_down_locked()).
  713  *  3. Detaching all of the port devices created during the
  714  *     cxgb_controller_attach() routine.
  715  *  4. Removing the device children created via cxgb_controller_attach().
  716  *  5. Releasing PCI resources associated with the device.
  717  *  6. Turning off the offload support, iff it was turned on.
  718  *  7. Destroying the mutexes created in cxgb_controller_attach().
  719  *
  720  */
  721 static void
  722 cxgb_free(struct adapter *sc)
  723 {
  724         int i, nqsets = 0;
  725 
  726         ADAPTER_LOCK(sc);
  727         sc->flags |= CXGB_SHUTDOWN;
  728         ADAPTER_UNLOCK(sc);
  729 
  730         /*
  731          * Make sure all child devices are gone.
  732          */
  733         bus_generic_detach(sc->dev);
  734         for (i = 0; i < (sc)->params.nports; i++) {
  735                 if (sc->portdev[i] &&
  736                     device_delete_child(sc->dev, sc->portdev[i]) != 0)
  737                         device_printf(sc->dev, "failed to delete child port\n");
  738                 nqsets += sc->port[i].nqsets;
  739         }
  740 
  741         /*
  742          * At this point, it is as if cxgb_port_detach has run on all ports, and
  743          * cxgb_down has run on the adapter.  All interrupts have been silenced,
  744          * all open devices have been closed.
  745          */
  746         KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
  747                                            __func__, sc->open_device_map));
  748         for (i = 0; i < sc->params.nports; i++) {
  749                 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
  750                                                   __func__, i));
  751         }
  752 
  753         /*
  754          * Finish off the adapter's callouts.
  755          */
  756         callout_drain(&sc->cxgb_tick_ch);
  757         callout_drain(&sc->sge_timer_ch);
  758 
  759         /*
  760          * Release resources grabbed under FULL_INIT_DONE by cxgb_up.  The
  761          * sysctls are cleaned up by the kernel linker.
  762          */
  763         if (sc->flags & FULL_INIT_DONE) {
  764                 t3_free_sge_resources(sc, nqsets);
  765                 sc->flags &= ~FULL_INIT_DONE;
  766         }
  767 
  768         /*
  769          * Release all interrupt resources.
  770          */
  771         cxgb_teardown_interrupts(sc);
  772         if (sc->flags & (USING_MSI | USING_MSIX)) {
  773                 device_printf(sc->dev, "releasing msi message(s)\n");
  774                 pci_release_msi(sc->dev);
  775         } else {
  776                 device_printf(sc->dev, "no msi message to release\n");
  777         }
  778 
  779         if (sc->msix_regs_res != NULL) {
  780                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
  781                     sc->msix_regs_res);
  782         }
  783 
  784         /*
  785          * Free the adapter's taskqueue.
  786          */
  787         if (sc->tq != NULL) {
  788                 taskqueue_free(sc->tq);
  789                 sc->tq = NULL;
  790         }
  791         
  792         free(sc->filters, M_DEVBUF);
  793         t3_sge_free(sc);
  794 
  795         if (sc->udbs_res != NULL)
  796                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
  797                     sc->udbs_res);
  798 
  799         if (sc->regs_res != NULL)
  800                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
  801                     sc->regs_res);
  802 
  803         MTX_DESTROY(&sc->mdio_lock);
  804         MTX_DESTROY(&sc->sge.reg_lock);
  805         MTX_DESTROY(&sc->elmer_lock);
  806         mtx_lock(&t3_list_lock);
  807         SLIST_REMOVE(&t3_list, sc, adapter, link);
  808         mtx_unlock(&t3_list_lock);
  809         ADAPTER_LOCK_DEINIT(sc);
  810 }
  811 
  812 /**
  813  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
  814  *      @sc: the controller softc
  815  *
  816  *      Determines how many sets of SGE queues to use and initializes them.
  817  *      We support multiple queue sets per port if we have MSI-X, otherwise
  818  *      just one queue set per port.
  819  */
  820 static int
  821 setup_sge_qsets(adapter_t *sc)
  822 {
  823         int i, j, err, irq_idx = 0, qset_idx = 0;
  824         u_int ntxq = SGE_TXQ_PER_SET;
  825 
  826         if ((err = t3_sge_alloc(sc)) != 0) {
  827                 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
  828                 return (err);
  829         }
  830 
  831         if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
  832                 irq_idx = -1;
  833 
  834         for (i = 0; i < (sc)->params.nports; i++) {
  835                 struct port_info *pi = &sc->port[i];
  836 
  837                 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
  838                         err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
  839                             (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
  840                             &sc->params.sge.qset[qset_idx], ntxq, pi);
  841                         if (err) {
  842                                 t3_free_sge_resources(sc, qset_idx);
  843                                 device_printf(sc->dev,
  844                                     "t3_sge_alloc_qset failed with %d\n", err);
  845                                 return (err);
  846                         }
  847                 }
  848         }
  849 
  850         return (0);
  851 }
  852 
  853 static void
  854 cxgb_teardown_interrupts(adapter_t *sc)
  855 {
  856         int i;
  857 
  858         for (i = 0; i < SGE_QSETS; i++) {
  859                 if (sc->msix_intr_tag[i] == NULL) {
  860 
  861                         /* Should have been setup fully or not at all */
  862                         KASSERT(sc->msix_irq_res[i] == NULL &&
  863                                 sc->msix_irq_rid[i] == 0,
  864                                 ("%s: half-done interrupt (%d).", __func__, i));
  865 
  866                         continue;
  867                 }
  868 
  869                 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
  870                                   sc->msix_intr_tag[i]);
  871                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
  872                                      sc->msix_irq_res[i]);
  873 
  874                 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
  875                 sc->msix_irq_rid[i] = 0;
  876         }
  877 
  878         if (sc->intr_tag) {
  879                 KASSERT(sc->irq_res != NULL,
  880                         ("%s: half-done interrupt.", __func__));
  881 
  882                 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
  883                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  884                                      sc->irq_res);
  885 
  886                 sc->irq_res = sc->intr_tag = NULL;
  887                 sc->irq_rid = 0;
  888         }
  889 }
  890 
  891 static int
  892 cxgb_setup_interrupts(adapter_t *sc)
  893 {
  894         struct resource *res;
  895         void *tag;
  896         int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
  897 
  898         sc->irq_rid = intr_flag ? 1 : 0;
  899         sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
  900                                              RF_SHAREABLE | RF_ACTIVE);
  901         if (sc->irq_res == NULL) {
  902                 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
  903                               intr_flag, sc->irq_rid);
  904                 err = EINVAL;
  905                 sc->irq_rid = 0;
  906         } else {
  907                 err = bus_setup_intr(sc->dev, sc->irq_res,
  908                     INTR_MPSAFE | INTR_TYPE_NET, NULL,
  909                     sc->cxgb_intr, sc, &sc->intr_tag);
  910 
  911                 if (err) {
  912                         device_printf(sc->dev,
  913                                       "Cannot set up interrupt (%x, %u, %d)\n",
  914                                       intr_flag, sc->irq_rid, err);
  915                         bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  916                                              sc->irq_res);
  917                         sc->irq_res = sc->intr_tag = NULL;
  918                         sc->irq_rid = 0;
  919                 }
  920         }
  921 
  922         /* That's all for INTx or MSI */
  923         if (!(intr_flag & USING_MSIX) || err)
  924                 return (err);
  925 
  926         bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
  927         for (i = 0; i < sc->msi_count - 1; i++) {
  928                 rid = i + 2;
  929                 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
  930                                              RF_SHAREABLE | RF_ACTIVE);
  931                 if (res == NULL) {
  932                         device_printf(sc->dev, "Cannot allocate interrupt "
  933                                       "for message %d\n", rid);
  934                         err = EINVAL;
  935                         break;
  936                 }
  937 
  938                 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
  939                                      NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
  940                 if (err) {
  941                         device_printf(sc->dev, "Cannot set up interrupt "
  942                                       "for message %d (%d)\n", rid, err);
  943                         bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
  944                         break;
  945                 }
  946 
  947                 sc->msix_irq_rid[i] = rid;
  948                 sc->msix_irq_res[i] = res;
  949                 sc->msix_intr_tag[i] = tag;
  950                 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
  951         }
  952 
  953         if (err)
  954                 cxgb_teardown_interrupts(sc);
  955 
  956         return (err);
  957 }
  958 
  959 
  960 static int
  961 cxgb_port_probe(device_t dev)
  962 {
  963         struct port_info *p;
  964         char buf[80];
  965         const char *desc;
  966         
  967         p = device_get_softc(dev);
  968         desc = p->phy.desc;
  969         snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
  970         device_set_desc_copy(dev, buf);
  971         return (0);
  972 }
  973 
  974 
  975 static int
  976 cxgb_makedev(struct port_info *pi)
  977 {
  978         
  979         pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
  980             UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
  981         
  982         if (pi->port_cdev == NULL)
  983                 return (ENOMEM);
  984 
  985         pi->port_cdev->si_drv1 = (void *)pi;
  986         
  987         return (0);
  988 }
  989 
  990 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
  991     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
  992     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
  993 #define CXGB_CAP_ENABLE CXGB_CAP
  994 
  995 static int
  996 cxgb_port_attach(device_t dev)
  997 {
  998         struct port_info *p;
  999         struct ifnet *ifp;
 1000         int err;
 1001         struct adapter *sc;
 1002 
 1003         p = device_get_softc(dev);
 1004         sc = p->adapter;
 1005         snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
 1006             device_get_unit(device_get_parent(dev)), p->port_id);
 1007         PORT_LOCK_INIT(p, p->lockbuf);
 1008 
 1009         callout_init(&p->link_check_ch, 1);
 1010         TASK_INIT(&p->link_check_task, 0, check_link_status, p);
 1011 
 1012         /* Allocate an ifnet object and set it up */
 1013         ifp = p->ifp = if_alloc(IFT_ETHER);
 1014         if (ifp == NULL) {
 1015                 device_printf(dev, "Cannot allocate ifnet\n");
 1016                 return (ENOMEM);
 1017         }
 1018         
 1019         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 1020         ifp->if_init = cxgb_init;
 1021         ifp->if_softc = p;
 1022         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 1023         ifp->if_ioctl = cxgb_ioctl;
 1024         ifp->if_transmit = cxgb_transmit;
 1025         ifp->if_qflush = cxgb_qflush;
 1026         ifp->if_get_counter = cxgb_get_counter;
 1027 
 1028         ifp->if_capabilities = CXGB_CAP;
 1029 #ifdef TCP_OFFLOAD
 1030         if (is_offload(sc))
 1031                 ifp->if_capabilities |= IFCAP_TOE4;
 1032 #endif
 1033         ifp->if_capenable = CXGB_CAP_ENABLE;
 1034         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
 1035             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
 1036 
 1037         /*
 1038          * Disable TSO on 4-port - it isn't supported by the firmware.
 1039          */     
 1040         if (sc->params.nports > 2) {
 1041                 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
 1042                 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
 1043                 ifp->if_hwassist &= ~CSUM_TSO;
 1044         }
 1045 
 1046         ether_ifattach(ifp, p->hw_addr);
 1047 
 1048 #ifdef DEFAULT_JUMBO
 1049         if (sc->params.nports <= 2)
 1050                 ifp->if_mtu = ETHERMTU_JUMBO;
 1051 #endif
 1052         if ((err = cxgb_makedev(p)) != 0) {
 1053                 printf("makedev failed %d\n", err);
 1054                 return (err);
 1055         }
 1056 
 1057         /* Create a list of media supported by this port */
 1058         ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
 1059             cxgb_media_status);
 1060         cxgb_build_medialist(p);
 1061       
 1062         t3_sge_init_port(p);
 1063 
 1064         return (err);
 1065 }
 1066 
 1067 /*
 1068  * cxgb_port_detach() is called via the device_detach methods when
 1069  * cxgb_free() calls the bus_generic_detach.  It is responsible for 
 1070  * removing the device from the view of the kernel, i.e. from all 
 1071  * interfaces lists etc.  This routine is only called when the driver is 
 1072  * being unloaded, not when the link goes down.
 1073  */
 1074 static int
 1075 cxgb_port_detach(device_t dev)
 1076 {
 1077         struct port_info *p;
 1078         struct adapter *sc;
 1079         int i;
 1080 
 1081         p = device_get_softc(dev);
 1082         sc = p->adapter;
 1083 
 1084         /* Tell cxgb_ioctl and if_init that the port is going away */
 1085         ADAPTER_LOCK(sc);
 1086         SET_DOOMED(p);
 1087         wakeup(&sc->flags);
 1088         while (IS_BUSY(sc))
 1089                 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
 1090         SET_BUSY(sc);
 1091         ADAPTER_UNLOCK(sc);
 1092 
 1093         if (p->port_cdev != NULL)
 1094                 destroy_dev(p->port_cdev);
 1095 
 1096         cxgb_uninit_synchronized(p);
 1097         ether_ifdetach(p->ifp);
 1098 
 1099         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
 1100                 struct sge_qset *qs = &sc->sge.qs[i];
 1101                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
 1102 
 1103                 callout_drain(&txq->txq_watchdog);
 1104                 callout_drain(&txq->txq_timer);
 1105         }
 1106 
 1107         PORT_LOCK_DEINIT(p);
 1108         if_free(p->ifp);
 1109         p->ifp = NULL;
 1110 
 1111         ADAPTER_LOCK(sc);
 1112         CLR_BUSY(sc);
 1113         wakeup_one(&sc->flags);
 1114         ADAPTER_UNLOCK(sc);
 1115         return (0);
 1116 }
 1117 
 1118 void
 1119 t3_fatal_err(struct adapter *sc)
 1120 {
 1121         u_int fw_status[4];
 1122 
 1123         if (sc->flags & FULL_INIT_DONE) {
 1124                 t3_sge_stop(sc);
 1125                 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
 1126                 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
 1127                 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
 1128                 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
 1129                 t3_intr_disable(sc);
 1130         }
 1131         device_printf(sc->dev,"encountered fatal error, operation suspended\n");
 1132         if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
 1133                 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
 1134                     fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
 1135 }
 1136 
 1137 int
 1138 t3_os_find_pci_capability(adapter_t *sc, int cap)
 1139 {
 1140         device_t dev;
 1141         struct pci_devinfo *dinfo;
 1142         pcicfgregs *cfg;
 1143         uint32_t status;
 1144         uint8_t ptr;
 1145 
 1146         dev = sc->dev;
 1147         dinfo = device_get_ivars(dev);
 1148         cfg = &dinfo->cfg;
 1149 
 1150         status = pci_read_config(dev, PCIR_STATUS, 2);
 1151         if (!(status & PCIM_STATUS_CAPPRESENT))
 1152                 return (0);
 1153 
 1154         switch (cfg->hdrtype & PCIM_HDRTYPE) {
 1155         case 0:
 1156         case 1:
 1157                 ptr = PCIR_CAP_PTR;
 1158                 break;
 1159         case 2:
 1160                 ptr = PCIR_CAP_PTR_2;
 1161                 break;
 1162         default:
 1163                 return (0);
 1164                 break;
 1165         }
 1166         ptr = pci_read_config(dev, ptr, 1);
 1167 
 1168         while (ptr != 0) {
 1169                 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
 1170                         return (ptr);
 1171                 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
 1172         }
 1173 
 1174         return (0);
 1175 }
 1176 
 1177 int
 1178 t3_os_pci_save_state(struct adapter *sc)
 1179 {
 1180         device_t dev;
 1181         struct pci_devinfo *dinfo;
 1182 
 1183         dev = sc->dev;
 1184         dinfo = device_get_ivars(dev);
 1185 
 1186         pci_cfg_save(dev, dinfo, 0);
 1187         return (0);
 1188 }
 1189 
 1190 int
 1191 t3_os_pci_restore_state(struct adapter *sc)
 1192 {
 1193         device_t dev;
 1194         struct pci_devinfo *dinfo;
 1195 
 1196         dev = sc->dev;
 1197         dinfo = device_get_ivars(dev);
 1198 
 1199         pci_cfg_restore(dev, dinfo);
 1200         return (0);
 1201 }
 1202 
 1203 /**
 1204  *      t3_os_link_changed - handle link status changes
 1205  *      @sc: the adapter associated with the link change
 1206  *      @port_id: the port index whose link status has changed
 1207  *      @link_status: the new status of the link
 1208  *      @speed: the new speed setting
 1209  *      @duplex: the new duplex setting
 1210  *      @fc: the new flow-control setting
 1211  *
 1212  *      This is the OS-dependent handler for link status changes.  The OS
 1213  *      neutral handler takes care of most of the processing for these events,
 1214  *      then calls this handler for any OS-specific processing.
 1215  */
 1216 void
 1217 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
 1218      int duplex, int fc, int mac_was_reset)
 1219 {
 1220         struct port_info *pi = &adapter->port[port_id];
 1221         struct ifnet *ifp = pi->ifp;
 1222 
 1223         /* no race with detach, so ifp should always be good */
 1224         KASSERT(ifp, ("%s: if detached.", __func__));
 1225 
 1226         /* Reapply mac settings if they were lost due to a reset */
 1227         if (mac_was_reset) {
 1228                 PORT_LOCK(pi);
 1229                 cxgb_update_mac_settings(pi);
 1230                 PORT_UNLOCK(pi);
 1231         }
 1232 
 1233         if (link_status) {
 1234                 ifp->if_baudrate = IF_Mbps(speed);
 1235                 if_link_state_change(ifp, LINK_STATE_UP);
 1236         } else
 1237                 if_link_state_change(ifp, LINK_STATE_DOWN);
 1238 }
 1239 
 1240 /**
 1241  *      t3_os_phymod_changed - handle PHY module changes
 1242  *      @phy: the PHY reporting the module change
 1243  *      @mod_type: new module type
 1244  *
 1245  *      This is the OS-dependent handler for PHY module changes.  It is
 1246  *      invoked when a PHY module is removed or inserted for any OS-specific
 1247  *      processing.
 1248  */
 1249 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 1250 {
 1251         static const char *mod_str[] = {
 1252                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
 1253         };
 1254         struct port_info *pi = &adap->port[port_id];
 1255         int mod = pi->phy.modtype;
 1256 
 1257         if (mod != pi->media.ifm_cur->ifm_data)
 1258                 cxgb_build_medialist(pi);
 1259 
 1260         if (mod == phy_modtype_none)
 1261                 if_printf(pi->ifp, "PHY module unplugged\n");
 1262         else {
 1263                 KASSERT(mod < ARRAY_SIZE(mod_str),
 1264                         ("invalid PHY module type %d", mod));
 1265                 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
 1266         }
 1267 }
 1268 
 1269 void
 1270 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
 1271 {
 1272 
 1273         /*
 1274          * The ifnet might not be allocated before this gets called,
 1275          * as this is called early on in attach by t3_prep_adapter
 1276          * save the address off in the port structure
 1277          */
 1278         if (cxgb_debug)
 1279                 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
 1280         bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
 1281 }
 1282 
 1283 /*
 1284  * Programs the XGMAC based on the settings in the ifnet.  These settings
 1285  * include MTU, MAC address, mcast addresses, etc.
 1286  */
 1287 static void
 1288 cxgb_update_mac_settings(struct port_info *p)
 1289 {
 1290         struct ifnet *ifp = p->ifp;
 1291         struct t3_rx_mode rm;
 1292         struct cmac *mac = &p->mac;
 1293         int mtu, hwtagging;
 1294 
 1295         PORT_LOCK_ASSERT_OWNED(p);
 1296 
 1297         bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
 1298 
 1299         mtu = ifp->if_mtu;
 1300         if (ifp->if_capenable & IFCAP_VLAN_MTU)
 1301                 mtu += ETHER_VLAN_ENCAP_LEN;
 1302 
 1303         hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
 1304 
 1305         t3_mac_set_mtu(mac, mtu);
 1306         t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
 1307         t3_mac_set_address(mac, 0, p->hw_addr);
 1308         t3_init_rx_mode(&rm, p);
 1309         t3_mac_set_rx_mode(mac, &rm);
 1310 }
 1311 
 1312 
 1313 static int
 1314 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 1315                               unsigned long n)
 1316 {
 1317         int attempts = 5;
 1318 
 1319         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 1320                 if (!--attempts)
 1321                         return (ETIMEDOUT);
 1322                 t3_os_sleep(10);
 1323         }
 1324         return 0;
 1325 }
 1326 
 1327 static int
 1328 init_tp_parity(struct adapter *adap)
 1329 {
 1330         int i;
 1331         struct mbuf *m;
 1332         struct cpl_set_tcb_field *greq;
 1333         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 1334 
 1335         t3_tp_set_offload_mode(adap, 1);
 1336 
 1337         for (i = 0; i < 16; i++) {
 1338                 struct cpl_smt_write_req *req;
 1339 
 1340                 m = m_gethdr(M_WAITOK, MT_DATA);
 1341                 req = mtod(m, struct cpl_smt_write_req *);
 1342                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1343                 memset(req, 0, sizeof(*req));
 1344                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1345                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 1346                 req->iff = i;
 1347                 t3_mgmt_tx(adap, m);
 1348         }
 1349 
 1350         for (i = 0; i < 2048; i++) {
 1351                 struct cpl_l2t_write_req *req;
 1352 
 1353                 m = m_gethdr(M_WAITOK, MT_DATA);
 1354                 req = mtod(m, struct cpl_l2t_write_req *);
 1355                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1356                 memset(req, 0, sizeof(*req));
 1357                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1358                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 1359                 req->params = htonl(V_L2T_W_IDX(i));
 1360                 t3_mgmt_tx(adap, m);
 1361         }
 1362 
 1363         for (i = 0; i < 2048; i++) {
 1364                 struct cpl_rte_write_req *req;
 1365 
 1366                 m = m_gethdr(M_WAITOK, MT_DATA);
 1367                 req = mtod(m, struct cpl_rte_write_req *);
 1368                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1369                 memset(req, 0, sizeof(*req));
 1370                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1371                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 1372                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 1373                 t3_mgmt_tx(adap, m);
 1374         }
 1375 
 1376         m = m_gethdr(M_WAITOK, MT_DATA);
 1377         greq = mtod(m, struct cpl_set_tcb_field *);
 1378         m->m_len = m->m_pkthdr.len = sizeof(*greq);
 1379         memset(greq, 0, sizeof(*greq));
 1380         greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1381         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 1382         greq->mask = htobe64(1);
 1383         t3_mgmt_tx(adap, m);
 1384 
 1385         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 1386         t3_tp_set_offload_mode(adap, 0);
 1387         return (i);
 1388 }
 1389 
 1390 /**
 1391  *      setup_rss - configure Receive Side Steering (per-queue connection demux) 
 1392  *      @adap: the adapter
 1393  *
 1394  *      Sets up RSS to distribute packets to multiple receive queues.  We
 1395  *      configure the RSS CPU lookup table to distribute to the number of HW
 1396  *      receive queues, and the response queue lookup table to narrow that
 1397  *      down to the response queues actually configured for each port.
 1398  *      We always configure the RSS mapping for two ports since the mapping
 1399  *      table has plenty of entries.
 1400  */
 1401 static void
 1402 setup_rss(adapter_t *adap)
 1403 {
 1404         int i;
 1405         u_int nq[2]; 
 1406         uint8_t cpus[SGE_QSETS + 1];
 1407         uint16_t rspq_map[RSS_TABLE_SIZE];
 1408         
 1409         for (i = 0; i < SGE_QSETS; ++i)
 1410                 cpus[i] = i;
 1411         cpus[SGE_QSETS] = 0xff;
 1412 
 1413         nq[0] = nq[1] = 0;
 1414         for_each_port(adap, i) {
 1415                 const struct port_info *pi = adap2pinfo(adap, i);
 1416 
 1417                 nq[pi->tx_chan] += pi->nqsets;
 1418         }
 1419         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 1420                 rspq_map[i] = nq[0] ? i % nq[0] : 0;
 1421                 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
 1422         }
 1423 
 1424         /* Calculate the reverse RSS map table */
 1425         for (i = 0; i < SGE_QSETS; ++i)
 1426                 adap->rrss_map[i] = 0xff;
 1427         for (i = 0; i < RSS_TABLE_SIZE; ++i)
 1428                 if (adap->rrss_map[rspq_map[i]] == 0xff)
 1429                         adap->rrss_map[rspq_map[i]] = i;
 1430 
 1431         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 1432                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
 1433                       F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
 1434                       cpus, rspq_map);
 1435 
 1436 }
 1437 static void
 1438 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 1439                               int hi, int port)
 1440 {
 1441         struct mbuf *m;
 1442         struct mngt_pktsched_wr *req;
 1443 
 1444         m = m_gethdr(M_NOWAIT, MT_DATA);
 1445         if (m) {        
 1446                 req = mtod(m, struct mngt_pktsched_wr *);
 1447                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 1448                 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 1449                 req->sched = sched;
 1450                 req->idx = qidx;
 1451                 req->min = lo;
 1452                 req->max = hi;
 1453                 req->binding = port;
 1454                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1455                 t3_mgmt_tx(adap, m);
 1456         }
 1457 }
 1458 
 1459 static void
 1460 bind_qsets(adapter_t *sc)
 1461 {
 1462         int i, j;
 1463 
 1464         for (i = 0; i < (sc)->params.nports; ++i) {
 1465                 const struct port_info *pi = adap2pinfo(sc, i);
 1466 
 1467                 for (j = 0; j < pi->nqsets; ++j) {
 1468                         send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
 1469                                           -1, pi->tx_chan);
 1470 
 1471                 }
 1472         }
 1473 }
 1474 
 1475 static void
 1476 update_tpeeprom(struct adapter *adap)
 1477 {
 1478         const struct firmware *tpeeprom;
 1479 
 1480         uint32_t version;
 1481         unsigned int major, minor;
 1482         int ret, len;
 1483         char rev, name[32];
 1484 
 1485         t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
 1486 
 1487         major = G_TP_VERSION_MAJOR(version);
 1488         minor = G_TP_VERSION_MINOR(version);
 1489         if (major == TP_VERSION_MAJOR  && minor == TP_VERSION_MINOR)
 1490                 return; 
 1491 
 1492         rev = t3rev2char(adap);
 1493         snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
 1494 
 1495         tpeeprom = firmware_get(name);
 1496         if (tpeeprom == NULL) {
 1497                 device_printf(adap->dev,
 1498                               "could not load TP EEPROM: unable to load %s\n",
 1499                               name);
 1500                 return;
 1501         }
 1502 
 1503         len = tpeeprom->datasize - 4;
 1504         
 1505         ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
 1506         if (ret)
 1507                 goto release_tpeeprom;
 1508 
 1509         if (len != TP_SRAM_LEN) {
 1510                 device_printf(adap->dev,
 1511                               "%s length is wrong len=%d expected=%d\n", name,
 1512                               len, TP_SRAM_LEN);
 1513                 return;
 1514         }
 1515         
 1516         ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
 1517             TP_SRAM_OFFSET);
 1518         
 1519         if (!ret) {
 1520                 device_printf(adap->dev,
 1521                         "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
 1522                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 1523         } else 
 1524                 device_printf(adap->dev,
 1525                               "Protocol SRAM image update in EEPROM failed\n");
 1526 
 1527 release_tpeeprom:
 1528         firmware_put(tpeeprom, FIRMWARE_UNLOAD);
 1529         
 1530         return;
 1531 }
 1532 
 1533 static int
 1534 update_tpsram(struct adapter *adap)
 1535 {
 1536         const struct firmware *tpsram;
 1537         int ret;
 1538         char rev, name[32];
 1539 
 1540         rev = t3rev2char(adap);
 1541         snprintf(name, sizeof(name), TPSRAM_NAME, rev);
 1542 
 1543         update_tpeeprom(adap);
 1544 
 1545         tpsram = firmware_get(name);
 1546         if (tpsram == NULL){
 1547                 device_printf(adap->dev, "could not load TP SRAM\n");
 1548                 return (EINVAL);
 1549         } else
 1550                 device_printf(adap->dev, "updating TP SRAM\n");
 1551         
 1552         ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
 1553         if (ret)
 1554                 goto release_tpsram;    
 1555 
 1556         ret = t3_set_proto_sram(adap, tpsram->data);
 1557         if (ret)
 1558                 device_printf(adap->dev, "loading protocol SRAM failed\n");
 1559 
 1560 release_tpsram:
 1561         firmware_put(tpsram, FIRMWARE_UNLOAD);
 1562         
 1563         return ret;
 1564 }
 1565 
 1566 /**
 1567  *      cxgb_up - enable the adapter
 1568  *      @adap: adapter being enabled
 1569  *
 1570  *      Called when the first port is enabled, this function performs the
 1571  *      actions necessary to make an adapter operational, such as completing
 1572  *      the initialization of HW modules, and enabling interrupts.
 1573  */
 1574 static int
 1575 cxgb_up(struct adapter *sc)
 1576 {
 1577         int err = 0;
 1578         unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
 1579 
 1580         KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
 1581                                            __func__, sc->open_device_map));
 1582 
 1583         if ((sc->flags & FULL_INIT_DONE) == 0) {
 1584 
 1585                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1586 
 1587                 if ((sc->flags & FW_UPTODATE) == 0)
 1588                         if ((err = upgrade_fw(sc)))
 1589                                 goto out;
 1590 
 1591                 if ((sc->flags & TPS_UPTODATE) == 0)
 1592                         if ((err = update_tpsram(sc)))
 1593                                 goto out;
 1594 
 1595                 if (is_offload(sc) && nfilters != 0) {
 1596                         sc->params.mc5.nservers = 0;
 1597 
 1598                         if (nfilters < 0)
 1599                                 sc->params.mc5.nfilters = mxf;
 1600                         else
 1601                                 sc->params.mc5.nfilters = min(nfilters, mxf);
 1602                 }
 1603 
 1604                 err = t3_init_hw(sc, 0);
 1605                 if (err)
 1606                         goto out;
 1607 
 1608                 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 1609                 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 1610 
 1611                 err = setup_sge_qsets(sc);
 1612                 if (err)
 1613                         goto out;
 1614 
 1615                 alloc_filters(sc);
 1616                 setup_rss(sc);
 1617 
 1618                 t3_add_configured_sysctls(sc);
 1619                 sc->flags |= FULL_INIT_DONE;
 1620         }
 1621 
 1622         t3_intr_clear(sc);
 1623         t3_sge_start(sc);
 1624         t3_intr_enable(sc);
 1625 
 1626         if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
 1627             is_offload(sc) && init_tp_parity(sc) == 0)
 1628                 sc->flags |= TP_PARITY_INIT;
 1629 
 1630         if (sc->flags & TP_PARITY_INIT) {
 1631                 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
 1632                 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
 1633         }
 1634         
 1635         if (!(sc->flags & QUEUES_BOUND)) {
 1636                 bind_qsets(sc);
 1637                 setup_hw_filters(sc);
 1638                 sc->flags |= QUEUES_BOUND;              
 1639         }
 1640 
 1641         t3_sge_reset_adapter(sc);
 1642 out:
 1643         return (err);
 1644 }
 1645 
 1646 /*
 1647  * Called when the last open device is closed.  Does NOT undo all of cxgb_up's
 1648  * work.  Specifically, the resources grabbed under FULL_INIT_DONE are released
 1649  * during controller_detach, not here.
 1650  */
 1651 static void
 1652 cxgb_down(struct adapter *sc)
 1653 {
 1654         t3_sge_stop(sc);
 1655         t3_intr_disable(sc);
 1656 }
 1657 
 1658 /*
 1659  * if_init for cxgb ports.
 1660  */
 1661 static void
 1662 cxgb_init(void *arg)
 1663 {
 1664         struct port_info *p = arg;
 1665         struct adapter *sc = p->adapter;
 1666 
 1667         ADAPTER_LOCK(sc);
 1668         cxgb_init_locked(p); /* releases adapter lock */
 1669         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1670 }
 1671 
 1672 static int
 1673 cxgb_init_locked(struct port_info *p)
 1674 {
 1675         struct adapter *sc = p->adapter;
 1676         struct ifnet *ifp = p->ifp;
 1677         struct cmac *mac = &p->mac;
 1678         int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
 1679 
 1680         ADAPTER_LOCK_ASSERT_OWNED(sc);
 1681 
 1682         while (!IS_DOOMED(p) && IS_BUSY(sc)) {
 1683                 gave_up_lock = 1;
 1684                 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
 1685                         rc = EINTR;
 1686                         goto done;
 1687                 }
 1688         }
 1689         if (IS_DOOMED(p)) {
 1690                 rc = ENXIO;
 1691                 goto done;
 1692         }
 1693         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
 1694 
 1695         /*
 1696          * The code that runs during one-time adapter initialization can sleep
 1697          * so it's important not to hold any locks across it.
 1698          */
 1699         may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
 1700 
 1701         if (may_sleep) {
 1702                 SET_BUSY(sc);
 1703                 gave_up_lock = 1;
 1704                 ADAPTER_UNLOCK(sc);
 1705         }
 1706 
 1707         if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
 1708                         goto done;
 1709 
 1710         PORT_LOCK(p);
 1711         if (isset(&sc->open_device_map, p->port_id) &&
 1712             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 1713                 PORT_UNLOCK(p);
 1714                 goto done;
 1715         }
 1716         t3_port_intr_enable(sc, p->port_id);
 1717         if (!mac->multiport) 
 1718                 t3_mac_init(mac);
 1719         cxgb_update_mac_settings(p);
 1720         t3_link_start(&p->phy, mac, &p->link_config);
 1721         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 1722         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 1723         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 1724         PORT_UNLOCK(p);
 1725 
 1726         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
 1727                 struct sge_qset *qs = &sc->sge.qs[i];
 1728                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
 1729 
 1730                 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
 1731                                  txq->txq_watchdog.c_cpu);
 1732         }
 1733 
 1734         /* all ok */
 1735         setbit(&sc->open_device_map, p->port_id);
 1736         callout_reset(&p->link_check_ch,
 1737             p->phy.caps & SUPPORTED_LINK_IRQ ?  hz * 3 : hz / 4,
 1738             link_check_callout, p);
 1739 
 1740 done:
 1741         if (may_sleep) {
 1742                 ADAPTER_LOCK(sc);
 1743                 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
 1744                 CLR_BUSY(sc);
 1745         }
 1746         if (gave_up_lock)
 1747                 wakeup_one(&sc->flags);
 1748         ADAPTER_UNLOCK(sc);
 1749         return (rc);
 1750 }
 1751 
 1752 static int
 1753 cxgb_uninit_locked(struct port_info *p)
 1754 {
 1755         struct adapter *sc = p->adapter;
 1756         int rc;
 1757 
 1758         ADAPTER_LOCK_ASSERT_OWNED(sc);
 1759 
 1760         while (!IS_DOOMED(p) && IS_BUSY(sc)) {
 1761                 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
 1762                         rc = EINTR;
 1763                         goto done;
 1764                 }
 1765         }
 1766         if (IS_DOOMED(p)) {
 1767                 rc = ENXIO;
 1768                 goto done;
 1769         }
 1770         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
 1771         SET_BUSY(sc);
 1772         ADAPTER_UNLOCK(sc);
 1773 
 1774         rc = cxgb_uninit_synchronized(p);
 1775 
 1776         ADAPTER_LOCK(sc);
 1777         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
 1778         CLR_BUSY(sc);
 1779         wakeup_one(&sc->flags);
 1780 done:
 1781         ADAPTER_UNLOCK(sc);
 1782         return (rc);
 1783 }
 1784 
 1785 /*
 1786  * Called on "ifconfig down", and from port_detach
 1787  */
 1788 static int
 1789 cxgb_uninit_synchronized(struct port_info *pi)
 1790 {
 1791         struct adapter *sc = pi->adapter;
 1792         struct ifnet *ifp = pi->ifp;
 1793 
 1794         /*
 1795          * taskqueue_drain may cause a deadlock if the adapter lock is held.
 1796          */
 1797         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1798 
 1799         /*
 1800          * Clear this port's bit from the open device map, and then drain all
 1801          * the tasks that can access/manipulate this port's port_info or ifp.
 1802          * We disable this port's interrupts here and so the slow/ext
 1803          * interrupt tasks won't be enqueued.  The tick task will continue to
 1804          * be enqueued every second but the runs after this drain will not see
 1805          * this port in the open device map.
 1806          *
 1807          * A well behaved task must take open_device_map into account and ignore
 1808          * ports that are not open.
 1809          */
 1810         clrbit(&sc->open_device_map, pi->port_id);
 1811         t3_port_intr_disable(sc, pi->port_id);
 1812         taskqueue_drain(sc->tq, &sc->slow_intr_task);
 1813         taskqueue_drain(sc->tq, &sc->tick_task);
 1814 
 1815         callout_drain(&pi->link_check_ch);
 1816         taskqueue_drain(sc->tq, &pi->link_check_task);
 1817 
 1818         PORT_LOCK(pi);
 1819         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 1820 
 1821         /* disable pause frames */
 1822         t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
 1823 
 1824         /* Reset RX FIFO HWM */
 1825         t3_set_reg_field(sc, A_XGM_RXFIFO_CFG +  pi->mac.offset,
 1826                          V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
 1827 
 1828         DELAY(100 * 1000);
 1829 
 1830         /* Wait for TXFIFO empty */
 1831         t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
 1832                         F_TXFIFO_EMPTY, 1, 20, 5);
 1833 
 1834         DELAY(100 * 1000);
 1835         t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
 1836 
 1837         pi->phy.ops->power_down(&pi->phy, 1);
 1838 
 1839         PORT_UNLOCK(pi);
 1840 
 1841         pi->link_config.link_ok = 0;
 1842         t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
 1843 
 1844         if (sc->open_device_map == 0)
 1845                 cxgb_down(pi->adapter);
 1846 
 1847         return (0);
 1848 }
 1849 
 1850 /*
 1851  * Mark lro enabled or disabled in all qsets for this port
 1852  */
 1853 static int
 1854 cxgb_set_lro(struct port_info *p, int enabled)
 1855 {
 1856         int i;
 1857         struct adapter *adp = p->adapter;
 1858         struct sge_qset *q;
 1859 
 1860         for (i = 0; i < p->nqsets; i++) {
 1861                 q = &adp->sge.qs[p->first_qset + i];
 1862                 q->lro.enabled = (enabled != 0);
 1863         }
 1864         return (0);
 1865 }
 1866 
 1867 static int
 1868 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 1869 {
 1870         struct port_info *p = ifp->if_softc;
 1871         struct adapter *sc = p->adapter;
 1872         struct ifreq *ifr = (struct ifreq *)data;
 1873         int flags, error = 0, mtu;
 1874         uint32_t mask;
 1875 
 1876         switch (command) {
 1877         case SIOCSIFMTU:
 1878                 ADAPTER_LOCK(sc);
 1879                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 1880                 if (error) {
 1881 fail:
 1882                         ADAPTER_UNLOCK(sc);
 1883                         return (error);
 1884                 }
 1885 
 1886                 mtu = ifr->ifr_mtu;
 1887                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
 1888                         error = EINVAL;
 1889                 } else {
 1890                         ifp->if_mtu = mtu;
 1891                         PORT_LOCK(p);
 1892                         cxgb_update_mac_settings(p);
 1893                         PORT_UNLOCK(p);
 1894                 }
 1895                 ADAPTER_UNLOCK(sc);
 1896                 break;
 1897         case SIOCSIFFLAGS:
 1898                 ADAPTER_LOCK(sc);
 1899                 if (IS_DOOMED(p)) {
 1900                         error = ENXIO;
 1901                         goto fail;
 1902                 }
 1903                 if (ifp->if_flags & IFF_UP) {
 1904                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1905                                 flags = p->if_flags;
 1906                                 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
 1907                                     ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
 1908                                         if (IS_BUSY(sc)) {
 1909                                                 error = EBUSY;
 1910                                                 goto fail;
 1911                                         }
 1912                                         PORT_LOCK(p);
 1913                                         cxgb_update_mac_settings(p);
 1914                                         PORT_UNLOCK(p);
 1915                                 }
 1916                                 ADAPTER_UNLOCK(sc);
 1917                         } else
 1918                                 error = cxgb_init_locked(p);
 1919                         p->if_flags = ifp->if_flags;
 1920                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1921                         error = cxgb_uninit_locked(p);
 1922                 else
 1923                         ADAPTER_UNLOCK(sc);
 1924 
 1925                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1926                 break;
 1927         case SIOCADDMULTI:
 1928         case SIOCDELMULTI:
 1929                 ADAPTER_LOCK(sc);
 1930                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 1931                 if (error)
 1932                         goto fail;
 1933 
 1934                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1935                         PORT_LOCK(p);
 1936                         cxgb_update_mac_settings(p);
 1937                         PORT_UNLOCK(p);
 1938                 }
 1939                 ADAPTER_UNLOCK(sc);
 1940 
 1941                 break;
 1942         case SIOCSIFCAP:
 1943                 ADAPTER_LOCK(sc);
 1944                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 1945                 if (error)
 1946                         goto fail;
 1947 
 1948                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 1949                 if (mask & IFCAP_TXCSUM) {
 1950                         ifp->if_capenable ^= IFCAP_TXCSUM;
 1951                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
 1952 
 1953                         if (IFCAP_TSO4 & ifp->if_capenable &&
 1954                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
 1955                                 ifp->if_capenable &= ~IFCAP_TSO4;
 1956                                 if_printf(ifp,
 1957                                     "tso4 disabled due to -txcsum.\n");
 1958                         }
 1959                 }
 1960                 if (mask & IFCAP_TXCSUM_IPV6) {
 1961                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
 1962                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
 1963 
 1964                         if (IFCAP_TSO6 & ifp->if_capenable &&
 1965                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
 1966                                 ifp->if_capenable &= ~IFCAP_TSO6;
 1967                                 if_printf(ifp,
 1968                                     "tso6 disabled due to -txcsum6.\n");
 1969                         }
 1970                 }
 1971                 if (mask & IFCAP_RXCSUM)
 1972                         ifp->if_capenable ^= IFCAP_RXCSUM;
 1973                 if (mask & IFCAP_RXCSUM_IPV6)
 1974                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
 1975 
 1976                 /*
 1977                  * Note that we leave CSUM_TSO alone (it is always set).  The
 1978                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
 1979                  * sending a TSO request our way, so it's sufficient to toggle
 1980                  * IFCAP_TSOx only.
 1981                  */
 1982                 if (mask & IFCAP_TSO4) {
 1983                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
 1984                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
 1985                                 if_printf(ifp, "enable txcsum first.\n");
 1986                                 error = EAGAIN;
 1987                                 goto fail;
 1988                         }
 1989                         ifp->if_capenable ^= IFCAP_TSO4;
 1990                 }
 1991                 if (mask & IFCAP_TSO6) {
 1992                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
 1993                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
 1994                                 if_printf(ifp, "enable txcsum6 first.\n");
 1995                                 error = EAGAIN;
 1996                                 goto fail;
 1997                         }
 1998                         ifp->if_capenable ^= IFCAP_TSO6;
 1999                 }
 2000                 if (mask & IFCAP_LRO) {
 2001                         ifp->if_capenable ^= IFCAP_LRO;
 2002 
 2003                         /* Safe to do this even if cxgb_up not called yet */
 2004                         cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
 2005                 }
 2006 #ifdef TCP_OFFLOAD
 2007                 if (mask & IFCAP_TOE4) {
 2008                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
 2009 
 2010                         error = toe_capability(p, enable);
 2011                         if (error == 0)
 2012                                 ifp->if_capenable ^= mask;
 2013                 }
 2014 #endif
 2015                 if (mask & IFCAP_VLAN_HWTAGGING) {
 2016                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 2017                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2018                                 PORT_LOCK(p);
 2019                                 cxgb_update_mac_settings(p);
 2020                                 PORT_UNLOCK(p);
 2021                         }
 2022                 }
 2023                 if (mask & IFCAP_VLAN_MTU) {
 2024                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
 2025                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2026                                 PORT_LOCK(p);
 2027                                 cxgb_update_mac_settings(p);
 2028                                 PORT_UNLOCK(p);
 2029                         }
 2030                 }
 2031                 if (mask & IFCAP_VLAN_HWTSO)
 2032                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 2033                 if (mask & IFCAP_VLAN_HWCSUM)
 2034                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
 2035 
 2036 #ifdef VLAN_CAPABILITIES
 2037                 VLAN_CAPABILITIES(ifp);
 2038 #endif
 2039                 ADAPTER_UNLOCK(sc);
 2040                 break;
 2041         case SIOCSIFMEDIA:
 2042         case SIOCGIFMEDIA:
 2043                 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
 2044                 break;
 2045         default:
 2046                 error = ether_ioctl(ifp, command, data);
 2047         }
 2048 
 2049         return (error);
 2050 }
 2051 
 2052 static int
 2053 cxgb_media_change(struct ifnet *ifp)
 2054 {
 2055         return (EOPNOTSUPP);
 2056 }
 2057 
 2058 /*
 2059  * Translates phy->modtype to the correct Ethernet media subtype.
 2060  */
 2061 static int
 2062 cxgb_ifm_type(int mod)
 2063 {
 2064         switch (mod) {
 2065         case phy_modtype_sr:
 2066                 return (IFM_10G_SR);
 2067         case phy_modtype_lr:
 2068                 return (IFM_10G_LR);
 2069         case phy_modtype_lrm:
 2070                 return (IFM_10G_LRM);
 2071         case phy_modtype_twinax:
 2072                 return (IFM_10G_TWINAX);
 2073         case phy_modtype_twinax_long:
 2074                 return (IFM_10G_TWINAX_LONG);
 2075         case phy_modtype_none:
 2076                 return (IFM_NONE);
 2077         case phy_modtype_unknown:
 2078                 return (IFM_UNKNOWN);
 2079         }
 2080 
 2081         KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
 2082         return (IFM_UNKNOWN);
 2083 }
 2084 
 2085 /*
 2086  * Rebuilds the ifmedia list for this port, and sets the current media.
 2087  */
 2088 static void
 2089 cxgb_build_medialist(struct port_info *p)
 2090 {
 2091         struct cphy *phy = &p->phy;
 2092         struct ifmedia *media = &p->media;
 2093         int mod = phy->modtype;
 2094         int m = IFM_ETHER | IFM_FDX;
 2095 
 2096         PORT_LOCK(p);
 2097 
 2098         ifmedia_removeall(media);
 2099         if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
 2100                 /* Copper (RJ45) */
 2101 
 2102                 if (phy->caps & SUPPORTED_10000baseT_Full)
 2103                         ifmedia_add(media, m | IFM_10G_T, mod, NULL);
 2104 
 2105                 if (phy->caps & SUPPORTED_1000baseT_Full)
 2106                         ifmedia_add(media, m | IFM_1000_T, mod, NULL);
 2107 
 2108                 if (phy->caps & SUPPORTED_100baseT_Full)
 2109                         ifmedia_add(media, m | IFM_100_TX, mod, NULL);
 2110 
 2111                 if (phy->caps & SUPPORTED_10baseT_Full)
 2112                         ifmedia_add(media, m | IFM_10_T, mod, NULL);
 2113 
 2114                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
 2115                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
 2116 
 2117         } else if (phy->caps & SUPPORTED_TP) {
 2118                 /* Copper (CX4) */
 2119 
 2120                 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
 2121                         ("%s: unexpected cap 0x%x", __func__, phy->caps));
 2122 
 2123                 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
 2124                 ifmedia_set(media, m | IFM_10G_CX4);
 2125 
 2126         } else if (phy->caps & SUPPORTED_FIBRE &&
 2127                    phy->caps & SUPPORTED_10000baseT_Full) {
 2128                 /* 10G optical (but includes SFP+ twinax) */
 2129 
 2130                 m |= cxgb_ifm_type(mod);
 2131                 if (IFM_SUBTYPE(m) == IFM_NONE)
 2132                         m &= ~IFM_FDX;
 2133 
 2134                 ifmedia_add(media, m, mod, NULL);
 2135                 ifmedia_set(media, m);
 2136 
 2137         } else if (phy->caps & SUPPORTED_FIBRE &&
 2138                    phy->caps & SUPPORTED_1000baseT_Full) {
 2139                 /* 1G optical */
 2140 
 2141                 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
 2142                 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
 2143                 ifmedia_set(media, m | IFM_1000_SX);
 2144 
 2145         } else {
 2146                 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
 2147                             phy->caps));
 2148         }
 2149 
 2150         PORT_UNLOCK(p);
 2151 }
 2152 
 2153 static void
 2154 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 2155 {
 2156         struct port_info *p = ifp->if_softc;
 2157         struct ifmedia_entry *cur = p->media.ifm_cur;
 2158         int speed = p->link_config.speed;
 2159 
 2160         if (cur->ifm_data != p->phy.modtype) {
 2161                 cxgb_build_medialist(p);
 2162                 cur = p->media.ifm_cur;
 2163         }
 2164 
 2165         ifmr->ifm_status = IFM_AVALID;
 2166         if (!p->link_config.link_ok)
 2167                 return;
 2168 
 2169         ifmr->ifm_status |= IFM_ACTIVE;
 2170 
 2171         /*
 2172          * active and current will differ iff current media is autoselect.  That
 2173          * can happen only for copper RJ45.
 2174          */
 2175         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
 2176                 return;
 2177         KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
 2178                 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
 2179 
 2180         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
 2181         if (speed == SPEED_10000)
 2182                 ifmr->ifm_active |= IFM_10G_T;
 2183         else if (speed == SPEED_1000)
 2184                 ifmr->ifm_active |= IFM_1000_T;
 2185         else if (speed == SPEED_100)
 2186                 ifmr->ifm_active |= IFM_100_TX;
 2187         else if (speed == SPEED_10)
 2188                 ifmr->ifm_active |= IFM_10_T;
 2189         else
 2190                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
 2191                             speed));
 2192 }
 2193 
 2194 static uint64_t
 2195 cxgb_get_counter(struct ifnet *ifp, ift_counter c)
 2196 {
 2197         struct port_info *pi = ifp->if_softc;
 2198         struct adapter *sc = pi->adapter;
 2199         struct cmac *mac = &pi->mac;
 2200         struct mac_stats *mstats = &mac->stats;
 2201 
 2202         cxgb_refresh_stats(pi);
 2203 
 2204         switch (c) {
 2205         case IFCOUNTER_IPACKETS:
 2206                 return (mstats->rx_frames);
 2207 
 2208         case IFCOUNTER_IERRORS:
 2209                 return (mstats->rx_jabber + mstats->rx_data_errs +
 2210                     mstats->rx_sequence_errs + mstats->rx_runt +
 2211                     mstats->rx_too_long + mstats->rx_mac_internal_errs +
 2212                     mstats->rx_short + mstats->rx_fcs_errs);
 2213 
 2214         case IFCOUNTER_OPACKETS:
 2215                 return (mstats->tx_frames);
 2216 
 2217         case IFCOUNTER_OERRORS:
 2218                 return (mstats->tx_excess_collisions + mstats->tx_underrun +
 2219                     mstats->tx_len_errs + mstats->tx_mac_internal_errs +
 2220                     mstats->tx_excess_deferral + mstats->tx_fcs_errs);
 2221 
 2222         case IFCOUNTER_COLLISIONS:
 2223                 return (mstats->tx_total_collisions);
 2224 
 2225         case IFCOUNTER_IBYTES:
 2226                 return (mstats->rx_octets);
 2227 
 2228         case IFCOUNTER_OBYTES:
 2229                 return (mstats->tx_octets);
 2230 
 2231         case IFCOUNTER_IMCASTS:
 2232                 return (mstats->rx_mcast_frames);
 2233 
 2234         case IFCOUNTER_OMCASTS:
 2235                 return (mstats->tx_mcast_frames);
 2236 
 2237         case IFCOUNTER_IQDROPS:
 2238                 return (mstats->rx_cong_drops);
 2239 
 2240         case IFCOUNTER_OQDROPS: {
 2241                 int i;
 2242                 uint64_t drops;
 2243 
 2244                 drops = 0;
 2245                 if (sc->flags & FULL_INIT_DONE) {
 2246                         for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
 2247                                 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
 2248                 }
 2249 
 2250                 return (drops);
 2251 
 2252         }
 2253 
 2254         default:
 2255                 return (if_get_counter_default(ifp, c));
 2256         }
 2257 }
 2258 
 2259 static void
 2260 cxgb_async_intr(void *data)
 2261 {
 2262         adapter_t *sc = data;
 2263 
 2264         t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
 2265         (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
 2266         taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
 2267 }
 2268 
 2269 static void
 2270 link_check_callout(void *arg)
 2271 {
 2272         struct port_info *pi = arg;
 2273         struct adapter *sc = pi->adapter;
 2274 
 2275         if (!isset(&sc->open_device_map, pi->port_id))
 2276                 return;
 2277 
 2278         taskqueue_enqueue(sc->tq, &pi->link_check_task);
 2279 }
 2280 
 2281 static void
 2282 check_link_status(void *arg, int pending)
 2283 {
 2284         struct port_info *pi = arg;
 2285         struct adapter *sc = pi->adapter;
 2286 
 2287         if (!isset(&sc->open_device_map, pi->port_id))
 2288                 return;
 2289 
 2290         t3_link_changed(sc, pi->port_id);
 2291 
 2292         if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
 2293             pi->link_config.link_ok == 0)
 2294                 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
 2295 }
 2296 
 2297 void
 2298 t3_os_link_intr(struct port_info *pi)
 2299 {
 2300         /*
 2301          * Schedule a link check in the near future.  If the link is flapping
 2302          * rapidly we'll keep resetting the callout and delaying the check until
 2303          * things stabilize a bit.
 2304          */
 2305         callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
 2306 }
 2307 
 2308 static void
 2309 check_t3b2_mac(struct adapter *sc)
 2310 {
 2311         int i;
 2312 
 2313         if (sc->flags & CXGB_SHUTDOWN)
 2314                 return;
 2315 
 2316         for_each_port(sc, i) {
 2317                 struct port_info *p = &sc->port[i];
 2318                 int status;
 2319 #ifdef INVARIANTS
 2320                 struct ifnet *ifp = p->ifp;
 2321 #endif          
 2322 
 2323                 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
 2324                     !p->link_config.link_ok)
 2325                         continue;
 2326 
 2327                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
 2328                         ("%s: state mismatch (drv_flags %x, device_map %x)",
 2329                          __func__, ifp->if_drv_flags, sc->open_device_map));
 2330 
 2331                 PORT_LOCK(p);
 2332                 status = t3b2_mac_watchdog_task(&p->mac);
 2333                 if (status == 1)
 2334                         p->mac.stats.num_toggled++;
 2335                 else if (status == 2) {
 2336                         struct cmac *mac = &p->mac;
 2337 
 2338                         cxgb_update_mac_settings(p);
 2339                         t3_link_start(&p->phy, mac, &p->link_config);
 2340                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 2341                         t3_port_intr_enable(sc, p->port_id);
 2342                         p->mac.stats.num_resets++;
 2343                 }
 2344                 PORT_UNLOCK(p);
 2345         }
 2346 }
 2347 
 2348 static void
 2349 cxgb_tick(void *arg)
 2350 {
 2351         adapter_t *sc = (adapter_t *)arg;
 2352 
 2353         if (sc->flags & CXGB_SHUTDOWN)
 2354                 return;
 2355 
 2356         taskqueue_enqueue(sc->tq, &sc->tick_task);      
 2357         callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
 2358 }
 2359 
 2360 void
 2361 cxgb_refresh_stats(struct port_info *pi)
 2362 {
 2363         struct timeval tv;
 2364         const struct timeval interval = {0, 250000};    /* 250ms */
 2365 
 2366         getmicrotime(&tv);
 2367         timevalsub(&tv, &interval);
 2368         if (timevalcmp(&tv, &pi->last_refreshed, <))
 2369                 return;
 2370 
 2371         PORT_LOCK(pi);
 2372         t3_mac_update_stats(&pi->mac);
 2373         PORT_UNLOCK(pi);
 2374         getmicrotime(&pi->last_refreshed);
 2375 }
 2376 
 2377 static void
 2378 cxgb_tick_handler(void *arg, int count)
 2379 {
 2380         adapter_t *sc = (adapter_t *)arg;
 2381         const struct adapter_params *p = &sc->params;
 2382         int i;
 2383         uint32_t cause, reset;
 2384 
 2385         if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
 2386                 return;
 2387 
 2388         if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 
 2389                 check_t3b2_mac(sc);
 2390 
 2391         cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
 2392         if (cause) {
 2393                 struct sge_qset *qs = &sc->sge.qs[0];
 2394                 uint32_t mask, v;
 2395 
 2396                 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
 2397 
 2398                 mask = 1;
 2399                 for (i = 0; i < SGE_QSETS; i++) {
 2400                         if (v & mask)
 2401                                 qs[i].rspq.starved++;
 2402                         mask <<= 1;
 2403                 }
 2404 
 2405                 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
 2406 
 2407                 for (i = 0; i < SGE_QSETS * 2; i++) {
 2408                         if (v & mask) {
 2409                                 qs[i / 2].fl[i % 2].empty++;
 2410                         }
 2411                         mask <<= 1;
 2412                 }
 2413 
 2414                 /* clear */
 2415                 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
 2416                 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
 2417         }
 2418 
 2419         for (i = 0; i < sc->params.nports; i++) {
 2420                 struct port_info *pi = &sc->port[i];
 2421                 struct cmac *mac = &pi->mac;
 2422 
 2423                 if (!isset(&sc->open_device_map, pi->port_id))
 2424                         continue;
 2425 
 2426                 cxgb_refresh_stats(pi);
 2427 
 2428                 if (mac->multiport)
 2429                         continue;
 2430 
 2431                 /* Count rx fifo overflows, once per second */
 2432                 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
 2433                 reset = 0;
 2434                 if (cause & F_RXFIFO_OVERFLOW) {
 2435                         mac->stats.rx_fifo_ovfl++;
 2436                         reset |= F_RXFIFO_OVERFLOW;
 2437                 }
 2438                 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
 2439         }
 2440 }
 2441 
 2442 static void
 2443 touch_bars(device_t dev)
 2444 {
 2445         /*
 2446          * Don't enable yet
 2447          */
 2448 #if !defined(__LP64__) && 0
 2449         u32 v;
 2450 
 2451         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
 2452         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
 2453         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
 2454         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
 2455         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
 2456         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
 2457 #endif
 2458 }
 2459 
 2460 static int
 2461 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
 2462 {
 2463         uint8_t *buf;
 2464         int err = 0;
 2465         u32 aligned_offset, aligned_len, *p;
 2466         struct adapter *adapter = pi->adapter;
 2467 
 2468 
 2469         aligned_offset = offset & ~3;
 2470         aligned_len = (len + (offset & 3) + 3) & ~3;
 2471 
 2472         if (aligned_offset != offset || aligned_len != len) {
 2473                 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);              
 2474                 if (!buf)
 2475                         return (ENOMEM);
 2476                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
 2477                 if (!err && aligned_len > 4)
 2478                         err = t3_seeprom_read(adapter,
 2479                                               aligned_offset + aligned_len - 4,
 2480                                               (u32 *)&buf[aligned_len - 4]);
 2481                 if (err)
 2482                         goto out;
 2483                 memcpy(buf + (offset & 3), data, len);
 2484         } else
 2485                 buf = (uint8_t *)(uintptr_t)data;
 2486 
 2487         err = t3_seeprom_wp(adapter, 0);
 2488         if (err)
 2489                 goto out;
 2490 
 2491         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
 2492                 err = t3_seeprom_write(adapter, aligned_offset, *p);
 2493                 aligned_offset += 4;
 2494         }
 2495 
 2496         if (!err)
 2497                 err = t3_seeprom_wp(adapter, 1);
 2498 out:
 2499         if (buf != data)
 2500                 free(buf, M_DEVBUF);
 2501         return err;
 2502 }
 2503 
 2504 
 2505 static int
 2506 in_range(int val, int lo, int hi)
 2507 {
 2508         return val < 0 || (val <= hi && val >= lo);
 2509 }
 2510 
 2511 static int
 2512 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
 2513 {
 2514        return (0);
 2515 }
 2516 
 2517 static int
 2518 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 2519 {
 2520        return (0);
 2521 }
 2522 
 2523 static int
 2524 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 2525     int fflag, struct thread *td)
 2526 {
 2527         int mmd, error = 0;
 2528         struct port_info *pi = dev->si_drv1;
 2529         adapter_t *sc = pi->adapter;
 2530 
 2531 #ifdef PRIV_SUPPORTED   
 2532         if (priv_check(td, PRIV_DRIVER)) {
 2533                 if (cxgb_debug) 
 2534                         printf("user does not have access to privileged ioctls\n");
 2535                 return (EPERM);
 2536         }
 2537 #else
 2538         if (suser(td)) {
 2539                 if (cxgb_debug)
 2540                         printf("user does not have access to privileged ioctls\n");
 2541                 return (EPERM);
 2542         }
 2543 #endif
 2544         
 2545         switch (cmd) {
 2546         case CHELSIO_GET_MIIREG: {
 2547                 uint32_t val;
 2548                 struct cphy *phy = &pi->phy;
 2549                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
 2550                 
 2551                 if (!phy->mdio_read)
 2552                         return (EOPNOTSUPP);
 2553                 if (is_10G(sc)) {
 2554                         mmd = mid->phy_id >> 8;
 2555                         if (!mmd)
 2556                                 mmd = MDIO_DEV_PCS;
 2557                         else if (mmd > MDIO_DEV_VEND2)
 2558                                 return (EINVAL);
 2559 
 2560                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
 2561                                              mid->reg_num, &val);
 2562                 } else
 2563                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
 2564                                              mid->reg_num & 0x1f, &val);
 2565                 if (error == 0)
 2566                         mid->val_out = val;
 2567                 break;
 2568         }
 2569         case CHELSIO_SET_MIIREG: {
 2570                 struct cphy *phy = &pi->phy;
 2571                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
 2572 
 2573                 if (!phy->mdio_write)
 2574                         return (EOPNOTSUPP);
 2575                 if (is_10G(sc)) {
 2576                         mmd = mid->phy_id >> 8;
 2577                         if (!mmd)
 2578                                 mmd = MDIO_DEV_PCS;
 2579                         else if (mmd > MDIO_DEV_VEND2)
 2580                                 return (EINVAL);
 2581                         
 2582                         error = phy->mdio_write(sc, mid->phy_id & 0x1f,
 2583                                               mmd, mid->reg_num, mid->val_in);
 2584                 } else
 2585                         error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
 2586                                               mid->reg_num & 0x1f,
 2587                                               mid->val_in);
 2588                 break;
 2589         }
 2590         case CHELSIO_SETREG: {
 2591                 struct ch_reg *edata = (struct ch_reg *)data;
 2592                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 2593                         return (EFAULT);
 2594                 t3_write_reg(sc, edata->addr, edata->val);
 2595                 break;
 2596         }
 2597         case CHELSIO_GETREG: {
 2598                 struct ch_reg *edata = (struct ch_reg *)data;
 2599                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 2600                         return (EFAULT);
 2601                 edata->val = t3_read_reg(sc, edata->addr);
 2602                 break;
 2603         }
 2604         case CHELSIO_GET_SGE_CONTEXT: {
 2605                 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
 2606                 mtx_lock_spin(&sc->sge.reg_lock);
 2607                 switch (ecntxt->cntxt_type) {
 2608                 case CNTXT_TYPE_EGRESS:
 2609                         error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
 2610                             ecntxt->data);
 2611                         break;
 2612                 case CNTXT_TYPE_FL:
 2613                         error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
 2614                             ecntxt->data);
 2615                         break;
 2616                 case CNTXT_TYPE_RSP:
 2617                         error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
 2618                             ecntxt->data);
 2619                         break;
 2620                 case CNTXT_TYPE_CQ:
 2621                         error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
 2622                             ecntxt->data);
 2623                         break;
 2624                 default:
 2625                         error = EINVAL;
 2626                         break;
 2627                 }
 2628                 mtx_unlock_spin(&sc->sge.reg_lock);
 2629                 break;
 2630         }
 2631         case CHELSIO_GET_SGE_DESC: {
 2632                 struct ch_desc *edesc = (struct ch_desc *)data;
 2633                 int ret;
 2634                 if (edesc->queue_num >= SGE_QSETS * 6)
 2635                         return (EINVAL);
 2636                 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
 2637                     edesc->queue_num % 6, edesc->idx, edesc->data);
 2638                 if (ret < 0)
 2639                         return (EINVAL);
 2640                 edesc->size = ret;
 2641                 break;
 2642         }
 2643         case CHELSIO_GET_QSET_PARAMS: {
 2644                 struct qset_params *q;
 2645                 struct ch_qset_params *t = (struct ch_qset_params *)data;
 2646                 int q1 = pi->first_qset;
 2647                 int nqsets = pi->nqsets;
 2648                 int i;
 2649 
 2650                 if (t->qset_idx >= nqsets)
 2651                         return EINVAL;
 2652 
 2653                 i = q1 + t->qset_idx;
 2654                 q = &sc->params.sge.qset[i];
 2655                 t->rspq_size   = q->rspq_size;
 2656                 t->txq_size[0] = q->txq_size[0];
 2657                 t->txq_size[1] = q->txq_size[1];
 2658                 t->txq_size[2] = q->txq_size[2];
 2659                 t->fl_size[0]  = q->fl_size;
 2660                 t->fl_size[1]  = q->jumbo_size;
 2661                 t->polling     = q->polling;
 2662                 t->lro         = q->lro;
 2663                 t->intr_lat    = q->coalesce_usecs;
 2664                 t->cong_thres  = q->cong_thres;
 2665                 t->qnum        = i;
 2666 
 2667                 if ((sc->flags & FULL_INIT_DONE) == 0)
 2668                         t->vector = 0;
 2669                 else if (sc->flags & USING_MSIX)
 2670                         t->vector = rman_get_start(sc->msix_irq_res[i]);
 2671                 else
 2672                         t->vector = rman_get_start(sc->irq_res);
 2673 
 2674                 break;
 2675         }
 2676         case CHELSIO_GET_QSET_NUM: {
 2677                 struct ch_reg *edata = (struct ch_reg *)data;
 2678                 edata->val = pi->nqsets;
 2679                 break;
 2680         }
 2681         case CHELSIO_LOAD_FW: {
 2682                 uint8_t *fw_data;
 2683                 uint32_t vers;
 2684                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2685 
 2686                 /*
 2687                  * You're allowed to load a firmware only before FULL_INIT_DONE
 2688                  *
 2689                  * FW_UPTODATE is also set so the rest of the initialization
 2690                  * will not overwrite what was loaded here.  This gives you the
 2691                  * flexibility to load any firmware (and maybe shoot yourself in
 2692                  * the foot).
 2693                  */
 2694 
 2695                 ADAPTER_LOCK(sc);
 2696                 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
 2697                         ADAPTER_UNLOCK(sc);
 2698                         return (EBUSY);
 2699                 }
 2700 
 2701                 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
 2702                 if (!fw_data)
 2703                         error = ENOMEM;
 2704                 else
 2705                         error = copyin(t->buf, fw_data, t->len);
 2706 
 2707                 if (!error)
 2708                         error = -t3_load_fw(sc, fw_data, t->len);
 2709 
 2710                 if (t3_get_fw_version(sc, &vers) == 0) {
 2711                         snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
 2712                             "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
 2713                             G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
 2714                 }
 2715 
 2716                 if (!error)
 2717                         sc->flags |= FW_UPTODATE;
 2718 
 2719                 free(fw_data, M_DEVBUF);
 2720                 ADAPTER_UNLOCK(sc);
 2721                 break;
 2722         }
 2723         case CHELSIO_LOAD_BOOT: {
 2724                 uint8_t *boot_data;
 2725                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2726 
 2727                 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
 2728                 if (!boot_data)
 2729                         return ENOMEM;
 2730 
 2731                 error = copyin(t->buf, boot_data, t->len);
 2732                 if (!error)
 2733                         error = -t3_load_boot(sc, boot_data, t->len);
 2734 
 2735                 free(boot_data, M_DEVBUF);
 2736                 break;
 2737         }
 2738         case CHELSIO_GET_PM: {
 2739                 struct ch_pm *m = (struct ch_pm *)data;
 2740                 struct tp_params *p = &sc->params.tp;
 2741 
 2742                 if (!is_offload(sc))
 2743                         return (EOPNOTSUPP);
 2744 
 2745                 m->tx_pg_sz = p->tx_pg_size;
 2746                 m->tx_num_pg = p->tx_num_pgs;
 2747                 m->rx_pg_sz  = p->rx_pg_size;
 2748                 m->rx_num_pg = p->rx_num_pgs;
 2749                 m->pm_total  = p->pmtx_size + p->chan_rx_size * p->nchan;
 2750 
 2751                 break;
 2752         }
 2753         case CHELSIO_SET_PM: {
 2754                 struct ch_pm *m = (struct ch_pm *)data;
 2755                 struct tp_params *p = &sc->params.tp;
 2756 
 2757                 if (!is_offload(sc))
 2758                         return (EOPNOTSUPP);
 2759                 if (sc->flags & FULL_INIT_DONE)
 2760                         return (EBUSY);
 2761 
 2762                 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
 2763                     !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
 2764                         return (EINVAL);        /* not power of 2 */
 2765                 if (!(m->rx_pg_sz & 0x14000))
 2766                         return (EINVAL);        /* not 16KB or 64KB */
 2767                 if (!(m->tx_pg_sz & 0x1554000))
 2768                         return (EINVAL);
 2769                 if (m->tx_num_pg == -1)
 2770                         m->tx_num_pg = p->tx_num_pgs;
 2771                 if (m->rx_num_pg == -1)
 2772                         m->rx_num_pg = p->rx_num_pgs;
 2773                 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
 2774                         return (EINVAL);
 2775                 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
 2776                     m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
 2777                         return (EINVAL);
 2778 
 2779                 p->rx_pg_size = m->rx_pg_sz;
 2780                 p->tx_pg_size = m->tx_pg_sz;
 2781                 p->rx_num_pgs = m->rx_num_pg;
 2782                 p->tx_num_pgs = m->tx_num_pg;
 2783                 break;
 2784         }
 2785         case CHELSIO_SETMTUTAB: {
 2786                 struct ch_mtus *m = (struct ch_mtus *)data;
 2787                 int i;
 2788                 
 2789                 if (!is_offload(sc))
 2790                         return (EOPNOTSUPP);
 2791                 if (offload_running(sc))
 2792                         return (EBUSY);
 2793                 if (m->nmtus != NMTUS)
 2794                         return (EINVAL);
 2795                 if (m->mtus[0] < 81)         /* accommodate SACK */
 2796                         return (EINVAL);
 2797                 
 2798                 /*
 2799                  * MTUs must be in ascending order
 2800                  */
 2801                 for (i = 1; i < NMTUS; ++i)
 2802                         if (m->mtus[i] < m->mtus[i - 1])
 2803                                 return (EINVAL);
 2804 
 2805                 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
 2806                 break;
 2807         }
 2808         case CHELSIO_GETMTUTAB: {
 2809                 struct ch_mtus *m = (struct ch_mtus *)data;
 2810 
 2811                 if (!is_offload(sc))
 2812                         return (EOPNOTSUPP);
 2813 
 2814                 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
 2815                 m->nmtus = NMTUS;
 2816                 break;
 2817         }
 2818         case CHELSIO_GET_MEM: {
 2819                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2820                 struct mc7 *mem;
 2821                 uint8_t *useraddr;
 2822                 u64 buf[32];
 2823 
 2824                 /*
 2825                  * Use these to avoid modifying len/addr in the return
 2826                  * struct
 2827                  */
 2828                 uint32_t len = t->len, addr = t->addr;
 2829 
 2830                 if (!is_offload(sc))
 2831                         return (EOPNOTSUPP);
 2832                 if (!(sc->flags & FULL_INIT_DONE))
 2833                         return (EIO);         /* need the memory controllers */
 2834                 if ((addr & 0x7) || (len & 0x7))
 2835                         return (EINVAL);
 2836                 if (t->mem_id == MEM_CM)
 2837                         mem = &sc->cm;
 2838                 else if (t->mem_id == MEM_PMRX)
 2839                         mem = &sc->pmrx;
 2840                 else if (t->mem_id == MEM_PMTX)
 2841                         mem = &sc->pmtx;
 2842                 else
 2843                         return (EINVAL);
 2844 
 2845                 /*
 2846                  * Version scheme:
 2847                  * bits 0..9: chip version
 2848                  * bits 10..15: chip revision
 2849                  */
 2850                 t->version = 3 | (sc->params.rev << 10);
 2851                 
 2852                 /*
 2853                  * Read 256 bytes at a time as len can be large and we don't
 2854                  * want to use huge intermediate buffers.
 2855                  */
 2856                 useraddr = (uint8_t *)t->buf; 
 2857                 while (len) {
 2858                         unsigned int chunk = min(len, sizeof(buf));
 2859 
 2860                         error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
 2861                         if (error)
 2862                                 return (-error);
 2863                         if (copyout(buf, useraddr, chunk))
 2864                                 return (EFAULT);
 2865                         useraddr += chunk;
 2866                         addr += chunk;
 2867                         len -= chunk;
 2868                 }
 2869                 break;
 2870         }
 2871         case CHELSIO_READ_TCAM_WORD: {
 2872                 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
 2873 
 2874                 if (!is_offload(sc))
 2875                         return (EOPNOTSUPP);
 2876                 if (!(sc->flags & FULL_INIT_DONE))
 2877                         return (EIO);         /* need MC5 */            
 2878                 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
 2879                 break;
 2880         }
 2881         case CHELSIO_SET_TRACE_FILTER: {
 2882                 struct ch_trace *t = (struct ch_trace *)data;
 2883                 const struct trace_params *tp;
 2884 
 2885                 tp = (const struct trace_params *)&t->sip;
 2886                 if (t->config_tx)
 2887                         t3_config_trace_filter(sc, tp, 0, t->invert_match,
 2888                                                t->trace_tx);
 2889                 if (t->config_rx)
 2890                         t3_config_trace_filter(sc, tp, 1, t->invert_match,
 2891                                                t->trace_rx);
 2892                 break;
 2893         }
 2894         case CHELSIO_SET_PKTSCHED: {
 2895                 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
 2896                 if (sc->open_device_map == 0)
 2897                         return (EAGAIN);
 2898                 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
 2899                     p->binding);
 2900                 break;
 2901         }
 2902         case CHELSIO_IFCONF_GETREGS: {
 2903                 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
 2904                 int reglen = cxgb_get_regs_len();
 2905                 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
 2906                 if (buf == NULL) {
 2907                         return (ENOMEM);
 2908                 }
 2909                 if (regs->len > reglen)
 2910                         regs->len = reglen;
 2911                 else if (regs->len < reglen)
 2912                         error = ENOBUFS;
 2913 
 2914                 if (!error) {
 2915                         cxgb_get_regs(sc, regs, buf);
 2916                         error = copyout(buf, regs->data, reglen);
 2917                 }
 2918                 free(buf, M_DEVBUF);
 2919 
 2920                 break;
 2921         }
 2922         case CHELSIO_SET_HW_SCHED: {
 2923                 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
 2924                 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
 2925 
 2926                 if ((sc->flags & FULL_INIT_DONE) == 0)
 2927                         return (EAGAIN);       /* need TP to be initialized */
 2928                 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
 2929                     !in_range(t->channel, 0, 1) ||
 2930                     !in_range(t->kbps, 0, 10000000) ||
 2931                     !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
 2932                     !in_range(t->flow_ipg, 0,
 2933                               dack_ticks_to_usec(sc, 0x7ff)))
 2934                         return (EINVAL);
 2935 
 2936                 if (t->kbps >= 0) {
 2937                         error = t3_config_sched(sc, t->kbps, t->sched);
 2938                         if (error < 0)
 2939                                 return (-error);
 2940                 }
 2941                 if (t->class_ipg >= 0)
 2942                         t3_set_sched_ipg(sc, t->sched, t->class_ipg);
 2943                 if (t->flow_ipg >= 0) {
 2944                         t->flow_ipg *= 1000;     /* us -> ns */
 2945                         t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
 2946                 }
 2947                 if (t->mode >= 0) {
 2948                         int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
 2949 
 2950                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
 2951                                          bit, t->mode ? bit : 0);
 2952                 }
 2953                 if (t->channel >= 0)
 2954                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
 2955                                          1 << t->sched, t->channel << t->sched);
 2956                 break;
 2957         }
 2958         case CHELSIO_GET_EEPROM: {
 2959                 int i;
 2960                 struct ch_eeprom *e = (struct ch_eeprom *)data;
 2961                 uint8_t *buf;
 2962 
 2963                 if (e->offset & 3 || e->offset >= EEPROMSIZE ||
 2964                     e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
 2965                         return (EINVAL);
 2966                 }
 2967 
 2968                 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
 2969                 if (buf == NULL) {
 2970                         return (ENOMEM);
 2971                 }
 2972                 e->magic = EEPROM_MAGIC;
 2973                 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
 2974                         error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
 2975 
 2976                 if (!error)
 2977                         error = copyout(buf + e->offset, e->data, e->len);
 2978 
 2979                 free(buf, M_DEVBUF);
 2980                 break;
 2981         }
 2982         case CHELSIO_CLEAR_STATS: {
 2983                 if (!(sc->flags & FULL_INIT_DONE))
 2984                         return EAGAIN;
 2985 
 2986                 PORT_LOCK(pi);
 2987                 t3_mac_update_stats(&pi->mac);
 2988                 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
 2989                 PORT_UNLOCK(pi);
 2990                 break;
 2991         }
 2992         case CHELSIO_GET_UP_LA: {
 2993                 struct ch_up_la *la = (struct ch_up_la *)data;
 2994                 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
 2995                 if (buf == NULL) {
 2996                         return (ENOMEM);
 2997                 }
 2998                 if (la->bufsize < LA_BUFSIZE)
 2999                         error = ENOBUFS;
 3000 
 3001                 if (!error)
 3002                         error = -t3_get_up_la(sc, &la->stopped, &la->idx,
 3003                                               &la->bufsize, buf);
 3004                 if (!error)
 3005                         error = copyout(buf, la->data, la->bufsize);
 3006 
 3007                 free(buf, M_DEVBUF);
 3008                 break;
 3009         }
 3010         case CHELSIO_GET_UP_IOQS: {
 3011                 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
 3012                 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
 3013                 uint32_t *v;
 3014 
 3015                 if (buf == NULL) {
 3016                         return (ENOMEM);
 3017                 }
 3018                 if (ioqs->bufsize < IOQS_BUFSIZE)
 3019                         error = ENOBUFS;
 3020 
 3021                 if (!error)
 3022                         error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
 3023 
 3024                 if (!error) {
 3025                         v = (uint32_t *)buf;
 3026 
 3027                         ioqs->ioq_rx_enable = *v++;
 3028                         ioqs->ioq_tx_enable = *v++;
 3029                         ioqs->ioq_rx_status = *v++;
 3030                         ioqs->ioq_tx_status = *v++;
 3031 
 3032                         error = copyout(v, ioqs->data, ioqs->bufsize);
 3033                 }
 3034 
 3035                 free(buf, M_DEVBUF);
 3036                 break;
 3037         }
 3038         case CHELSIO_SET_FILTER: {
 3039                 struct ch_filter *f = (struct ch_filter *)data;
 3040                 struct filter_info *p;
 3041                 unsigned int nfilters = sc->params.mc5.nfilters;
 3042 
 3043                 if (!is_offload(sc))
 3044                         return (EOPNOTSUPP);    /* No TCAM */
 3045                 if (!(sc->flags & FULL_INIT_DONE))
 3046                         return (EAGAIN);        /* mc5 not setup yet */
 3047                 if (nfilters == 0)
 3048                         return (EBUSY);         /* TOE will use TCAM */
 3049 
 3050                 /* sanity checks */
 3051                 if (f->filter_id >= nfilters ||
 3052                     (f->val.dip && f->mask.dip != 0xffffffff) ||
 3053                     (f->val.sport && f->mask.sport != 0xffff) ||
 3054                     (f->val.dport && f->mask.dport != 0xffff) ||
 3055                     (f->val.vlan && f->mask.vlan != 0xfff) ||
 3056                     (f->val.vlan_prio &&
 3057                         f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
 3058                     (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
 3059                     f->qset >= SGE_QSETS ||
 3060                     sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
 3061                         return (EINVAL);
 3062 
 3063                 /* Was allocated with M_WAITOK */
 3064                 KASSERT(sc->filters, ("filter table NULL\n"));
 3065 
 3066                 p = &sc->filters[f->filter_id];
 3067                 if (p->locked)
 3068                         return (EPERM);
 3069 
 3070                 bzero(p, sizeof(*p));
 3071                 p->sip = f->val.sip;
 3072                 p->sip_mask = f->mask.sip;
 3073                 p->dip = f->val.dip;
 3074                 p->sport = f->val.sport;
 3075                 p->dport = f->val.dport;
 3076                 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
 3077                 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
 3078                     FILTER_NO_VLAN_PRI;
 3079                 p->mac_hit = f->mac_hit;
 3080                 p->mac_vld = f->mac_addr_idx != 0xffff;
 3081                 p->mac_idx = f->mac_addr_idx;
 3082                 p->pkt_type = f->proto;
 3083                 p->report_filter_id = f->want_filter_id;
 3084                 p->pass = f->pass;
 3085                 p->rss = f->rss;
 3086                 p->qset = f->qset;
 3087 
 3088                 error = set_filter(sc, f->filter_id, p);
 3089                 if (error == 0)
 3090                         p->valid = 1;
 3091                 break;
 3092         }
 3093         case CHELSIO_DEL_FILTER: {
 3094                 struct ch_filter *f = (struct ch_filter *)data;
 3095                 struct filter_info *p;
 3096                 unsigned int nfilters = sc->params.mc5.nfilters;
 3097 
 3098                 if (!is_offload(sc))
 3099                         return (EOPNOTSUPP);
 3100                 if (!(sc->flags & FULL_INIT_DONE))
 3101                         return (EAGAIN);
 3102                 if (nfilters == 0 || sc->filters == NULL)
 3103                         return (EINVAL);
 3104                 if (f->filter_id >= nfilters)
 3105                        return (EINVAL);
 3106 
 3107                 p = &sc->filters[f->filter_id];
 3108                 if (p->locked)
 3109                         return (EPERM);
 3110                 if (!p->valid)
 3111                         return (EFAULT); /* Read "Bad address" as "Bad index" */
 3112 
 3113                 bzero(p, sizeof(*p));
 3114                 p->sip = p->sip_mask = 0xffffffff;
 3115                 p->vlan = 0xfff;
 3116                 p->vlan_prio = FILTER_NO_VLAN_PRI;
 3117                 p->pkt_type = 1;
 3118                 error = set_filter(sc, f->filter_id, p);
 3119                 break;
 3120         }
 3121         case CHELSIO_GET_FILTER: {
 3122                 struct ch_filter *f = (struct ch_filter *)data;
 3123                 struct filter_info *p;
 3124                 unsigned int i, nfilters = sc->params.mc5.nfilters;
 3125 
 3126                 if (!is_offload(sc))
 3127                         return (EOPNOTSUPP);
 3128                 if (!(sc->flags & FULL_INIT_DONE))
 3129                         return (EAGAIN);
 3130                 if (nfilters == 0 || sc->filters == NULL)
 3131                         return (EINVAL);
 3132 
 3133                 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
 3134                 for (; i < nfilters; i++) {
 3135                         p = &sc->filters[i];
 3136                         if (!p->valid)
 3137                                 continue;
 3138 
 3139                         bzero(f, sizeof(*f));
 3140 
 3141                         f->filter_id = i;
 3142                         f->val.sip = p->sip;
 3143                         f->mask.sip = p->sip_mask;
 3144                         f->val.dip = p->dip;
 3145                         f->mask.dip = p->dip ? 0xffffffff : 0;
 3146                         f->val.sport = p->sport;
 3147                         f->mask.sport = p->sport ? 0xffff : 0;
 3148                         f->val.dport = p->dport;
 3149                         f->mask.dport = p->dport ? 0xffff : 0;
 3150                         f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
 3151                         f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
 3152                         f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
 3153                             0 : p->vlan_prio;
 3154                         f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
 3155                             0 : FILTER_NO_VLAN_PRI;
 3156                         f->mac_hit = p->mac_hit;
 3157                         f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
 3158                         f->proto = p->pkt_type;
 3159                         f->want_filter_id = p->report_filter_id;
 3160                         f->pass = p->pass;
 3161                         f->rss = p->rss;
 3162                         f->qset = p->qset;
 3163 
 3164                         break;
 3165                 }
 3166                 
 3167                 if (i == nfilters)
 3168                         f->filter_id = 0xffffffff;
 3169                 break;
 3170         }
 3171         default:
 3172                 return (EOPNOTSUPP);
 3173                 break;
 3174         }
 3175 
 3176         return (error);
 3177 }
 3178 
 3179 static __inline void
 3180 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
 3181     unsigned int end)
 3182 {
 3183         uint32_t *p = (uint32_t *)(buf + start);
 3184 
 3185         for ( ; start <= end; start += sizeof(uint32_t))
 3186                 *p++ = t3_read_reg(ap, start);
 3187 }
 3188 
 3189 #define T3_REGMAP_SIZE (3 * 1024)
 3190 static int
 3191 cxgb_get_regs_len(void)
 3192 {
 3193         return T3_REGMAP_SIZE;
 3194 }
 3195 
 3196 static void
 3197 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
 3198 {           
 3199         
 3200         /*
 3201          * Version scheme:
 3202          * bits 0..9: chip version
 3203          * bits 10..15: chip revision
 3204          * bit 31: set for PCIe cards
 3205          */
 3206         regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
 3207 
 3208         /*
 3209          * We skip the MAC statistics registers because they are clear-on-read.
 3210          * Also reading multi-register stats would need to synchronize with the
 3211          * periodic mac stats accumulation.  Hard to justify the complexity.
 3212          */
 3213         memset(buf, 0, cxgb_get_regs_len());
 3214         reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
 3215         reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
 3216         reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
 3217         reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
 3218         reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
 3219         reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
 3220                        XGM_REG(A_XGM_SERDES_STAT3, 1));
 3221         reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
 3222                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
 3223 }
 3224 
 3225 static int
 3226 alloc_filters(struct adapter *sc)
 3227 {
 3228         struct filter_info *p;
 3229         unsigned int nfilters = sc->params.mc5.nfilters;
 3230 
 3231         if (nfilters == 0)
 3232                 return (0);
 3233 
 3234         p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
 3235         sc->filters = p;
 3236 
 3237         p = &sc->filters[nfilters - 1];
 3238         p->vlan = 0xfff;
 3239         p->vlan_prio = FILTER_NO_VLAN_PRI;
 3240         p->pass = p->rss = p->valid = p->locked = 1;
 3241 
 3242         return (0);
 3243 }
 3244 
 3245 static int
 3246 setup_hw_filters(struct adapter *sc)
 3247 {
 3248         int i, rc;
 3249         unsigned int nfilters = sc->params.mc5.nfilters;
 3250 
 3251         if (!sc->filters)
 3252                 return (0);
 3253 
 3254         t3_enable_filters(sc);
 3255 
 3256         for (i = rc = 0; i < nfilters && !rc; i++) {
 3257                 if (sc->filters[i].locked)
 3258                         rc = set_filter(sc, i, &sc->filters[i]);
 3259         }
 3260 
 3261         return (rc);
 3262 }
 3263 
 3264 static int
 3265 set_filter(struct adapter *sc, int id, const struct filter_info *f)
 3266 {
 3267         int len;
 3268         struct mbuf *m;
 3269         struct ulp_txpkt *txpkt;
 3270         struct work_request_hdr *wr;
 3271         struct cpl_pass_open_req *oreq;
 3272         struct cpl_set_tcb_field *sreq;
 3273 
 3274         len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
 3275         KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
 3276 
 3277         id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
 3278               sc->params.mc5.nfilters;
 3279 
 3280         m = m_gethdr(M_WAITOK, MT_DATA);
 3281         m->m_len = m->m_pkthdr.len = len;
 3282         bzero(mtod(m, char *), len);
 3283 
 3284         wr = mtod(m, struct work_request_hdr *);
 3285         wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
 3286 
 3287         oreq = (struct cpl_pass_open_req *)(wr + 1);
 3288         txpkt = (struct ulp_txpkt *)oreq;
 3289         txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
 3290         txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
 3291         OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
 3292         oreq->local_port = htons(f->dport);
 3293         oreq->peer_port = htons(f->sport);
 3294         oreq->local_ip = htonl(f->dip);
 3295         oreq->peer_ip = htonl(f->sip);
 3296         oreq->peer_netmask = htonl(f->sip_mask);
 3297         oreq->opt0h = 0;
 3298         oreq->opt0l = htonl(F_NO_OFFLOAD);
 3299         oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
 3300                          V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
 3301                          V_VLAN_PRI(f->vlan_prio >> 1) |
 3302                          V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
 3303                          V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
 3304                          V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
 3305 
 3306         sreq = (struct cpl_set_tcb_field *)(oreq + 1);
 3307         set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
 3308                           (f->report_filter_id << 15) | (1 << 23) |
 3309                           ((u64)f->pass << 35) | ((u64)!f->rss << 36));
 3310         set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
 3311         t3_mgmt_tx(sc, m);
 3312 
 3313         if (f->pass && !f->rss) {
 3314                 len = sizeof(*sreq);
 3315                 m = m_gethdr(M_WAITOK, MT_DATA);
 3316                 m->m_len = m->m_pkthdr.len = len;
 3317                 bzero(mtod(m, char *), len);
 3318                 sreq = mtod(m, struct cpl_set_tcb_field *);
 3319                 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 3320                 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
 3321                                  (u64)sc->rrss_map[f->qset] << 19);
 3322                 t3_mgmt_tx(sc, m);
 3323         }
 3324         return 0;
 3325 }
 3326 
 3327 static inline void
 3328 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
 3329     unsigned int word, u64 mask, u64 val)
 3330 {
 3331         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
 3332         req->reply = V_NO_REPLY(1);
 3333         req->cpu_idx = 0;
 3334         req->word = htons(word);
 3335         req->mask = htobe64(mask);
 3336         req->val = htobe64(val);
 3337 }
 3338 
 3339 static inline void
 3340 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
 3341     unsigned int word, u64 mask, u64 val)
 3342 {
 3343         struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
 3344 
 3345         txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
 3346         txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
 3347         mk_set_tcb_field(req, tid, word, mask, val);
 3348 }
 3349 
 3350 void
 3351 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
 3352 {
 3353         struct adapter *sc;
 3354 
 3355         mtx_lock(&t3_list_lock);
 3356         SLIST_FOREACH(sc, &t3_list, link) {
 3357                 /*
 3358                  * func should not make any assumptions about what state sc is
 3359                  * in - the only guarantee is that sc->sc_lock is a valid lock.
 3360                  */
 3361                 func(sc, arg);
 3362         }
 3363         mtx_unlock(&t3_list_lock);
 3364 }
 3365 
 3366 #ifdef TCP_OFFLOAD
 3367 static int
 3368 toe_capability(struct port_info *pi, int enable)
 3369 {
 3370         int rc;
 3371         struct adapter *sc = pi->adapter;
 3372 
 3373         ADAPTER_LOCK_ASSERT_OWNED(sc);
 3374 
 3375         if (!is_offload(sc))
 3376                 return (ENODEV);
 3377 
 3378         if (enable) {
 3379                 if (!(sc->flags & FULL_INIT_DONE)) {
 3380                         log(LOG_WARNING,
 3381                             "You must enable a cxgb interface first\n");
 3382                         return (EAGAIN);
 3383                 }
 3384 
 3385                 if (isset(&sc->offload_map, pi->port_id))
 3386                         return (0);
 3387 
 3388                 if (!(sc->flags & TOM_INIT_DONE)) {
 3389                         rc = t3_activate_uld(sc, ULD_TOM);
 3390                         if (rc == EAGAIN) {
 3391                                 log(LOG_WARNING,
 3392                                     "You must kldload t3_tom.ko before trying "
 3393                                     "to enable TOE on a cxgb interface.\n");
 3394                         }
 3395                         if (rc != 0)
 3396                                 return (rc);
 3397                         KASSERT(sc->tom_softc != NULL,
 3398                             ("%s: TOM activated but softc NULL", __func__));
 3399                         KASSERT(sc->flags & TOM_INIT_DONE,
 3400                             ("%s: TOM activated but flag not set", __func__));
 3401                 }
 3402 
 3403                 setbit(&sc->offload_map, pi->port_id);
 3404 
 3405                 /*
 3406                  * XXX: Temporary code to allow iWARP to be enabled when TOE is
 3407                  * enabled on any port.  Need to figure out how to enable,
 3408                  * disable, load, and unload iWARP cleanly.
 3409                  */
 3410                 if (!isset(&sc->offload_map, MAX_NPORTS) &&
 3411                     t3_activate_uld(sc, ULD_IWARP) == 0)
 3412                         setbit(&sc->offload_map, MAX_NPORTS);
 3413         } else {
 3414                 if (!isset(&sc->offload_map, pi->port_id))
 3415                         return (0);
 3416 
 3417                 KASSERT(sc->flags & TOM_INIT_DONE,
 3418                     ("%s: TOM never initialized?", __func__));
 3419                 clrbit(&sc->offload_map, pi->port_id);
 3420         }
 3421 
 3422         return (0);
 3423 }
 3424 
 3425 /*
 3426  * Add an upper layer driver to the global list.
 3427  */
 3428 int
 3429 t3_register_uld(struct uld_info *ui)
 3430 {
 3431         int rc = 0;
 3432         struct uld_info *u;
 3433 
 3434         mtx_lock(&t3_uld_list_lock);
 3435         SLIST_FOREACH(u, &t3_uld_list, link) {
 3436             if (u->uld_id == ui->uld_id) {
 3437                     rc = EEXIST;
 3438                     goto done;
 3439             }
 3440         }
 3441 
 3442         SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
 3443         ui->refcount = 0;
 3444 done:
 3445         mtx_unlock(&t3_uld_list_lock);
 3446         return (rc);
 3447 }
 3448 
 3449 int
 3450 t3_unregister_uld(struct uld_info *ui)
 3451 {
 3452         int rc = EINVAL;
 3453         struct uld_info *u;
 3454 
 3455         mtx_lock(&t3_uld_list_lock);
 3456 
 3457         SLIST_FOREACH(u, &t3_uld_list, link) {
 3458             if (u == ui) {
 3459                     if (ui->refcount > 0) {
 3460                             rc = EBUSY;
 3461                             goto done;
 3462                     }
 3463 
 3464                     SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
 3465                     rc = 0;
 3466                     goto done;
 3467             }
 3468         }
 3469 done:
 3470         mtx_unlock(&t3_uld_list_lock);
 3471         return (rc);
 3472 }
 3473 
 3474 int
 3475 t3_activate_uld(struct adapter *sc, int id)
 3476 {
 3477         int rc = EAGAIN;
 3478         struct uld_info *ui;
 3479 
 3480         mtx_lock(&t3_uld_list_lock);
 3481 
 3482         SLIST_FOREACH(ui, &t3_uld_list, link) {
 3483                 if (ui->uld_id == id) {
 3484                         rc = ui->activate(sc);
 3485                         if (rc == 0)
 3486                                 ui->refcount++;
 3487                         goto done;
 3488                 }
 3489         }
 3490 done:
 3491         mtx_unlock(&t3_uld_list_lock);
 3492 
 3493         return (rc);
 3494 }
 3495 
 3496 int
 3497 t3_deactivate_uld(struct adapter *sc, int id)
 3498 {
 3499         int rc = EINVAL;
 3500         struct uld_info *ui;
 3501 
 3502         mtx_lock(&t3_uld_list_lock);
 3503 
 3504         SLIST_FOREACH(ui, &t3_uld_list, link) {
 3505                 if (ui->uld_id == id) {
 3506                         rc = ui->deactivate(sc);
 3507                         if (rc == 0)
 3508                                 ui->refcount--;
 3509                         goto done;
 3510                 }
 3511         }
 3512 done:
 3513         mtx_unlock(&t3_uld_list_lock);
 3514 
 3515         return (rc);
 3516 }
 3517 
 3518 static int
 3519 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
 3520     struct mbuf *m)
 3521 {
 3522         m_freem(m);
 3523         return (EDOOFUS);
 3524 }
 3525 
 3526 int
 3527 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
 3528 {
 3529         uintptr_t *loc, new;
 3530 
 3531         if (opcode >= NUM_CPL_HANDLERS)
 3532                 return (EINVAL);
 3533 
 3534         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
 3535         loc = (uintptr_t *) &sc->cpl_handler[opcode];
 3536         atomic_store_rel_ptr(loc, new);
 3537 
 3538         return (0);
 3539 }
 3540 #endif
 3541 
 3542 static int
 3543 cxgbc_mod_event(module_t mod, int cmd, void *arg)
 3544 {
 3545         int rc = 0;
 3546 
 3547         switch (cmd) {
 3548         case MOD_LOAD:
 3549                 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
 3550                 SLIST_INIT(&t3_list);
 3551 #ifdef TCP_OFFLOAD
 3552                 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
 3553                 SLIST_INIT(&t3_uld_list);
 3554 #endif
 3555                 break;
 3556 
 3557         case MOD_UNLOAD:
 3558 #ifdef TCP_OFFLOAD
 3559                 mtx_lock(&t3_uld_list_lock);
 3560                 if (!SLIST_EMPTY(&t3_uld_list)) {
 3561                         rc = EBUSY;
 3562                         mtx_unlock(&t3_uld_list_lock);
 3563                         break;
 3564                 }
 3565                 mtx_unlock(&t3_uld_list_lock);
 3566                 mtx_destroy(&t3_uld_list_lock);
 3567 #endif
 3568                 mtx_lock(&t3_list_lock);
 3569                 if (!SLIST_EMPTY(&t3_list)) {
 3570                         rc = EBUSY;
 3571                         mtx_unlock(&t3_list_lock);
 3572                         break;
 3573                 }
 3574                 mtx_unlock(&t3_list_lock);
 3575                 mtx_destroy(&t3_list_lock);
 3576                 break;
 3577         }
 3578 
 3579         return (rc);
 3580 }

Cache object: c6f3d20b97067290402d9efda47ff224


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.