The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/cxgb/cxgb_main.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /**************************************************************************
    2 SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3 
    4 Copyright (c) 2007-2009, Chelsio Inc.
    5 All rights reserved.
    6 
    7 Redistribution and use in source and binary forms, with or without
    8 modification, are permitted provided that the following conditions are met:
    9 
   10  1. Redistributions of source code must retain the above copyright notice,
   11     this list of conditions and the following disclaimer.
   12 
   13  2. Neither the name of the Chelsio Corporation nor the names of its
   14     contributors may be used to endorse or promote products derived from
   15     this software without specific prior written permission.
   16 
   17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   27 POSSIBILITY OF SUCH DAMAGE.
   28 
   29 ***************************************************************************/
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD: releng/12.0/sys/dev/cxgb/cxgb_main.c 338949 2018-09-26 17:12:30Z imp $");
   33 
   34 #include "opt_inet.h"
   35 
   36 #include <sys/param.h>
   37 #include <sys/systm.h>
   38 #include <sys/kernel.h>
   39 #include <sys/bus.h>
   40 #include <sys/module.h>
   41 #include <sys/pciio.h>
   42 #include <sys/conf.h>
   43 #include <machine/bus.h>
   44 #include <machine/resource.h>
   45 #include <sys/ktr.h>
   46 #include <sys/rman.h>
   47 #include <sys/ioccom.h>
   48 #include <sys/mbuf.h>
   49 #include <sys/linker.h>
   50 #include <sys/firmware.h>
   51 #include <sys/socket.h>
   52 #include <sys/sockio.h>
   53 #include <sys/smp.h>
   54 #include <sys/sysctl.h>
   55 #include <sys/syslog.h>
   56 #include <sys/queue.h>
   57 #include <sys/taskqueue.h>
   58 #include <sys/proc.h>
   59 
   60 #include <net/bpf.h>
   61 #include <net/ethernet.h>
   62 #include <net/if.h>
   63 #include <net/if_var.h>
   64 #include <net/if_arp.h>
   65 #include <net/if_dl.h>
   66 #include <net/if_media.h>
   67 #include <net/if_types.h>
   68 #include <net/if_vlan_var.h>
   69 
   70 #include <netinet/in_systm.h>
   71 #include <netinet/in.h>
   72 #include <netinet/if_ether.h>
   73 #include <netinet/ip.h>
   74 #include <netinet/ip.h>
   75 #include <netinet/tcp.h>
   76 #include <netinet/udp.h>
   77 #include <netinet/netdump/netdump.h>
   78 
   79 #include <dev/pci/pcireg.h>
   80 #include <dev/pci/pcivar.h>
   81 #include <dev/pci/pci_private.h>
   82 
   83 #include <cxgb_include.h>
   84 
   85 #ifdef PRIV_SUPPORTED
   86 #include <sys/priv.h>
   87 #endif
   88 
   89 static int cxgb_setup_interrupts(adapter_t *);
   90 static void cxgb_teardown_interrupts(adapter_t *);
   91 static void cxgb_init(void *);
   92 static int cxgb_init_locked(struct port_info *);
   93 static int cxgb_uninit_locked(struct port_info *);
   94 static int cxgb_uninit_synchronized(struct port_info *);
   95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
   96 static int cxgb_media_change(struct ifnet *);
   97 static int cxgb_ifm_type(int);
   98 static void cxgb_build_medialist(struct port_info *);
   99 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
  100 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
  101 static int setup_sge_qsets(adapter_t *);
  102 static void cxgb_async_intr(void *);
  103 static void cxgb_tick_handler(void *, int);
  104 static void cxgb_tick(void *);
  105 static void link_check_callout(void *);
  106 static void check_link_status(void *, int);
  107 static void setup_rss(adapter_t *sc);
  108 static int alloc_filters(struct adapter *);
  109 static int setup_hw_filters(struct adapter *);
  110 static int set_filter(struct adapter *, int, const struct filter_info *);
  111 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
  112     unsigned int, u64, u64);
  113 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
  114     unsigned int, u64, u64);
  115 #ifdef TCP_OFFLOAD
  116 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
  117 #endif
  118 
  119 /* Attachment glue for the PCI controller end of the device.  Each port of
  120  * the device is attached separately, as defined later.
  121  */
  122 static int cxgb_controller_probe(device_t);
  123 static int cxgb_controller_attach(device_t);
  124 static int cxgb_controller_detach(device_t);
  125 static void cxgb_free(struct adapter *);
  126 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
  127     unsigned int end);
  128 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
  129 static int cxgb_get_regs_len(void);
  130 static void touch_bars(device_t dev);
  131 static void cxgb_update_mac_settings(struct port_info *p);
  132 #ifdef TCP_OFFLOAD
  133 static int toe_capability(struct port_info *, int);
  134 #endif
  135 
  136 /* Table for probing the cards.  The desc field isn't actually used */
  137 struct cxgb_ident {
  138         uint16_t        vendor;
  139         uint16_t        device;
  140         int             index;
  141         char            *desc;
  142 } cxgb_identifiers[] = {
  143         {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
  144         {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
  145         {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
  146         {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
  147         {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
  148         {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
  149         {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
  150         {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
  151         {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
  152         {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
  153         {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
  154         {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
  155         {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
  156         {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
  157         {0, 0, 0, NULL}
  158 };
  159 
  160 static device_method_t cxgb_controller_methods[] = {
  161         DEVMETHOD(device_probe,         cxgb_controller_probe),
  162         DEVMETHOD(device_attach,        cxgb_controller_attach),
  163         DEVMETHOD(device_detach,        cxgb_controller_detach),
  164 
  165         DEVMETHOD_END
  166 };
  167 
  168 static driver_t cxgb_controller_driver = {
  169         "cxgbc",
  170         cxgb_controller_methods,
  171         sizeof(struct adapter)
  172 };
  173 
  174 static int cxgbc_mod_event(module_t, int, void *);
  175 static devclass_t       cxgb_controller_devclass;
  176 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
  177     cxgbc_mod_event, 0);
  178 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers,
  179     nitems(cxgb_identifiers) - 1);
  180 MODULE_VERSION(cxgbc, 1);
  181 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
  182 
  183 /*
  184  * Attachment glue for the ports.  Attachment is done directly to the
  185  * controller device.
  186  */
  187 static int cxgb_port_probe(device_t);
  188 static int cxgb_port_attach(device_t);
  189 static int cxgb_port_detach(device_t);
  190 
  191 static device_method_t cxgb_port_methods[] = {
  192         DEVMETHOD(device_probe,         cxgb_port_probe),
  193         DEVMETHOD(device_attach,        cxgb_port_attach),
  194         DEVMETHOD(device_detach,        cxgb_port_detach),
  195         { 0, 0 }
  196 };
  197 
  198 static driver_t cxgb_port_driver = {
  199         "cxgb",
  200         cxgb_port_methods,
  201         0
  202 };
  203 
  204 static d_ioctl_t cxgb_extension_ioctl;
  205 static d_open_t cxgb_extension_open;
  206 static d_close_t cxgb_extension_close;
  207 
  208 static struct cdevsw cxgb_cdevsw = {
  209        .d_version =    D_VERSION,
  210        .d_flags =      0,
  211        .d_open =       cxgb_extension_open,
  212        .d_close =      cxgb_extension_close,
  213        .d_ioctl =      cxgb_extension_ioctl,
  214        .d_name =       "cxgb",
  215 };
  216 
  217 static devclass_t       cxgb_port_devclass;
  218 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
  219 MODULE_VERSION(cxgb, 1);
  220 
  221 NETDUMP_DEFINE(cxgb);
  222 
  223 static struct mtx t3_list_lock;
  224 static SLIST_HEAD(, adapter) t3_list;
  225 #ifdef TCP_OFFLOAD
  226 static struct mtx t3_uld_list_lock;
  227 static SLIST_HEAD(, uld_info) t3_uld_list;
  228 #endif
  229 
  230 /*
  231  * The driver uses the best interrupt scheme available on a platform in the
  232  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
  233  * of these schemes the driver may consider as follows:
  234  *
  235  * msi = 2: choose from among all three options
  236  * msi = 1 : only consider MSI and pin interrupts
  237  * msi = 0: force pin interrupts
  238  */
  239 static int msi_allowed = 2;
  240 
  241 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
  242 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
  243     "MSI-X, MSI, INTx selector");
  244 
  245 /*
  246  * The driver uses an auto-queue algorithm by default.
  247  * To disable it and force a single queue-set per port, use multiq = 0
  248  */
  249 static int multiq = 1;
  250 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
  251     "use min(ncpus/ports, 8) queue-sets per port");
  252 
  253 /*
  254  * By default the driver will not update the firmware unless
  255  * it was compiled against a newer version
  256  * 
  257  */
  258 static int force_fw_update = 0;
  259 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
  260     "update firmware even if up to date");
  261 
  262 int cxgb_use_16k_clusters = -1;
  263 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
  264     &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
  265 
  266 static int nfilters = -1;
  267 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
  268     &nfilters, 0, "max number of entries in the filter table");
  269 
  270 enum {
  271         MAX_TXQ_ENTRIES      = 16384,
  272         MAX_CTRL_TXQ_ENTRIES = 1024,
  273         MAX_RSPQ_ENTRIES     = 16384,
  274         MAX_RX_BUFFERS       = 16384,
  275         MAX_RX_JUMBO_BUFFERS = 16384,
  276         MIN_TXQ_ENTRIES      = 4,
  277         MIN_CTRL_TXQ_ENTRIES = 4,
  278         MIN_RSPQ_ENTRIES     = 32,
  279         MIN_FL_ENTRIES       = 32,
  280         MIN_FL_JUMBO_ENTRIES = 32
  281 };
  282 
  283 struct filter_info {
  284         u32 sip;
  285         u32 sip_mask;
  286         u32 dip;
  287         u16 sport;
  288         u16 dport;
  289         u32 vlan:12;
  290         u32 vlan_prio:3;
  291         u32 mac_hit:1;
  292         u32 mac_idx:4;
  293         u32 mac_vld:1;
  294         u32 pkt_type:2;
  295         u32 report_filter_id:1;
  296         u32 pass:1;
  297         u32 rss:1;
  298         u32 qset:3;
  299         u32 locked:1;
  300         u32 valid:1;
  301 };
  302 
  303 enum { FILTER_NO_VLAN_PRI = 7 };
  304 
  305 #define EEPROM_MAGIC 0x38E2F10C
  306 
  307 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  308 
  309 
  310 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
  311 
  312 
  313 static __inline char
  314 t3rev2char(struct adapter *adapter)
  315 {
  316         char rev = 'z';
  317 
  318         switch(adapter->params.rev) {
  319         case T3_REV_A:
  320                 rev = 'a';
  321                 break;
  322         case T3_REV_B:
  323         case T3_REV_B2:
  324                 rev = 'b';
  325                 break;
  326         case T3_REV_C:
  327                 rev = 'c';
  328                 break;
  329         }
  330         return rev;
  331 }
  332 
  333 static struct cxgb_ident *
  334 cxgb_get_ident(device_t dev)
  335 {
  336         struct cxgb_ident *id;
  337 
  338         for (id = cxgb_identifiers; id->desc != NULL; id++) {
  339                 if ((id->vendor == pci_get_vendor(dev)) &&
  340                     (id->device == pci_get_device(dev))) {
  341                         return (id);
  342                 }
  343         }
  344         return (NULL);
  345 }
  346 
  347 static const struct adapter_info *
  348 cxgb_get_adapter_info(device_t dev)
  349 {
  350         struct cxgb_ident *id;
  351         const struct adapter_info *ai;
  352 
  353         id = cxgb_get_ident(dev);
  354         if (id == NULL)
  355                 return (NULL);
  356 
  357         ai = t3_get_adapter_info(id->index);
  358 
  359         return (ai);
  360 }
  361 
  362 static int
  363 cxgb_controller_probe(device_t dev)
  364 {
  365         const struct adapter_info *ai;
  366         char *ports, buf[80];
  367         int nports;
  368 
  369         ai = cxgb_get_adapter_info(dev);
  370         if (ai == NULL)
  371                 return (ENXIO);
  372 
  373         nports = ai->nports0 + ai->nports1;
  374         if (nports == 1)
  375                 ports = "port";
  376         else
  377                 ports = "ports";
  378 
  379         snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
  380         device_set_desc_copy(dev, buf);
  381         return (BUS_PROBE_DEFAULT);
  382 }
  383 
  384 #define FW_FNAME "cxgb_t3fw"
  385 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
  386 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
  387 
  388 static int
  389 upgrade_fw(adapter_t *sc)
  390 {
  391         const struct firmware *fw;
  392         int status;
  393         u32 vers;
  394         
  395         if ((fw = firmware_get(FW_FNAME)) == NULL)  {
  396                 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
  397                 return (ENOENT);
  398         } else
  399                 device_printf(sc->dev, "installing firmware on card\n");
  400         status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
  401 
  402         if (status != 0) {
  403                 device_printf(sc->dev, "failed to install firmware: %d\n",
  404                     status);
  405         } else {
  406                 t3_get_fw_version(sc, &vers);
  407                 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
  408                     G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
  409                     G_FW_VERSION_MICRO(vers));
  410         }
  411 
  412         firmware_put(fw, FIRMWARE_UNLOAD);
  413 
  414         return (status);        
  415 }
  416 
  417 /*
  418  * The cxgb_controller_attach function is responsible for the initial
  419  * bringup of the device.  Its responsibilities include:
  420  *
  421  *  1. Determine if the device supports MSI or MSI-X.
  422  *  2. Allocate bus resources so that we can access the Base Address Register
  423  *  3. Create and initialize mutexes for the controller and its control
  424  *     logic such as SGE and MDIO.
  425  *  4. Call hardware specific setup routine for the adapter as a whole.
  426  *  5. Allocate the BAR for doing MSI-X.
  427  *  6. Setup the line interrupt iff MSI-X is not supported.
  428  *  7. Create the driver's taskq.
  429  *  8. Start one task queue service thread.
  430  *  9. Check if the firmware and SRAM are up-to-date.  They will be
  431  *     auto-updated later (before FULL_INIT_DONE), if required.
  432  * 10. Create a child device for each MAC (port)
  433  * 11. Initialize T3 private state.
  434  * 12. Trigger the LED
  435  * 13. Setup offload iff supported.
  436  * 14. Reset/restart the tick callout.
  437  * 15. Attach sysctls
  438  *
  439  * NOTE: Any modification or deviation from this list MUST be reflected in
  440  * the above comment.  Failure to do so will result in problems on various
  441  * error conditions including link flapping.
  442  */
  443 static int
  444 cxgb_controller_attach(device_t dev)
  445 {
  446         device_t child;
  447         const struct adapter_info *ai;
  448         struct adapter *sc;
  449         int i, error = 0;
  450         uint32_t vers;
  451         int port_qsets = 1;
  452         int msi_needed, reg;
  453         char buf[80];
  454 
  455         sc = device_get_softc(dev);
  456         sc->dev = dev;
  457         sc->msi_count = 0;
  458         ai = cxgb_get_adapter_info(dev);
  459 
  460         snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
  461             device_get_unit(dev));
  462         ADAPTER_LOCK_INIT(sc, sc->lockbuf);
  463 
  464         snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
  465             device_get_unit(dev));
  466         snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
  467             device_get_unit(dev));
  468         snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
  469             device_get_unit(dev));
  470         
  471         MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
  472         MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
  473         MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
  474 
  475         mtx_lock(&t3_list_lock);
  476         SLIST_INSERT_HEAD(&t3_list, sc, link);
  477         mtx_unlock(&t3_list_lock);
  478 
  479         /* find the PCIe link width and set max read request to 4KB*/
  480         if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
  481                 uint16_t lnk;
  482 
  483                 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
  484                 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
  485                 if (sc->link_width < 8 &&
  486                     (ai->caps & SUPPORTED_10000baseT_Full)) {
  487                         device_printf(sc->dev,
  488                             "PCIe x%d Link, expect reduced performance\n",
  489                             sc->link_width);
  490                 }
  491 
  492                 pci_set_max_read_req(dev, 4096);
  493         }
  494 
  495         touch_bars(dev);
  496         pci_enable_busmaster(dev);
  497         /*
  498          * Allocate the registers and make them available to the driver.
  499          * The registers that we care about for NIC mode are in BAR 0
  500          */
  501         sc->regs_rid = PCIR_BAR(0);
  502         if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  503             &sc->regs_rid, RF_ACTIVE)) == NULL) {
  504                 device_printf(dev, "Cannot allocate BAR region 0\n");
  505                 error = ENXIO;
  506                 goto out;
  507         }
  508 
  509         sc->bt = rman_get_bustag(sc->regs_res);
  510         sc->bh = rman_get_bushandle(sc->regs_res);
  511         sc->mmio_len = rman_get_size(sc->regs_res);
  512 
  513         for (i = 0; i < MAX_NPORTS; i++)
  514                 sc->port[i].adapter = sc;
  515 
  516         if (t3_prep_adapter(sc, ai, 1) < 0) {
  517                 printf("prep adapter failed\n");
  518                 error = ENODEV;
  519                 goto out;
  520         }
  521 
  522         sc->udbs_rid = PCIR_BAR(2);
  523         sc->udbs_res = NULL;
  524         if (is_offload(sc) &&
  525             ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  526                    &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
  527                 device_printf(dev, "Cannot allocate BAR region 1\n");
  528                 error = ENXIO;
  529                 goto out;
  530         }
  531 
  532         /* Allocate the BAR for doing MSI-X.  If it succeeds, try to allocate
  533          * enough messages for the queue sets.  If that fails, try falling
  534          * back to MSI.  If that fails, then try falling back to the legacy
  535          * interrupt pin model.
  536          */
  537         sc->msix_regs_rid = 0x20;
  538         if ((msi_allowed >= 2) &&
  539             (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  540             &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
  541 
  542                 if (multiq)
  543                         port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
  544                 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
  545 
  546                 if (pci_msix_count(dev) == 0 ||
  547                     (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
  548                     sc->msi_count != msi_needed) {
  549                         device_printf(dev, "alloc msix failed - "
  550                                       "msi_count=%d, msi_needed=%d, err=%d; "
  551                                       "will try MSI\n", sc->msi_count,
  552                                       msi_needed, error);
  553                         sc->msi_count = 0;
  554                         port_qsets = 1;
  555                         pci_release_msi(dev);
  556                         bus_release_resource(dev, SYS_RES_MEMORY,
  557                             sc->msix_regs_rid, sc->msix_regs_res);
  558                         sc->msix_regs_res = NULL;
  559                 } else {
  560                         sc->flags |= USING_MSIX;
  561                         sc->cxgb_intr = cxgb_async_intr;
  562                         device_printf(dev,
  563                                       "using MSI-X interrupts (%u vectors)\n",
  564                                       sc->msi_count);
  565                 }
  566         }
  567 
  568         if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
  569                 sc->msi_count = 1;
  570                 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
  571                         device_printf(dev, "alloc msi failed - "
  572                                       "err=%d; will try INTx\n", error);
  573                         sc->msi_count = 0;
  574                         port_qsets = 1;
  575                         pci_release_msi(dev);
  576                 } else {
  577                         sc->flags |= USING_MSI;
  578                         sc->cxgb_intr = t3_intr_msi;
  579                         device_printf(dev, "using MSI interrupts\n");
  580                 }
  581         }
  582         if (sc->msi_count == 0) {
  583                 device_printf(dev, "using line interrupts\n");
  584                 sc->cxgb_intr = t3b_intr;
  585         }
  586 
  587         /* Create a private taskqueue thread for handling driver events */
  588         sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
  589             taskqueue_thread_enqueue, &sc->tq);
  590         if (sc->tq == NULL) {
  591                 device_printf(dev, "failed to allocate controller task queue\n");
  592                 goto out;
  593         }
  594 
  595         taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
  596             device_get_nameunit(dev));
  597         TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
  598 
  599         
  600         /* Create a periodic callout for checking adapter status */
  601         callout_init(&sc->cxgb_tick_ch, 1);
  602         
  603         if (t3_check_fw_version(sc) < 0 || force_fw_update) {
  604                 /*
  605                  * Warn user that a firmware update will be attempted in init.
  606                  */
  607                 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
  608                     FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
  609                 sc->flags &= ~FW_UPTODATE;
  610         } else {
  611                 sc->flags |= FW_UPTODATE;
  612         }
  613 
  614         if (t3_check_tpsram_version(sc) < 0) {
  615                 /*
  616                  * Warn user that a firmware update will be attempted in init.
  617                  */
  618                 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
  619                     t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
  620                 sc->flags &= ~TPS_UPTODATE;
  621         } else {
  622                 sc->flags |= TPS_UPTODATE;
  623         }
  624 
  625         /*
  626          * Create a child device for each MAC.  The ethernet attachment
  627          * will be done in these children.
  628          */     
  629         for (i = 0; i < (sc)->params.nports; i++) {
  630                 struct port_info *pi;
  631                 
  632                 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
  633                         device_printf(dev, "failed to add child port\n");
  634                         error = EINVAL;
  635                         goto out;
  636                 }
  637                 pi = &sc->port[i];
  638                 pi->adapter = sc;
  639                 pi->nqsets = port_qsets;
  640                 pi->first_qset = i*port_qsets;
  641                 pi->port_id = i;
  642                 pi->tx_chan = i >= ai->nports0;
  643                 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
  644                 sc->rxpkt_map[pi->txpkt_intf] = i;
  645                 sc->port[i].tx_chan = i >= ai->nports0;
  646                 sc->portdev[i] = child;
  647                 device_set_softc(child, pi);
  648         }
  649         if ((error = bus_generic_attach(dev)) != 0)
  650                 goto out;
  651 
  652         /* initialize sge private state */
  653         t3_sge_init_adapter(sc);
  654 
  655         t3_led_ready(sc);
  656 
  657         error = t3_get_fw_version(sc, &vers);
  658         if (error)
  659                 goto out;
  660 
  661         snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
  662             G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
  663             G_FW_VERSION_MICRO(vers));
  664 
  665         snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
  666                  ai->desc, is_offload(sc) ? "R" : "",
  667                  sc->params.vpd.ec, sc->params.vpd.sn);
  668         device_set_desc_copy(dev, buf);
  669 
  670         snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
  671                  sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
  672                  sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
  673 
  674         device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
  675         callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
  676         t3_add_attach_sysctls(sc);
  677 
  678 #ifdef TCP_OFFLOAD
  679         for (i = 0; i < NUM_CPL_HANDLERS; i++)
  680                 sc->cpl_handler[i] = cpl_not_handled;
  681 #endif
  682 
  683         t3_intr_clear(sc);
  684         error = cxgb_setup_interrupts(sc);
  685 out:
  686         if (error)
  687                 cxgb_free(sc);
  688 
  689         return (error);
  690 }
  691 
  692 /*
  693  * The cxgb_controller_detach routine is called with the device is
  694  * unloaded from the system.
  695  */
  696 
  697 static int
  698 cxgb_controller_detach(device_t dev)
  699 {
  700         struct adapter *sc;
  701 
  702         sc = device_get_softc(dev);
  703 
  704         cxgb_free(sc);
  705 
  706         return (0);
  707 }
  708 
  709 /*
  710  * The cxgb_free() is called by the cxgb_controller_detach() routine
  711  * to tear down the structures that were built up in
  712  * cxgb_controller_attach(), and should be the final piece of work
  713  * done when fully unloading the driver.
  714  * 
  715  *
  716  *  1. Shutting down the threads started by the cxgb_controller_attach()
  717  *     routine.
  718  *  2. Stopping the lower level device and all callouts (cxgb_down_locked()).
  719  *  3. Detaching all of the port devices created during the
  720  *     cxgb_controller_attach() routine.
  721  *  4. Removing the device children created via cxgb_controller_attach().
  722  *  5. Releasing PCI resources associated with the device.
  723  *  6. Turning off the offload support, iff it was turned on.
  724  *  7. Destroying the mutexes created in cxgb_controller_attach().
  725  *
  726  */
  727 static void
  728 cxgb_free(struct adapter *sc)
  729 {
  730         int i, nqsets = 0;
  731 
  732         ADAPTER_LOCK(sc);
  733         sc->flags |= CXGB_SHUTDOWN;
  734         ADAPTER_UNLOCK(sc);
  735 
  736         /*
  737          * Make sure all child devices are gone.
  738          */
  739         bus_generic_detach(sc->dev);
  740         for (i = 0; i < (sc)->params.nports; i++) {
  741                 if (sc->portdev[i] &&
  742                     device_delete_child(sc->dev, sc->portdev[i]) != 0)
  743                         device_printf(sc->dev, "failed to delete child port\n");
  744                 nqsets += sc->port[i].nqsets;
  745         }
  746 
  747         /*
  748          * At this point, it is as if cxgb_port_detach has run on all ports, and
  749          * cxgb_down has run on the adapter.  All interrupts have been silenced,
  750          * all open devices have been closed.
  751          */
  752         KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
  753                                            __func__, sc->open_device_map));
  754         for (i = 0; i < sc->params.nports; i++) {
  755                 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
  756                                                   __func__, i));
  757         }
  758 
  759         /*
  760          * Finish off the adapter's callouts.
  761          */
  762         callout_drain(&sc->cxgb_tick_ch);
  763         callout_drain(&sc->sge_timer_ch);
  764 
  765         /*
  766          * Release resources grabbed under FULL_INIT_DONE by cxgb_up.  The
  767          * sysctls are cleaned up by the kernel linker.
  768          */
  769         if (sc->flags & FULL_INIT_DONE) {
  770                 t3_free_sge_resources(sc, nqsets);
  771                 sc->flags &= ~FULL_INIT_DONE;
  772         }
  773 
  774         /*
  775          * Release all interrupt resources.
  776          */
  777         cxgb_teardown_interrupts(sc);
  778         if (sc->flags & (USING_MSI | USING_MSIX)) {
  779                 device_printf(sc->dev, "releasing msi message(s)\n");
  780                 pci_release_msi(sc->dev);
  781         } else {
  782                 device_printf(sc->dev, "no msi message to release\n");
  783         }
  784 
  785         if (sc->msix_regs_res != NULL) {
  786                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
  787                     sc->msix_regs_res);
  788         }
  789 
  790         /*
  791          * Free the adapter's taskqueue.
  792          */
  793         if (sc->tq != NULL) {
  794                 taskqueue_free(sc->tq);
  795                 sc->tq = NULL;
  796         }
  797         
  798         free(sc->filters, M_DEVBUF);
  799         t3_sge_free(sc);
  800 
  801         if (sc->udbs_res != NULL)
  802                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
  803                     sc->udbs_res);
  804 
  805         if (sc->regs_res != NULL)
  806                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
  807                     sc->regs_res);
  808 
  809         MTX_DESTROY(&sc->mdio_lock);
  810         MTX_DESTROY(&sc->sge.reg_lock);
  811         MTX_DESTROY(&sc->elmer_lock);
  812         mtx_lock(&t3_list_lock);
  813         SLIST_REMOVE(&t3_list, sc, adapter, link);
  814         mtx_unlock(&t3_list_lock);
  815         ADAPTER_LOCK_DEINIT(sc);
  816 }
  817 
  818 /**
  819  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
  820  *      @sc: the controller softc
  821  *
  822  *      Determines how many sets of SGE queues to use and initializes them.
  823  *      We support multiple queue sets per port if we have MSI-X, otherwise
  824  *      just one queue set per port.
  825  */
  826 static int
  827 setup_sge_qsets(adapter_t *sc)
  828 {
  829         int i, j, err, irq_idx = 0, qset_idx = 0;
  830         u_int ntxq = SGE_TXQ_PER_SET;
  831 
  832         if ((err = t3_sge_alloc(sc)) != 0) {
  833                 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
  834                 return (err);
  835         }
  836 
  837         if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
  838                 irq_idx = -1;
  839 
  840         for (i = 0; i < (sc)->params.nports; i++) {
  841                 struct port_info *pi = &sc->port[i];
  842 
  843                 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
  844                         err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
  845                             (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
  846                             &sc->params.sge.qset[qset_idx], ntxq, pi);
  847                         if (err) {
  848                                 t3_free_sge_resources(sc, qset_idx);
  849                                 device_printf(sc->dev,
  850                                     "t3_sge_alloc_qset failed with %d\n", err);
  851                                 return (err);
  852                         }
  853                 }
  854         }
  855 
  856         return (0);
  857 }
  858 
  859 static void
  860 cxgb_teardown_interrupts(adapter_t *sc)
  861 {
  862         int i;
  863 
  864         for (i = 0; i < SGE_QSETS; i++) {
  865                 if (sc->msix_intr_tag[i] == NULL) {
  866 
  867                         /* Should have been setup fully or not at all */
  868                         KASSERT(sc->msix_irq_res[i] == NULL &&
  869                                 sc->msix_irq_rid[i] == 0,
  870                                 ("%s: half-done interrupt (%d).", __func__, i));
  871 
  872                         continue;
  873                 }
  874 
  875                 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
  876                                   sc->msix_intr_tag[i]);
  877                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
  878                                      sc->msix_irq_res[i]);
  879 
  880                 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
  881                 sc->msix_irq_rid[i] = 0;
  882         }
  883 
  884         if (sc->intr_tag) {
  885                 KASSERT(sc->irq_res != NULL,
  886                         ("%s: half-done interrupt.", __func__));
  887 
  888                 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
  889                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  890                                      sc->irq_res);
  891 
  892                 sc->irq_res = sc->intr_tag = NULL;
  893                 sc->irq_rid = 0;
  894         }
  895 }
  896 
  897 static int
  898 cxgb_setup_interrupts(adapter_t *sc)
  899 {
  900         struct resource *res;
  901         void *tag;
  902         int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
  903 
  904         sc->irq_rid = intr_flag ? 1 : 0;
  905         sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
  906                                              RF_SHAREABLE | RF_ACTIVE);
  907         if (sc->irq_res == NULL) {
  908                 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
  909                               intr_flag, sc->irq_rid);
  910                 err = EINVAL;
  911                 sc->irq_rid = 0;
  912         } else {
  913                 err = bus_setup_intr(sc->dev, sc->irq_res,
  914                     INTR_MPSAFE | INTR_TYPE_NET, NULL,
  915                     sc->cxgb_intr, sc, &sc->intr_tag);
  916 
  917                 if (err) {
  918                         device_printf(sc->dev,
  919                                       "Cannot set up interrupt (%x, %u, %d)\n",
  920                                       intr_flag, sc->irq_rid, err);
  921                         bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  922                                              sc->irq_res);
  923                         sc->irq_res = sc->intr_tag = NULL;
  924                         sc->irq_rid = 0;
  925                 }
  926         }
  927 
  928         /* That's all for INTx or MSI */
  929         if (!(intr_flag & USING_MSIX) || err)
  930                 return (err);
  931 
  932         bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
  933         for (i = 0; i < sc->msi_count - 1; i++) {
  934                 rid = i + 2;
  935                 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
  936                                              RF_SHAREABLE | RF_ACTIVE);
  937                 if (res == NULL) {
  938                         device_printf(sc->dev, "Cannot allocate interrupt "
  939                                       "for message %d\n", rid);
  940                         err = EINVAL;
  941                         break;
  942                 }
  943 
  944                 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
  945                                      NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
  946                 if (err) {
  947                         device_printf(sc->dev, "Cannot set up interrupt "
  948                                       "for message %d (%d)\n", rid, err);
  949                         bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
  950                         break;
  951                 }
  952 
  953                 sc->msix_irq_rid[i] = rid;
  954                 sc->msix_irq_res[i] = res;
  955                 sc->msix_intr_tag[i] = tag;
  956                 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
  957         }
  958 
  959         if (err)
  960                 cxgb_teardown_interrupts(sc);
  961 
  962         return (err);
  963 }
  964 
  965 
  966 static int
  967 cxgb_port_probe(device_t dev)
  968 {
  969         struct port_info *p;
  970         char buf[80];
  971         const char *desc;
  972         
  973         p = device_get_softc(dev);
  974         desc = p->phy.desc;
  975         snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
  976         device_set_desc_copy(dev, buf);
  977         return (0);
  978 }
  979 
  980 
  981 static int
  982 cxgb_makedev(struct port_info *pi)
  983 {
  984         
  985         pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
  986             UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
  987         
  988         if (pi->port_cdev == NULL)
  989                 return (ENOMEM);
  990 
  991         pi->port_cdev->si_drv1 = (void *)pi;
  992         
  993         return (0);
  994 }
  995 
  996 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
  997     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
  998     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
  999 #define CXGB_CAP_ENABLE CXGB_CAP
 1000 
 1001 static int
 1002 cxgb_port_attach(device_t dev)
 1003 {
 1004         struct port_info *p;
 1005         struct ifnet *ifp;
 1006         int err;
 1007         struct adapter *sc;
 1008 
 1009         p = device_get_softc(dev);
 1010         sc = p->adapter;
 1011         snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
 1012             device_get_unit(device_get_parent(dev)), p->port_id);
 1013         PORT_LOCK_INIT(p, p->lockbuf);
 1014 
 1015         callout_init(&p->link_check_ch, 1);
 1016         TASK_INIT(&p->link_check_task, 0, check_link_status, p);
 1017 
 1018         /* Allocate an ifnet object and set it up */
 1019         ifp = p->ifp = if_alloc(IFT_ETHER);
 1020         if (ifp == NULL) {
 1021                 device_printf(dev, "Cannot allocate ifnet\n");
 1022                 return (ENOMEM);
 1023         }
 1024         
 1025         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 1026         ifp->if_init = cxgb_init;
 1027         ifp->if_softc = p;
 1028         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 1029         ifp->if_ioctl = cxgb_ioctl;
 1030         ifp->if_transmit = cxgb_transmit;
 1031         ifp->if_qflush = cxgb_qflush;
 1032         ifp->if_get_counter = cxgb_get_counter;
 1033 
 1034         ifp->if_capabilities = CXGB_CAP;
 1035 #ifdef TCP_OFFLOAD
 1036         if (is_offload(sc))
 1037                 ifp->if_capabilities |= IFCAP_TOE4;
 1038 #endif
 1039         ifp->if_capenable = CXGB_CAP_ENABLE;
 1040         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
 1041             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
 1042 
 1043         /*
 1044          * Disable TSO on 4-port - it isn't supported by the firmware.
 1045          */     
 1046         if (sc->params.nports > 2) {
 1047                 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
 1048                 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
 1049                 ifp->if_hwassist &= ~CSUM_TSO;
 1050         }
 1051 
 1052         ether_ifattach(ifp, p->hw_addr);
 1053 
 1054         /* Attach driver netdump methods. */
 1055         NETDUMP_SET(ifp, cxgb);
 1056 
 1057 #ifdef DEFAULT_JUMBO
 1058         if (sc->params.nports <= 2)
 1059                 ifp->if_mtu = ETHERMTU_JUMBO;
 1060 #endif
 1061         if ((err = cxgb_makedev(p)) != 0) {
 1062                 printf("makedev failed %d\n", err);
 1063                 return (err);
 1064         }
 1065 
 1066         /* Create a list of media supported by this port */
 1067         ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
 1068             cxgb_media_status);
 1069         cxgb_build_medialist(p);
 1070       
 1071         t3_sge_init_port(p);
 1072 
 1073         return (err);
 1074 }
 1075 
 1076 /*
 1077  * cxgb_port_detach() is called via the device_detach methods when
 1078  * cxgb_free() calls the bus_generic_detach.  It is responsible for 
 1079  * removing the device from the view of the kernel, i.e. from all 
 1080  * interfaces lists etc.  This routine is only called when the driver is 
 1081  * being unloaded, not when the link goes down.
 1082  */
 1083 static int
 1084 cxgb_port_detach(device_t dev)
 1085 {
 1086         struct port_info *p;
 1087         struct adapter *sc;
 1088         int i;
 1089 
 1090         p = device_get_softc(dev);
 1091         sc = p->adapter;
 1092 
 1093         /* Tell cxgb_ioctl and if_init that the port is going away */
 1094         ADAPTER_LOCK(sc);
 1095         SET_DOOMED(p);
 1096         wakeup(&sc->flags);
 1097         while (IS_BUSY(sc))
 1098                 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
 1099         SET_BUSY(sc);
 1100         ADAPTER_UNLOCK(sc);
 1101 
 1102         if (p->port_cdev != NULL)
 1103                 destroy_dev(p->port_cdev);
 1104 
 1105         cxgb_uninit_synchronized(p);
 1106         ether_ifdetach(p->ifp);
 1107 
 1108         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
 1109                 struct sge_qset *qs = &sc->sge.qs[i];
 1110                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
 1111 
 1112                 callout_drain(&txq->txq_watchdog);
 1113                 callout_drain(&txq->txq_timer);
 1114         }
 1115 
 1116         PORT_LOCK_DEINIT(p);
 1117         if_free(p->ifp);
 1118         p->ifp = NULL;
 1119 
 1120         ADAPTER_LOCK(sc);
 1121         CLR_BUSY(sc);
 1122         wakeup_one(&sc->flags);
 1123         ADAPTER_UNLOCK(sc);
 1124         return (0);
 1125 }
 1126 
 1127 void
 1128 t3_fatal_err(struct adapter *sc)
 1129 {
 1130         u_int fw_status[4];
 1131 
 1132         if (sc->flags & FULL_INIT_DONE) {
 1133                 t3_sge_stop(sc);
 1134                 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
 1135                 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
 1136                 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
 1137                 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
 1138                 t3_intr_disable(sc);
 1139         }
 1140         device_printf(sc->dev,"encountered fatal error, operation suspended\n");
 1141         if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
 1142                 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
 1143                     fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
 1144 }
 1145 
 1146 int
 1147 t3_os_find_pci_capability(adapter_t *sc, int cap)
 1148 {
 1149         device_t dev;
 1150         struct pci_devinfo *dinfo;
 1151         pcicfgregs *cfg;
 1152         uint32_t status;
 1153         uint8_t ptr;
 1154 
 1155         dev = sc->dev;
 1156         dinfo = device_get_ivars(dev);
 1157         cfg = &dinfo->cfg;
 1158 
 1159         status = pci_read_config(dev, PCIR_STATUS, 2);
 1160         if (!(status & PCIM_STATUS_CAPPRESENT))
 1161                 return (0);
 1162 
 1163         switch (cfg->hdrtype & PCIM_HDRTYPE) {
 1164         case 0:
 1165         case 1:
 1166                 ptr = PCIR_CAP_PTR;
 1167                 break;
 1168         case 2:
 1169                 ptr = PCIR_CAP_PTR_2;
 1170                 break;
 1171         default:
 1172                 return (0);
 1173                 break;
 1174         }
 1175         ptr = pci_read_config(dev, ptr, 1);
 1176 
 1177         while (ptr != 0) {
 1178                 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
 1179                         return (ptr);
 1180                 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
 1181         }
 1182 
 1183         return (0);
 1184 }
 1185 
 1186 int
 1187 t3_os_pci_save_state(struct adapter *sc)
 1188 {
 1189         device_t dev;
 1190         struct pci_devinfo *dinfo;
 1191 
 1192         dev = sc->dev;
 1193         dinfo = device_get_ivars(dev);
 1194 
 1195         pci_cfg_save(dev, dinfo, 0);
 1196         return (0);
 1197 }
 1198 
 1199 int
 1200 t3_os_pci_restore_state(struct adapter *sc)
 1201 {
 1202         device_t dev;
 1203         struct pci_devinfo *dinfo;
 1204 
 1205         dev = sc->dev;
 1206         dinfo = device_get_ivars(dev);
 1207 
 1208         pci_cfg_restore(dev, dinfo);
 1209         return (0);
 1210 }
 1211 
 1212 /**
 1213  *      t3_os_link_changed - handle link status changes
 1214  *      @sc: the adapter associated with the link change
 1215  *      @port_id: the port index whose link status has changed
 1216  *      @link_status: the new status of the link
 1217  *      @speed: the new speed setting
 1218  *      @duplex: the new duplex setting
 1219  *      @fc: the new flow-control setting
 1220  *
 1221  *      This is the OS-dependent handler for link status changes.  The OS
 1222  *      neutral handler takes care of most of the processing for these events,
 1223  *      then calls this handler for any OS-specific processing.
 1224  */
 1225 void
 1226 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
 1227      int duplex, int fc, int mac_was_reset)
 1228 {
 1229         struct port_info *pi = &adapter->port[port_id];
 1230         struct ifnet *ifp = pi->ifp;
 1231 
 1232         /* no race with detach, so ifp should always be good */
 1233         KASSERT(ifp, ("%s: if detached.", __func__));
 1234 
 1235         /* Reapply mac settings if they were lost due to a reset */
 1236         if (mac_was_reset) {
 1237                 PORT_LOCK(pi);
 1238                 cxgb_update_mac_settings(pi);
 1239                 PORT_UNLOCK(pi);
 1240         }
 1241 
 1242         if (link_status) {
 1243                 ifp->if_baudrate = IF_Mbps(speed);
 1244                 if_link_state_change(ifp, LINK_STATE_UP);
 1245         } else
 1246                 if_link_state_change(ifp, LINK_STATE_DOWN);
 1247 }
 1248 
 1249 /**
 1250  *      t3_os_phymod_changed - handle PHY module changes
 1251  *      @phy: the PHY reporting the module change
 1252  *      @mod_type: new module type
 1253  *
 1254  *      This is the OS-dependent handler for PHY module changes.  It is
 1255  *      invoked when a PHY module is removed or inserted for any OS-specific
 1256  *      processing.
 1257  */
 1258 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 1259 {
 1260         static const char *mod_str[] = {
 1261                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
 1262         };
 1263         struct port_info *pi = &adap->port[port_id];
 1264         int mod = pi->phy.modtype;
 1265 
 1266         if (mod != pi->media.ifm_cur->ifm_data)
 1267                 cxgb_build_medialist(pi);
 1268 
 1269         if (mod == phy_modtype_none)
 1270                 if_printf(pi->ifp, "PHY module unplugged\n");
 1271         else {
 1272                 KASSERT(mod < ARRAY_SIZE(mod_str),
 1273                         ("invalid PHY module type %d", mod));
 1274                 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
 1275         }
 1276 }
 1277 
 1278 void
 1279 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
 1280 {
 1281 
 1282         /*
 1283          * The ifnet might not be allocated before this gets called,
 1284          * as this is called early on in attach by t3_prep_adapter
 1285          * save the address off in the port structure
 1286          */
 1287         if (cxgb_debug)
 1288                 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
 1289         bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
 1290 }
 1291 
 1292 /*
 1293  * Programs the XGMAC based on the settings in the ifnet.  These settings
 1294  * include MTU, MAC address, mcast addresses, etc.
 1295  */
 1296 static void
 1297 cxgb_update_mac_settings(struct port_info *p)
 1298 {
 1299         struct ifnet *ifp = p->ifp;
 1300         struct t3_rx_mode rm;
 1301         struct cmac *mac = &p->mac;
 1302         int mtu, hwtagging;
 1303 
 1304         PORT_LOCK_ASSERT_OWNED(p);
 1305 
 1306         bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
 1307 
 1308         mtu = ifp->if_mtu;
 1309         if (ifp->if_capenable & IFCAP_VLAN_MTU)
 1310                 mtu += ETHER_VLAN_ENCAP_LEN;
 1311 
 1312         hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
 1313 
 1314         t3_mac_set_mtu(mac, mtu);
 1315         t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
 1316         t3_mac_set_address(mac, 0, p->hw_addr);
 1317         t3_init_rx_mode(&rm, p);
 1318         t3_mac_set_rx_mode(mac, &rm);
 1319 }
 1320 
 1321 
 1322 static int
 1323 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 1324                               unsigned long n)
 1325 {
 1326         int attempts = 5;
 1327 
 1328         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 1329                 if (!--attempts)
 1330                         return (ETIMEDOUT);
 1331                 t3_os_sleep(10);
 1332         }
 1333         return 0;
 1334 }
 1335 
 1336 static int
 1337 init_tp_parity(struct adapter *adap)
 1338 {
 1339         int i;
 1340         struct mbuf *m;
 1341         struct cpl_set_tcb_field *greq;
 1342         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 1343 
 1344         t3_tp_set_offload_mode(adap, 1);
 1345 
 1346         for (i = 0; i < 16; i++) {
 1347                 struct cpl_smt_write_req *req;
 1348 
 1349                 m = m_gethdr(M_WAITOK, MT_DATA);
 1350                 req = mtod(m, struct cpl_smt_write_req *);
 1351                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1352                 memset(req, 0, sizeof(*req));
 1353                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1354                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 1355                 req->iff = i;
 1356                 t3_mgmt_tx(adap, m);
 1357         }
 1358 
 1359         for (i = 0; i < 2048; i++) {
 1360                 struct cpl_l2t_write_req *req;
 1361 
 1362                 m = m_gethdr(M_WAITOK, MT_DATA);
 1363                 req = mtod(m, struct cpl_l2t_write_req *);
 1364                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1365                 memset(req, 0, sizeof(*req));
 1366                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1367                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 1368                 req->params = htonl(V_L2T_W_IDX(i));
 1369                 t3_mgmt_tx(adap, m);
 1370         }
 1371 
 1372         for (i = 0; i < 2048; i++) {
 1373                 struct cpl_rte_write_req *req;
 1374 
 1375                 m = m_gethdr(M_WAITOK, MT_DATA);
 1376                 req = mtod(m, struct cpl_rte_write_req *);
 1377                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1378                 memset(req, 0, sizeof(*req));
 1379                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1380                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 1381                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 1382                 t3_mgmt_tx(adap, m);
 1383         }
 1384 
 1385         m = m_gethdr(M_WAITOK, MT_DATA);
 1386         greq = mtod(m, struct cpl_set_tcb_field *);
 1387         m->m_len = m->m_pkthdr.len = sizeof(*greq);
 1388         memset(greq, 0, sizeof(*greq));
 1389         greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1390         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 1391         greq->mask = htobe64(1);
 1392         t3_mgmt_tx(adap, m);
 1393 
 1394         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 1395         t3_tp_set_offload_mode(adap, 0);
 1396         return (i);
 1397 }
 1398 
 1399 /**
 1400  *      setup_rss - configure Receive Side Steering (per-queue connection demux) 
 1401  *      @adap: the adapter
 1402  *
 1403  *      Sets up RSS to distribute packets to multiple receive queues.  We
 1404  *      configure the RSS CPU lookup table to distribute to the number of HW
 1405  *      receive queues, and the response queue lookup table to narrow that
 1406  *      down to the response queues actually configured for each port.
 1407  *      We always configure the RSS mapping for two ports since the mapping
 1408  *      table has plenty of entries.
 1409  */
 1410 static void
 1411 setup_rss(adapter_t *adap)
 1412 {
 1413         int i;
 1414         u_int nq[2]; 
 1415         uint8_t cpus[SGE_QSETS + 1];
 1416         uint16_t rspq_map[RSS_TABLE_SIZE];
 1417         
 1418         for (i = 0; i < SGE_QSETS; ++i)
 1419                 cpus[i] = i;
 1420         cpus[SGE_QSETS] = 0xff;
 1421 
 1422         nq[0] = nq[1] = 0;
 1423         for_each_port(adap, i) {
 1424                 const struct port_info *pi = adap2pinfo(adap, i);
 1425 
 1426                 nq[pi->tx_chan] += pi->nqsets;
 1427         }
 1428         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 1429                 rspq_map[i] = nq[0] ? i % nq[0] : 0;
 1430                 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
 1431         }
 1432 
 1433         /* Calculate the reverse RSS map table */
 1434         for (i = 0; i < SGE_QSETS; ++i)
 1435                 adap->rrss_map[i] = 0xff;
 1436         for (i = 0; i < RSS_TABLE_SIZE; ++i)
 1437                 if (adap->rrss_map[rspq_map[i]] == 0xff)
 1438                         adap->rrss_map[rspq_map[i]] = i;
 1439 
 1440         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 1441                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
 1442                       F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
 1443                       cpus, rspq_map);
 1444 
 1445 }
 1446 static void
 1447 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 1448                               int hi, int port)
 1449 {
 1450         struct mbuf *m;
 1451         struct mngt_pktsched_wr *req;
 1452 
 1453         m = m_gethdr(M_NOWAIT, MT_DATA);
 1454         if (m) {        
 1455                 req = mtod(m, struct mngt_pktsched_wr *);
 1456                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 1457                 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 1458                 req->sched = sched;
 1459                 req->idx = qidx;
 1460                 req->min = lo;
 1461                 req->max = hi;
 1462                 req->binding = port;
 1463                 m->m_len = m->m_pkthdr.len = sizeof(*req);
 1464                 t3_mgmt_tx(adap, m);
 1465         }
 1466 }
 1467 
 1468 static void
 1469 bind_qsets(adapter_t *sc)
 1470 {
 1471         int i, j;
 1472 
 1473         for (i = 0; i < (sc)->params.nports; ++i) {
 1474                 const struct port_info *pi = adap2pinfo(sc, i);
 1475 
 1476                 for (j = 0; j < pi->nqsets; ++j) {
 1477                         send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
 1478                                           -1, pi->tx_chan);
 1479 
 1480                 }
 1481         }
 1482 }
 1483 
 1484 static void
 1485 update_tpeeprom(struct adapter *adap)
 1486 {
 1487         const struct firmware *tpeeprom;
 1488 
 1489         uint32_t version;
 1490         unsigned int major, minor;
 1491         int ret, len;
 1492         char rev, name[32];
 1493 
 1494         t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
 1495 
 1496         major = G_TP_VERSION_MAJOR(version);
 1497         minor = G_TP_VERSION_MINOR(version);
 1498         if (major == TP_VERSION_MAJOR  && minor == TP_VERSION_MINOR)
 1499                 return; 
 1500 
 1501         rev = t3rev2char(adap);
 1502         snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
 1503 
 1504         tpeeprom = firmware_get(name);
 1505         if (tpeeprom == NULL) {
 1506                 device_printf(adap->dev,
 1507                               "could not load TP EEPROM: unable to load %s\n",
 1508                               name);
 1509                 return;
 1510         }
 1511 
 1512         len = tpeeprom->datasize - 4;
 1513         
 1514         ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
 1515         if (ret)
 1516                 goto release_tpeeprom;
 1517 
 1518         if (len != TP_SRAM_LEN) {
 1519                 device_printf(adap->dev,
 1520                               "%s length is wrong len=%d expected=%d\n", name,
 1521                               len, TP_SRAM_LEN);
 1522                 return;
 1523         }
 1524         
 1525         ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
 1526             TP_SRAM_OFFSET);
 1527         
 1528         if (!ret) {
 1529                 device_printf(adap->dev,
 1530                         "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
 1531                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 1532         } else 
 1533                 device_printf(adap->dev,
 1534                               "Protocol SRAM image update in EEPROM failed\n");
 1535 
 1536 release_tpeeprom:
 1537         firmware_put(tpeeprom, FIRMWARE_UNLOAD);
 1538         
 1539         return;
 1540 }
 1541 
 1542 static int
 1543 update_tpsram(struct adapter *adap)
 1544 {
 1545         const struct firmware *tpsram;
 1546         int ret;
 1547         char rev, name[32];
 1548 
 1549         rev = t3rev2char(adap);
 1550         snprintf(name, sizeof(name), TPSRAM_NAME, rev);
 1551 
 1552         update_tpeeprom(adap);
 1553 
 1554         tpsram = firmware_get(name);
 1555         if (tpsram == NULL){
 1556                 device_printf(adap->dev, "could not load TP SRAM\n");
 1557                 return (EINVAL);
 1558         } else
 1559                 device_printf(adap->dev, "updating TP SRAM\n");
 1560         
 1561         ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
 1562         if (ret)
 1563                 goto release_tpsram;    
 1564 
 1565         ret = t3_set_proto_sram(adap, tpsram->data);
 1566         if (ret)
 1567                 device_printf(adap->dev, "loading protocol SRAM failed\n");
 1568 
 1569 release_tpsram:
 1570         firmware_put(tpsram, FIRMWARE_UNLOAD);
 1571         
 1572         return ret;
 1573 }
 1574 
 1575 /**
 1576  *      cxgb_up - enable the adapter
 1577  *      @adap: adapter being enabled
 1578  *
 1579  *      Called when the first port is enabled, this function performs the
 1580  *      actions necessary to make an adapter operational, such as completing
 1581  *      the initialization of HW modules, and enabling interrupts.
 1582  */
 1583 static int
 1584 cxgb_up(struct adapter *sc)
 1585 {
 1586         int err = 0;
 1587         unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
 1588 
 1589         KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
 1590                                            __func__, sc->open_device_map));
 1591 
 1592         if ((sc->flags & FULL_INIT_DONE) == 0) {
 1593 
 1594                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1595 
 1596                 if ((sc->flags & FW_UPTODATE) == 0)
 1597                         if ((err = upgrade_fw(sc)))
 1598                                 goto out;
 1599 
 1600                 if ((sc->flags & TPS_UPTODATE) == 0)
 1601                         if ((err = update_tpsram(sc)))
 1602                                 goto out;
 1603 
 1604                 if (is_offload(sc) && nfilters != 0) {
 1605                         sc->params.mc5.nservers = 0;
 1606 
 1607                         if (nfilters < 0)
 1608                                 sc->params.mc5.nfilters = mxf;
 1609                         else
 1610                                 sc->params.mc5.nfilters = min(nfilters, mxf);
 1611                 }
 1612 
 1613                 err = t3_init_hw(sc, 0);
 1614                 if (err)
 1615                         goto out;
 1616 
 1617                 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 1618                 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 1619 
 1620                 err = setup_sge_qsets(sc);
 1621                 if (err)
 1622                         goto out;
 1623 
 1624                 alloc_filters(sc);
 1625                 setup_rss(sc);
 1626 
 1627                 t3_add_configured_sysctls(sc);
 1628                 sc->flags |= FULL_INIT_DONE;
 1629         }
 1630 
 1631         t3_intr_clear(sc);
 1632         t3_sge_start(sc);
 1633         t3_intr_enable(sc);
 1634 
 1635         if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
 1636             is_offload(sc) && init_tp_parity(sc) == 0)
 1637                 sc->flags |= TP_PARITY_INIT;
 1638 
 1639         if (sc->flags & TP_PARITY_INIT) {
 1640                 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
 1641                 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
 1642         }
 1643         
 1644         if (!(sc->flags & QUEUES_BOUND)) {
 1645                 bind_qsets(sc);
 1646                 setup_hw_filters(sc);
 1647                 sc->flags |= QUEUES_BOUND;              
 1648         }
 1649 
 1650         t3_sge_reset_adapter(sc);
 1651 out:
 1652         return (err);
 1653 }
 1654 
 1655 /*
 1656  * Called when the last open device is closed.  Does NOT undo all of cxgb_up's
 1657  * work.  Specifically, the resources grabbed under FULL_INIT_DONE are released
 1658  * during controller_detach, not here.
 1659  */
 1660 static void
 1661 cxgb_down(struct adapter *sc)
 1662 {
 1663         t3_sge_stop(sc);
 1664         t3_intr_disable(sc);
 1665 }
 1666 
 1667 /*
 1668  * if_init for cxgb ports.
 1669  */
 1670 static void
 1671 cxgb_init(void *arg)
 1672 {
 1673         struct port_info *p = arg;
 1674         struct adapter *sc = p->adapter;
 1675 
 1676         ADAPTER_LOCK(sc);
 1677         cxgb_init_locked(p); /* releases adapter lock */
 1678         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1679 }
 1680 
 1681 static int
 1682 cxgb_init_locked(struct port_info *p)
 1683 {
 1684         struct adapter *sc = p->adapter;
 1685         struct ifnet *ifp = p->ifp;
 1686         struct cmac *mac = &p->mac;
 1687         int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
 1688 
 1689         ADAPTER_LOCK_ASSERT_OWNED(sc);
 1690 
 1691         while (!IS_DOOMED(p) && IS_BUSY(sc)) {
 1692                 gave_up_lock = 1;
 1693                 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
 1694                         rc = EINTR;
 1695                         goto done;
 1696                 }
 1697         }
 1698         if (IS_DOOMED(p)) {
 1699                 rc = ENXIO;
 1700                 goto done;
 1701         }
 1702         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
 1703 
 1704         /*
 1705          * The code that runs during one-time adapter initialization can sleep
 1706          * so it's important not to hold any locks across it.
 1707          */
 1708         may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
 1709 
 1710         if (may_sleep) {
 1711                 SET_BUSY(sc);
 1712                 gave_up_lock = 1;
 1713                 ADAPTER_UNLOCK(sc);
 1714         }
 1715 
 1716         if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
 1717                         goto done;
 1718 
 1719         PORT_LOCK(p);
 1720         if (isset(&sc->open_device_map, p->port_id) &&
 1721             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 1722                 PORT_UNLOCK(p);
 1723                 goto done;
 1724         }
 1725         t3_port_intr_enable(sc, p->port_id);
 1726         if (!mac->multiport) 
 1727                 t3_mac_init(mac);
 1728         cxgb_update_mac_settings(p);
 1729         t3_link_start(&p->phy, mac, &p->link_config);
 1730         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 1731         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 1732         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 1733         PORT_UNLOCK(p);
 1734 
 1735         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
 1736                 struct sge_qset *qs = &sc->sge.qs[i];
 1737                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
 1738 
 1739                 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
 1740                                  txq->txq_watchdog.c_cpu);
 1741         }
 1742 
 1743         /* all ok */
 1744         setbit(&sc->open_device_map, p->port_id);
 1745         callout_reset(&p->link_check_ch,
 1746             p->phy.caps & SUPPORTED_LINK_IRQ ?  hz * 3 : hz / 4,
 1747             link_check_callout, p);
 1748 
 1749 done:
 1750         if (may_sleep) {
 1751                 ADAPTER_LOCK(sc);
 1752                 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
 1753                 CLR_BUSY(sc);
 1754         }
 1755         if (gave_up_lock)
 1756                 wakeup_one(&sc->flags);
 1757         ADAPTER_UNLOCK(sc);
 1758         return (rc);
 1759 }
 1760 
 1761 static int
 1762 cxgb_uninit_locked(struct port_info *p)
 1763 {
 1764         struct adapter *sc = p->adapter;
 1765         int rc;
 1766 
 1767         ADAPTER_LOCK_ASSERT_OWNED(sc);
 1768 
 1769         while (!IS_DOOMED(p) && IS_BUSY(sc)) {
 1770                 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
 1771                         rc = EINTR;
 1772                         goto done;
 1773                 }
 1774         }
 1775         if (IS_DOOMED(p)) {
 1776                 rc = ENXIO;
 1777                 goto done;
 1778         }
 1779         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
 1780         SET_BUSY(sc);
 1781         ADAPTER_UNLOCK(sc);
 1782 
 1783         rc = cxgb_uninit_synchronized(p);
 1784 
 1785         ADAPTER_LOCK(sc);
 1786         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
 1787         CLR_BUSY(sc);
 1788         wakeup_one(&sc->flags);
 1789 done:
 1790         ADAPTER_UNLOCK(sc);
 1791         return (rc);
 1792 }
 1793 
 1794 /*
 1795  * Called on "ifconfig down", and from port_detach
 1796  */
 1797 static int
 1798 cxgb_uninit_synchronized(struct port_info *pi)
 1799 {
 1800         struct adapter *sc = pi->adapter;
 1801         struct ifnet *ifp = pi->ifp;
 1802 
 1803         /*
 1804          * taskqueue_drain may cause a deadlock if the adapter lock is held.
 1805          */
 1806         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1807 
 1808         /*
 1809          * Clear this port's bit from the open device map, and then drain all
 1810          * the tasks that can access/manipulate this port's port_info or ifp.
 1811          * We disable this port's interrupts here and so the slow/ext
 1812          * interrupt tasks won't be enqueued.  The tick task will continue to
 1813          * be enqueued every second but the runs after this drain will not see
 1814          * this port in the open device map.
 1815          *
 1816          * A well behaved task must take open_device_map into account and ignore
 1817          * ports that are not open.
 1818          */
 1819         clrbit(&sc->open_device_map, pi->port_id);
 1820         t3_port_intr_disable(sc, pi->port_id);
 1821         taskqueue_drain(sc->tq, &sc->slow_intr_task);
 1822         taskqueue_drain(sc->tq, &sc->tick_task);
 1823 
 1824         callout_drain(&pi->link_check_ch);
 1825         taskqueue_drain(sc->tq, &pi->link_check_task);
 1826 
 1827         PORT_LOCK(pi);
 1828         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 1829 
 1830         /* disable pause frames */
 1831         t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
 1832 
 1833         /* Reset RX FIFO HWM */
 1834         t3_set_reg_field(sc, A_XGM_RXFIFO_CFG +  pi->mac.offset,
 1835                          V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
 1836 
 1837         DELAY(100 * 1000);
 1838 
 1839         /* Wait for TXFIFO empty */
 1840         t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
 1841                         F_TXFIFO_EMPTY, 1, 20, 5);
 1842 
 1843         DELAY(100 * 1000);
 1844         t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
 1845 
 1846         pi->phy.ops->power_down(&pi->phy, 1);
 1847 
 1848         PORT_UNLOCK(pi);
 1849 
 1850         pi->link_config.link_ok = 0;
 1851         t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
 1852 
 1853         if (sc->open_device_map == 0)
 1854                 cxgb_down(pi->adapter);
 1855 
 1856         return (0);
 1857 }
 1858 
 1859 /*
 1860  * Mark lro enabled or disabled in all qsets for this port
 1861  */
 1862 static int
 1863 cxgb_set_lro(struct port_info *p, int enabled)
 1864 {
 1865         int i;
 1866         struct adapter *adp = p->adapter;
 1867         struct sge_qset *q;
 1868 
 1869         for (i = 0; i < p->nqsets; i++) {
 1870                 q = &adp->sge.qs[p->first_qset + i];
 1871                 q->lro.enabled = (enabled != 0);
 1872         }
 1873         return (0);
 1874 }
 1875 
 1876 static int
 1877 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 1878 {
 1879         struct port_info *p = ifp->if_softc;
 1880         struct adapter *sc = p->adapter;
 1881         struct ifreq *ifr = (struct ifreq *)data;
 1882         int flags, error = 0, mtu;
 1883         uint32_t mask;
 1884 
 1885         switch (command) {
 1886         case SIOCSIFMTU:
 1887                 ADAPTER_LOCK(sc);
 1888                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 1889                 if (error) {
 1890 fail:
 1891                         ADAPTER_UNLOCK(sc);
 1892                         return (error);
 1893                 }
 1894 
 1895                 mtu = ifr->ifr_mtu;
 1896                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
 1897                         error = EINVAL;
 1898                 } else {
 1899                         ifp->if_mtu = mtu;
 1900                         PORT_LOCK(p);
 1901                         cxgb_update_mac_settings(p);
 1902                         PORT_UNLOCK(p);
 1903                 }
 1904                 ADAPTER_UNLOCK(sc);
 1905                 break;
 1906         case SIOCSIFFLAGS:
 1907                 ADAPTER_LOCK(sc);
 1908                 if (IS_DOOMED(p)) {
 1909                         error = ENXIO;
 1910                         goto fail;
 1911                 }
 1912                 if (ifp->if_flags & IFF_UP) {
 1913                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1914                                 flags = p->if_flags;
 1915                                 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
 1916                                     ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
 1917                                         if (IS_BUSY(sc)) {
 1918                                                 error = EBUSY;
 1919                                                 goto fail;
 1920                                         }
 1921                                         PORT_LOCK(p);
 1922                                         cxgb_update_mac_settings(p);
 1923                                         PORT_UNLOCK(p);
 1924                                 }
 1925                                 ADAPTER_UNLOCK(sc);
 1926                         } else
 1927                                 error = cxgb_init_locked(p);
 1928                         p->if_flags = ifp->if_flags;
 1929                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1930                         error = cxgb_uninit_locked(p);
 1931                 else
 1932                         ADAPTER_UNLOCK(sc);
 1933 
 1934                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 1935                 break;
 1936         case SIOCADDMULTI:
 1937         case SIOCDELMULTI:
 1938                 ADAPTER_LOCK(sc);
 1939                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 1940                 if (error)
 1941                         goto fail;
 1942 
 1943                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1944                         PORT_LOCK(p);
 1945                         cxgb_update_mac_settings(p);
 1946                         PORT_UNLOCK(p);
 1947                 }
 1948                 ADAPTER_UNLOCK(sc);
 1949 
 1950                 break;
 1951         case SIOCSIFCAP:
 1952                 ADAPTER_LOCK(sc);
 1953                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 1954                 if (error)
 1955                         goto fail;
 1956 
 1957                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 1958                 if (mask & IFCAP_TXCSUM) {
 1959                         ifp->if_capenable ^= IFCAP_TXCSUM;
 1960                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
 1961 
 1962                         if (IFCAP_TSO4 & ifp->if_capenable &&
 1963                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
 1964                                 ifp->if_capenable &= ~IFCAP_TSO4;
 1965                                 if_printf(ifp,
 1966                                     "tso4 disabled due to -txcsum.\n");
 1967                         }
 1968                 }
 1969                 if (mask & IFCAP_TXCSUM_IPV6) {
 1970                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
 1971                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
 1972 
 1973                         if (IFCAP_TSO6 & ifp->if_capenable &&
 1974                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
 1975                                 ifp->if_capenable &= ~IFCAP_TSO6;
 1976                                 if_printf(ifp,
 1977                                     "tso6 disabled due to -txcsum6.\n");
 1978                         }
 1979                 }
 1980                 if (mask & IFCAP_RXCSUM)
 1981                         ifp->if_capenable ^= IFCAP_RXCSUM;
 1982                 if (mask & IFCAP_RXCSUM_IPV6)
 1983                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
 1984 
 1985                 /*
 1986                  * Note that we leave CSUM_TSO alone (it is always set).  The
 1987                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
 1988                  * sending a TSO request our way, so it's sufficient to toggle
 1989                  * IFCAP_TSOx only.
 1990                  */
 1991                 if (mask & IFCAP_TSO4) {
 1992                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
 1993                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
 1994                                 if_printf(ifp, "enable txcsum first.\n");
 1995                                 error = EAGAIN;
 1996                                 goto fail;
 1997                         }
 1998                         ifp->if_capenable ^= IFCAP_TSO4;
 1999                 }
 2000                 if (mask & IFCAP_TSO6) {
 2001                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
 2002                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
 2003                                 if_printf(ifp, "enable txcsum6 first.\n");
 2004                                 error = EAGAIN;
 2005                                 goto fail;
 2006                         }
 2007                         ifp->if_capenable ^= IFCAP_TSO6;
 2008                 }
 2009                 if (mask & IFCAP_LRO) {
 2010                         ifp->if_capenable ^= IFCAP_LRO;
 2011 
 2012                         /* Safe to do this even if cxgb_up not called yet */
 2013                         cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
 2014                 }
 2015 #ifdef TCP_OFFLOAD
 2016                 if (mask & IFCAP_TOE4) {
 2017                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
 2018 
 2019                         error = toe_capability(p, enable);
 2020                         if (error == 0)
 2021                                 ifp->if_capenable ^= mask;
 2022                 }
 2023 #endif
 2024                 if (mask & IFCAP_VLAN_HWTAGGING) {
 2025                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 2026                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2027                                 PORT_LOCK(p);
 2028                                 cxgb_update_mac_settings(p);
 2029                                 PORT_UNLOCK(p);
 2030                         }
 2031                 }
 2032                 if (mask & IFCAP_VLAN_MTU) {
 2033                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
 2034                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 2035                                 PORT_LOCK(p);
 2036                                 cxgb_update_mac_settings(p);
 2037                                 PORT_UNLOCK(p);
 2038                         }
 2039                 }
 2040                 if (mask & IFCAP_VLAN_HWTSO)
 2041                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 2042                 if (mask & IFCAP_VLAN_HWCSUM)
 2043                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
 2044 
 2045 #ifdef VLAN_CAPABILITIES
 2046                 VLAN_CAPABILITIES(ifp);
 2047 #endif
 2048                 ADAPTER_UNLOCK(sc);
 2049                 break;
 2050         case SIOCSIFMEDIA:
 2051         case SIOCGIFMEDIA:
 2052                 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
 2053                 break;
 2054         default:
 2055                 error = ether_ioctl(ifp, command, data);
 2056         }
 2057 
 2058         return (error);
 2059 }
 2060 
 2061 static int
 2062 cxgb_media_change(struct ifnet *ifp)
 2063 {
 2064         return (EOPNOTSUPP);
 2065 }
 2066 
 2067 /*
 2068  * Translates phy->modtype to the correct Ethernet media subtype.
 2069  */
 2070 static int
 2071 cxgb_ifm_type(int mod)
 2072 {
 2073         switch (mod) {
 2074         case phy_modtype_sr:
 2075                 return (IFM_10G_SR);
 2076         case phy_modtype_lr:
 2077                 return (IFM_10G_LR);
 2078         case phy_modtype_lrm:
 2079                 return (IFM_10G_LRM);
 2080         case phy_modtype_twinax:
 2081                 return (IFM_10G_TWINAX);
 2082         case phy_modtype_twinax_long:
 2083                 return (IFM_10G_TWINAX_LONG);
 2084         case phy_modtype_none:
 2085                 return (IFM_NONE);
 2086         case phy_modtype_unknown:
 2087                 return (IFM_UNKNOWN);
 2088         }
 2089 
 2090         KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
 2091         return (IFM_UNKNOWN);
 2092 }
 2093 
 2094 /*
 2095  * Rebuilds the ifmedia list for this port, and sets the current media.
 2096  */
 2097 static void
 2098 cxgb_build_medialist(struct port_info *p)
 2099 {
 2100         struct cphy *phy = &p->phy;
 2101         struct ifmedia *media = &p->media;
 2102         int mod = phy->modtype;
 2103         int m = IFM_ETHER | IFM_FDX;
 2104 
 2105         PORT_LOCK(p);
 2106 
 2107         ifmedia_removeall(media);
 2108         if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
 2109                 /* Copper (RJ45) */
 2110 
 2111                 if (phy->caps & SUPPORTED_10000baseT_Full)
 2112                         ifmedia_add(media, m | IFM_10G_T, mod, NULL);
 2113 
 2114                 if (phy->caps & SUPPORTED_1000baseT_Full)
 2115                         ifmedia_add(media, m | IFM_1000_T, mod, NULL);
 2116 
 2117                 if (phy->caps & SUPPORTED_100baseT_Full)
 2118                         ifmedia_add(media, m | IFM_100_TX, mod, NULL);
 2119 
 2120                 if (phy->caps & SUPPORTED_10baseT_Full)
 2121                         ifmedia_add(media, m | IFM_10_T, mod, NULL);
 2122 
 2123                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
 2124                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
 2125 
 2126         } else if (phy->caps & SUPPORTED_TP) {
 2127                 /* Copper (CX4) */
 2128 
 2129                 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
 2130                         ("%s: unexpected cap 0x%x", __func__, phy->caps));
 2131 
 2132                 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
 2133                 ifmedia_set(media, m | IFM_10G_CX4);
 2134 
 2135         } else if (phy->caps & SUPPORTED_FIBRE &&
 2136                    phy->caps & SUPPORTED_10000baseT_Full) {
 2137                 /* 10G optical (but includes SFP+ twinax) */
 2138 
 2139                 m |= cxgb_ifm_type(mod);
 2140                 if (IFM_SUBTYPE(m) == IFM_NONE)
 2141                         m &= ~IFM_FDX;
 2142 
 2143                 ifmedia_add(media, m, mod, NULL);
 2144                 ifmedia_set(media, m);
 2145 
 2146         } else if (phy->caps & SUPPORTED_FIBRE &&
 2147                    phy->caps & SUPPORTED_1000baseT_Full) {
 2148                 /* 1G optical */
 2149 
 2150                 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
 2151                 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
 2152                 ifmedia_set(media, m | IFM_1000_SX);
 2153 
 2154         } else {
 2155                 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
 2156                             phy->caps));
 2157         }
 2158 
 2159         PORT_UNLOCK(p);
 2160 }
 2161 
 2162 static void
 2163 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 2164 {
 2165         struct port_info *p = ifp->if_softc;
 2166         struct ifmedia_entry *cur = p->media.ifm_cur;
 2167         int speed = p->link_config.speed;
 2168 
 2169         if (cur->ifm_data != p->phy.modtype) {
 2170                 cxgb_build_medialist(p);
 2171                 cur = p->media.ifm_cur;
 2172         }
 2173 
 2174         ifmr->ifm_status = IFM_AVALID;
 2175         if (!p->link_config.link_ok)
 2176                 return;
 2177 
 2178         ifmr->ifm_status |= IFM_ACTIVE;
 2179 
 2180         /*
 2181          * active and current will differ iff current media is autoselect.  That
 2182          * can happen only for copper RJ45.
 2183          */
 2184         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
 2185                 return;
 2186         KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
 2187                 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
 2188 
 2189         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
 2190         if (speed == SPEED_10000)
 2191                 ifmr->ifm_active |= IFM_10G_T;
 2192         else if (speed == SPEED_1000)
 2193                 ifmr->ifm_active |= IFM_1000_T;
 2194         else if (speed == SPEED_100)
 2195                 ifmr->ifm_active |= IFM_100_TX;
 2196         else if (speed == SPEED_10)
 2197                 ifmr->ifm_active |= IFM_10_T;
 2198         else
 2199                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
 2200                             speed));
 2201 }
 2202 
 2203 static uint64_t
 2204 cxgb_get_counter(struct ifnet *ifp, ift_counter c)
 2205 {
 2206         struct port_info *pi = ifp->if_softc;
 2207         struct adapter *sc = pi->adapter;
 2208         struct cmac *mac = &pi->mac;
 2209         struct mac_stats *mstats = &mac->stats;
 2210 
 2211         cxgb_refresh_stats(pi);
 2212 
 2213         switch (c) {
 2214         case IFCOUNTER_IPACKETS:
 2215                 return (mstats->rx_frames);
 2216 
 2217         case IFCOUNTER_IERRORS:
 2218                 return (mstats->rx_jabber + mstats->rx_data_errs +
 2219                     mstats->rx_sequence_errs + mstats->rx_runt +
 2220                     mstats->rx_too_long + mstats->rx_mac_internal_errs +
 2221                     mstats->rx_short + mstats->rx_fcs_errs);
 2222 
 2223         case IFCOUNTER_OPACKETS:
 2224                 return (mstats->tx_frames);
 2225 
 2226         case IFCOUNTER_OERRORS:
 2227                 return (mstats->tx_excess_collisions + mstats->tx_underrun +
 2228                     mstats->tx_len_errs + mstats->tx_mac_internal_errs +
 2229                     mstats->tx_excess_deferral + mstats->tx_fcs_errs);
 2230 
 2231         case IFCOUNTER_COLLISIONS:
 2232                 return (mstats->tx_total_collisions);
 2233 
 2234         case IFCOUNTER_IBYTES:
 2235                 return (mstats->rx_octets);
 2236 
 2237         case IFCOUNTER_OBYTES:
 2238                 return (mstats->tx_octets);
 2239 
 2240         case IFCOUNTER_IMCASTS:
 2241                 return (mstats->rx_mcast_frames);
 2242 
 2243         case IFCOUNTER_OMCASTS:
 2244                 return (mstats->tx_mcast_frames);
 2245 
 2246         case IFCOUNTER_IQDROPS:
 2247                 return (mstats->rx_cong_drops);
 2248 
 2249         case IFCOUNTER_OQDROPS: {
 2250                 int i;
 2251                 uint64_t drops;
 2252 
 2253                 drops = 0;
 2254                 if (sc->flags & FULL_INIT_DONE) {
 2255                         for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
 2256                                 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
 2257                 }
 2258 
 2259                 return (drops);
 2260 
 2261         }
 2262 
 2263         default:
 2264                 return (if_get_counter_default(ifp, c));
 2265         }
 2266 }
 2267 
 2268 static void
 2269 cxgb_async_intr(void *data)
 2270 {
 2271         adapter_t *sc = data;
 2272 
 2273         t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
 2274         (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
 2275         taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
 2276 }
 2277 
 2278 static void
 2279 link_check_callout(void *arg)
 2280 {
 2281         struct port_info *pi = arg;
 2282         struct adapter *sc = pi->adapter;
 2283 
 2284         if (!isset(&sc->open_device_map, pi->port_id))
 2285                 return;
 2286 
 2287         taskqueue_enqueue(sc->tq, &pi->link_check_task);
 2288 }
 2289 
 2290 static void
 2291 check_link_status(void *arg, int pending)
 2292 {
 2293         struct port_info *pi = arg;
 2294         struct adapter *sc = pi->adapter;
 2295 
 2296         if (!isset(&sc->open_device_map, pi->port_id))
 2297                 return;
 2298 
 2299         t3_link_changed(sc, pi->port_id);
 2300 
 2301         if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
 2302             pi->link_config.link_ok == 0)
 2303                 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
 2304 }
 2305 
 2306 void
 2307 t3_os_link_intr(struct port_info *pi)
 2308 {
 2309         /*
 2310          * Schedule a link check in the near future.  If the link is flapping
 2311          * rapidly we'll keep resetting the callout and delaying the check until
 2312          * things stabilize a bit.
 2313          */
 2314         callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
 2315 }
 2316 
 2317 static void
 2318 check_t3b2_mac(struct adapter *sc)
 2319 {
 2320         int i;
 2321 
 2322         if (sc->flags & CXGB_SHUTDOWN)
 2323                 return;
 2324 
 2325         for_each_port(sc, i) {
 2326                 struct port_info *p = &sc->port[i];
 2327                 int status;
 2328 #ifdef INVARIANTS
 2329                 struct ifnet *ifp = p->ifp;
 2330 #endif          
 2331 
 2332                 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
 2333                     !p->link_config.link_ok)
 2334                         continue;
 2335 
 2336                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
 2337                         ("%s: state mismatch (drv_flags %x, device_map %x)",
 2338                          __func__, ifp->if_drv_flags, sc->open_device_map));
 2339 
 2340                 PORT_LOCK(p);
 2341                 status = t3b2_mac_watchdog_task(&p->mac);
 2342                 if (status == 1)
 2343                         p->mac.stats.num_toggled++;
 2344                 else if (status == 2) {
 2345                         struct cmac *mac = &p->mac;
 2346 
 2347                         cxgb_update_mac_settings(p);
 2348                         t3_link_start(&p->phy, mac, &p->link_config);
 2349                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 2350                         t3_port_intr_enable(sc, p->port_id);
 2351                         p->mac.stats.num_resets++;
 2352                 }
 2353                 PORT_UNLOCK(p);
 2354         }
 2355 }
 2356 
 2357 static void
 2358 cxgb_tick(void *arg)
 2359 {
 2360         adapter_t *sc = (adapter_t *)arg;
 2361 
 2362         if (sc->flags & CXGB_SHUTDOWN)
 2363                 return;
 2364 
 2365         taskqueue_enqueue(sc->tq, &sc->tick_task);      
 2366         callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
 2367 }
 2368 
 2369 void
 2370 cxgb_refresh_stats(struct port_info *pi)
 2371 {
 2372         struct timeval tv;
 2373         const struct timeval interval = {0, 250000};    /* 250ms */
 2374 
 2375         getmicrotime(&tv);
 2376         timevalsub(&tv, &interval);
 2377         if (timevalcmp(&tv, &pi->last_refreshed, <))
 2378                 return;
 2379 
 2380         PORT_LOCK(pi);
 2381         t3_mac_update_stats(&pi->mac);
 2382         PORT_UNLOCK(pi);
 2383         getmicrotime(&pi->last_refreshed);
 2384 }
 2385 
 2386 static void
 2387 cxgb_tick_handler(void *arg, int count)
 2388 {
 2389         adapter_t *sc = (adapter_t *)arg;
 2390         const struct adapter_params *p = &sc->params;
 2391         int i;
 2392         uint32_t cause, reset;
 2393 
 2394         if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
 2395                 return;
 2396 
 2397         if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 
 2398                 check_t3b2_mac(sc);
 2399 
 2400         cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
 2401         if (cause) {
 2402                 struct sge_qset *qs = &sc->sge.qs[0];
 2403                 uint32_t mask, v;
 2404 
 2405                 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
 2406 
 2407                 mask = 1;
 2408                 for (i = 0; i < SGE_QSETS; i++) {
 2409                         if (v & mask)
 2410                                 qs[i].rspq.starved++;
 2411                         mask <<= 1;
 2412                 }
 2413 
 2414                 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
 2415 
 2416                 for (i = 0; i < SGE_QSETS * 2; i++) {
 2417                         if (v & mask) {
 2418                                 qs[i / 2].fl[i % 2].empty++;
 2419                         }
 2420                         mask <<= 1;
 2421                 }
 2422 
 2423                 /* clear */
 2424                 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
 2425                 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
 2426         }
 2427 
 2428         for (i = 0; i < sc->params.nports; i++) {
 2429                 struct port_info *pi = &sc->port[i];
 2430                 struct cmac *mac = &pi->mac;
 2431 
 2432                 if (!isset(&sc->open_device_map, pi->port_id))
 2433                         continue;
 2434 
 2435                 cxgb_refresh_stats(pi);
 2436 
 2437                 if (mac->multiport)
 2438                         continue;
 2439 
 2440                 /* Count rx fifo overflows, once per second */
 2441                 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
 2442                 reset = 0;
 2443                 if (cause & F_RXFIFO_OVERFLOW) {
 2444                         mac->stats.rx_fifo_ovfl++;
 2445                         reset |= F_RXFIFO_OVERFLOW;
 2446                 }
 2447                 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
 2448         }
 2449 }
 2450 
 2451 static void
 2452 touch_bars(device_t dev)
 2453 {
 2454         /*
 2455          * Don't enable yet
 2456          */
 2457 #if !defined(__LP64__) && 0
 2458         u32 v;
 2459 
 2460         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
 2461         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
 2462         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
 2463         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
 2464         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
 2465         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
 2466 #endif
 2467 }
 2468 
 2469 static int
 2470 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
 2471 {
 2472         uint8_t *buf;
 2473         int err = 0;
 2474         u32 aligned_offset, aligned_len, *p;
 2475         struct adapter *adapter = pi->adapter;
 2476 
 2477 
 2478         aligned_offset = offset & ~3;
 2479         aligned_len = (len + (offset & 3) + 3) & ~3;
 2480 
 2481         if (aligned_offset != offset || aligned_len != len) {
 2482                 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);              
 2483                 if (!buf)
 2484                         return (ENOMEM);
 2485                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
 2486                 if (!err && aligned_len > 4)
 2487                         err = t3_seeprom_read(adapter,
 2488                                               aligned_offset + aligned_len - 4,
 2489                                               (u32 *)&buf[aligned_len - 4]);
 2490                 if (err)
 2491                         goto out;
 2492                 memcpy(buf + (offset & 3), data, len);
 2493         } else
 2494                 buf = (uint8_t *)(uintptr_t)data;
 2495 
 2496         err = t3_seeprom_wp(adapter, 0);
 2497         if (err)
 2498                 goto out;
 2499 
 2500         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
 2501                 err = t3_seeprom_write(adapter, aligned_offset, *p);
 2502                 aligned_offset += 4;
 2503         }
 2504 
 2505         if (!err)
 2506                 err = t3_seeprom_wp(adapter, 1);
 2507 out:
 2508         if (buf != data)
 2509                 free(buf, M_DEVBUF);
 2510         return err;
 2511 }
 2512 
 2513 
 2514 static int
 2515 in_range(int val, int lo, int hi)
 2516 {
 2517         return val < 0 || (val <= hi && val >= lo);
 2518 }
 2519 
 2520 static int
 2521 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
 2522 {
 2523        return (0);
 2524 }
 2525 
 2526 static int
 2527 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 2528 {
 2529        return (0);
 2530 }
 2531 
 2532 static int
 2533 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 2534     int fflag, struct thread *td)
 2535 {
 2536         int mmd, error = 0;
 2537         struct port_info *pi = dev->si_drv1;
 2538         adapter_t *sc = pi->adapter;
 2539 
 2540 #ifdef PRIV_SUPPORTED   
 2541         if (priv_check(td, PRIV_DRIVER)) {
 2542                 if (cxgb_debug) 
 2543                         printf("user does not have access to privileged ioctls\n");
 2544                 return (EPERM);
 2545         }
 2546 #else
 2547         if (suser(td)) {
 2548                 if (cxgb_debug)
 2549                         printf("user does not have access to privileged ioctls\n");
 2550                 return (EPERM);
 2551         }
 2552 #endif
 2553         
 2554         switch (cmd) {
 2555         case CHELSIO_GET_MIIREG: {
 2556                 uint32_t val;
 2557                 struct cphy *phy = &pi->phy;
 2558                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
 2559                 
 2560                 if (!phy->mdio_read)
 2561                         return (EOPNOTSUPP);
 2562                 if (is_10G(sc)) {
 2563                         mmd = mid->phy_id >> 8;
 2564                         if (!mmd)
 2565                                 mmd = MDIO_DEV_PCS;
 2566                         else if (mmd > MDIO_DEV_VEND2)
 2567                                 return (EINVAL);
 2568 
 2569                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
 2570                                              mid->reg_num, &val);
 2571                 } else
 2572                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
 2573                                              mid->reg_num & 0x1f, &val);
 2574                 if (error == 0)
 2575                         mid->val_out = val;
 2576                 break;
 2577         }
 2578         case CHELSIO_SET_MIIREG: {
 2579                 struct cphy *phy = &pi->phy;
 2580                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
 2581 
 2582                 if (!phy->mdio_write)
 2583                         return (EOPNOTSUPP);
 2584                 if (is_10G(sc)) {
 2585                         mmd = mid->phy_id >> 8;
 2586                         if (!mmd)
 2587                                 mmd = MDIO_DEV_PCS;
 2588                         else if (mmd > MDIO_DEV_VEND2)
 2589                                 return (EINVAL);
 2590                         
 2591                         error = phy->mdio_write(sc, mid->phy_id & 0x1f,
 2592                                               mmd, mid->reg_num, mid->val_in);
 2593                 } else
 2594                         error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
 2595                                               mid->reg_num & 0x1f,
 2596                                               mid->val_in);
 2597                 break;
 2598         }
 2599         case CHELSIO_SETREG: {
 2600                 struct ch_reg *edata = (struct ch_reg *)data;
 2601                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 2602                         return (EFAULT);
 2603                 t3_write_reg(sc, edata->addr, edata->val);
 2604                 break;
 2605         }
 2606         case CHELSIO_GETREG: {
 2607                 struct ch_reg *edata = (struct ch_reg *)data;
 2608                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 2609                         return (EFAULT);
 2610                 edata->val = t3_read_reg(sc, edata->addr);
 2611                 break;
 2612         }
 2613         case CHELSIO_GET_SGE_CONTEXT: {
 2614                 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
 2615                 mtx_lock_spin(&sc->sge.reg_lock);
 2616                 switch (ecntxt->cntxt_type) {
 2617                 case CNTXT_TYPE_EGRESS:
 2618                         error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
 2619                             ecntxt->data);
 2620                         break;
 2621                 case CNTXT_TYPE_FL:
 2622                         error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
 2623                             ecntxt->data);
 2624                         break;
 2625                 case CNTXT_TYPE_RSP:
 2626                         error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
 2627                             ecntxt->data);
 2628                         break;
 2629                 case CNTXT_TYPE_CQ:
 2630                         error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
 2631                             ecntxt->data);
 2632                         break;
 2633                 default:
 2634                         error = EINVAL;
 2635                         break;
 2636                 }
 2637                 mtx_unlock_spin(&sc->sge.reg_lock);
 2638                 break;
 2639         }
 2640         case CHELSIO_GET_SGE_DESC: {
 2641                 struct ch_desc *edesc = (struct ch_desc *)data;
 2642                 int ret;
 2643                 if (edesc->queue_num >= SGE_QSETS * 6)
 2644                         return (EINVAL);
 2645                 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
 2646                     edesc->queue_num % 6, edesc->idx, edesc->data);
 2647                 if (ret < 0)
 2648                         return (EINVAL);
 2649                 edesc->size = ret;
 2650                 break;
 2651         }
 2652         case CHELSIO_GET_QSET_PARAMS: {
 2653                 struct qset_params *q;
 2654                 struct ch_qset_params *t = (struct ch_qset_params *)data;
 2655                 int q1 = pi->first_qset;
 2656                 int nqsets = pi->nqsets;
 2657                 int i;
 2658 
 2659                 if (t->qset_idx >= nqsets)
 2660                         return EINVAL;
 2661 
 2662                 i = q1 + t->qset_idx;
 2663                 q = &sc->params.sge.qset[i];
 2664                 t->rspq_size   = q->rspq_size;
 2665                 t->txq_size[0] = q->txq_size[0];
 2666                 t->txq_size[1] = q->txq_size[1];
 2667                 t->txq_size[2] = q->txq_size[2];
 2668                 t->fl_size[0]  = q->fl_size;
 2669                 t->fl_size[1]  = q->jumbo_size;
 2670                 t->polling     = q->polling;
 2671                 t->lro         = q->lro;
 2672                 t->intr_lat    = q->coalesce_usecs;
 2673                 t->cong_thres  = q->cong_thres;
 2674                 t->qnum        = i;
 2675 
 2676                 if ((sc->flags & FULL_INIT_DONE) == 0)
 2677                         t->vector = 0;
 2678                 else if (sc->flags & USING_MSIX)
 2679                         t->vector = rman_get_start(sc->msix_irq_res[i]);
 2680                 else
 2681                         t->vector = rman_get_start(sc->irq_res);
 2682 
 2683                 break;
 2684         }
 2685         case CHELSIO_GET_QSET_NUM: {
 2686                 struct ch_reg *edata = (struct ch_reg *)data;
 2687                 edata->val = pi->nqsets;
 2688                 break;
 2689         }
 2690         case CHELSIO_LOAD_FW: {
 2691                 uint8_t *fw_data;
 2692                 uint32_t vers;
 2693                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2694 
 2695                 /*
 2696                  * You're allowed to load a firmware only before FULL_INIT_DONE
 2697                  *
 2698                  * FW_UPTODATE is also set so the rest of the initialization
 2699                  * will not overwrite what was loaded here.  This gives you the
 2700                  * flexibility to load any firmware (and maybe shoot yourself in
 2701                  * the foot).
 2702                  */
 2703 
 2704                 ADAPTER_LOCK(sc);
 2705                 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
 2706                         ADAPTER_UNLOCK(sc);
 2707                         return (EBUSY);
 2708                 }
 2709 
 2710                 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
 2711                 if (!fw_data)
 2712                         error = ENOMEM;
 2713                 else
 2714                         error = copyin(t->buf, fw_data, t->len);
 2715 
 2716                 if (!error)
 2717                         error = -t3_load_fw(sc, fw_data, t->len);
 2718 
 2719                 if (t3_get_fw_version(sc, &vers) == 0) {
 2720                         snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
 2721                             "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
 2722                             G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
 2723                 }
 2724 
 2725                 if (!error)
 2726                         sc->flags |= FW_UPTODATE;
 2727 
 2728                 free(fw_data, M_DEVBUF);
 2729                 ADAPTER_UNLOCK(sc);
 2730                 break;
 2731         }
 2732         case CHELSIO_LOAD_BOOT: {
 2733                 uint8_t *boot_data;
 2734                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2735 
 2736                 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
 2737                 if (!boot_data)
 2738                         return ENOMEM;
 2739 
 2740                 error = copyin(t->buf, boot_data, t->len);
 2741                 if (!error)
 2742                         error = -t3_load_boot(sc, boot_data, t->len);
 2743 
 2744                 free(boot_data, M_DEVBUF);
 2745                 break;
 2746         }
 2747         case CHELSIO_GET_PM: {
 2748                 struct ch_pm *m = (struct ch_pm *)data;
 2749                 struct tp_params *p = &sc->params.tp;
 2750 
 2751                 if (!is_offload(sc))
 2752                         return (EOPNOTSUPP);
 2753 
 2754                 m->tx_pg_sz = p->tx_pg_size;
 2755                 m->tx_num_pg = p->tx_num_pgs;
 2756                 m->rx_pg_sz  = p->rx_pg_size;
 2757                 m->rx_num_pg = p->rx_num_pgs;
 2758                 m->pm_total  = p->pmtx_size + p->chan_rx_size * p->nchan;
 2759 
 2760                 break;
 2761         }
 2762         case CHELSIO_SET_PM: {
 2763                 struct ch_pm *m = (struct ch_pm *)data;
 2764                 struct tp_params *p = &sc->params.tp;
 2765 
 2766                 if (!is_offload(sc))
 2767                         return (EOPNOTSUPP);
 2768                 if (sc->flags & FULL_INIT_DONE)
 2769                         return (EBUSY);
 2770 
 2771                 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
 2772                     !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
 2773                         return (EINVAL);        /* not power of 2 */
 2774                 if (!(m->rx_pg_sz & 0x14000))
 2775                         return (EINVAL);        /* not 16KB or 64KB */
 2776                 if (!(m->tx_pg_sz & 0x1554000))
 2777                         return (EINVAL);
 2778                 if (m->tx_num_pg == -1)
 2779                         m->tx_num_pg = p->tx_num_pgs;
 2780                 if (m->rx_num_pg == -1)
 2781                         m->rx_num_pg = p->rx_num_pgs;
 2782                 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
 2783                         return (EINVAL);
 2784                 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
 2785                     m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
 2786                         return (EINVAL);
 2787 
 2788                 p->rx_pg_size = m->rx_pg_sz;
 2789                 p->tx_pg_size = m->tx_pg_sz;
 2790                 p->rx_num_pgs = m->rx_num_pg;
 2791                 p->tx_num_pgs = m->tx_num_pg;
 2792                 break;
 2793         }
 2794         case CHELSIO_SETMTUTAB: {
 2795                 struct ch_mtus *m = (struct ch_mtus *)data;
 2796                 int i;
 2797                 
 2798                 if (!is_offload(sc))
 2799                         return (EOPNOTSUPP);
 2800                 if (offload_running(sc))
 2801                         return (EBUSY);
 2802                 if (m->nmtus != NMTUS)
 2803                         return (EINVAL);
 2804                 if (m->mtus[0] < 81)         /* accommodate SACK */
 2805                         return (EINVAL);
 2806                 
 2807                 /*
 2808                  * MTUs must be in ascending order
 2809                  */
 2810                 for (i = 1; i < NMTUS; ++i)
 2811                         if (m->mtus[i] < m->mtus[i - 1])
 2812                                 return (EINVAL);
 2813 
 2814                 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
 2815                 break;
 2816         }
 2817         case CHELSIO_GETMTUTAB: {
 2818                 struct ch_mtus *m = (struct ch_mtus *)data;
 2819 
 2820                 if (!is_offload(sc))
 2821                         return (EOPNOTSUPP);
 2822 
 2823                 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
 2824                 m->nmtus = NMTUS;
 2825                 break;
 2826         }
 2827         case CHELSIO_GET_MEM: {
 2828                 struct ch_mem_range *t = (struct ch_mem_range *)data;
 2829                 struct mc7 *mem;
 2830                 uint8_t *useraddr;
 2831                 u64 buf[32];
 2832 
 2833                 /*
 2834                  * Use these to avoid modifying len/addr in the return
 2835                  * struct
 2836                  */
 2837                 uint32_t len = t->len, addr = t->addr;
 2838 
 2839                 if (!is_offload(sc))
 2840                         return (EOPNOTSUPP);
 2841                 if (!(sc->flags & FULL_INIT_DONE))
 2842                         return (EIO);         /* need the memory controllers */
 2843                 if ((addr & 0x7) || (len & 0x7))
 2844                         return (EINVAL);
 2845                 if (t->mem_id == MEM_CM)
 2846                         mem = &sc->cm;
 2847                 else if (t->mem_id == MEM_PMRX)
 2848                         mem = &sc->pmrx;
 2849                 else if (t->mem_id == MEM_PMTX)
 2850                         mem = &sc->pmtx;
 2851                 else
 2852                         return (EINVAL);
 2853 
 2854                 /*
 2855                  * Version scheme:
 2856                  * bits 0..9: chip version
 2857                  * bits 10..15: chip revision
 2858                  */
 2859                 t->version = 3 | (sc->params.rev << 10);
 2860                 
 2861                 /*
 2862                  * Read 256 bytes at a time as len can be large and we don't
 2863                  * want to use huge intermediate buffers.
 2864                  */
 2865                 useraddr = (uint8_t *)t->buf; 
 2866                 while (len) {
 2867                         unsigned int chunk = min(len, sizeof(buf));
 2868 
 2869                         error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
 2870                         if (error)
 2871                                 return (-error);
 2872                         if (copyout(buf, useraddr, chunk))
 2873                                 return (EFAULT);
 2874                         useraddr += chunk;
 2875                         addr += chunk;
 2876                         len -= chunk;
 2877                 }
 2878                 break;
 2879         }
 2880         case CHELSIO_READ_TCAM_WORD: {
 2881                 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
 2882 
 2883                 if (!is_offload(sc))
 2884                         return (EOPNOTSUPP);
 2885                 if (!(sc->flags & FULL_INIT_DONE))
 2886                         return (EIO);         /* need MC5 */            
 2887                 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
 2888                 break;
 2889         }
 2890         case CHELSIO_SET_TRACE_FILTER: {
 2891                 struct ch_trace *t = (struct ch_trace *)data;
 2892                 const struct trace_params *tp;
 2893 
 2894                 tp = (const struct trace_params *)&t->sip;
 2895                 if (t->config_tx)
 2896                         t3_config_trace_filter(sc, tp, 0, t->invert_match,
 2897                                                t->trace_tx);
 2898                 if (t->config_rx)
 2899                         t3_config_trace_filter(sc, tp, 1, t->invert_match,
 2900                                                t->trace_rx);
 2901                 break;
 2902         }
 2903         case CHELSIO_SET_PKTSCHED: {
 2904                 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
 2905                 if (sc->open_device_map == 0)
 2906                         return (EAGAIN);
 2907                 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
 2908                     p->binding);
 2909                 break;
 2910         }
 2911         case CHELSIO_IFCONF_GETREGS: {
 2912                 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
 2913                 int reglen = cxgb_get_regs_len();
 2914                 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
 2915                 if (buf == NULL) {
 2916                         return (ENOMEM);
 2917                 }
 2918                 if (regs->len > reglen)
 2919                         regs->len = reglen;
 2920                 else if (regs->len < reglen)
 2921                         error = ENOBUFS;
 2922 
 2923                 if (!error) {
 2924                         cxgb_get_regs(sc, regs, buf);
 2925                         error = copyout(buf, regs->data, reglen);
 2926                 }
 2927                 free(buf, M_DEVBUF);
 2928 
 2929                 break;
 2930         }
 2931         case CHELSIO_SET_HW_SCHED: {
 2932                 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
 2933                 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
 2934 
 2935                 if ((sc->flags & FULL_INIT_DONE) == 0)
 2936                         return (EAGAIN);       /* need TP to be initialized */
 2937                 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
 2938                     !in_range(t->channel, 0, 1) ||
 2939                     !in_range(t->kbps, 0, 10000000) ||
 2940                     !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
 2941                     !in_range(t->flow_ipg, 0,
 2942                               dack_ticks_to_usec(sc, 0x7ff)))
 2943                         return (EINVAL);
 2944 
 2945                 if (t->kbps >= 0) {
 2946                         error = t3_config_sched(sc, t->kbps, t->sched);
 2947                         if (error < 0)
 2948                                 return (-error);
 2949                 }
 2950                 if (t->class_ipg >= 0)
 2951                         t3_set_sched_ipg(sc, t->sched, t->class_ipg);
 2952                 if (t->flow_ipg >= 0) {
 2953                         t->flow_ipg *= 1000;     /* us -> ns */
 2954                         t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
 2955                 }
 2956                 if (t->mode >= 0) {
 2957                         int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
 2958 
 2959                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
 2960                                          bit, t->mode ? bit : 0);
 2961                 }
 2962                 if (t->channel >= 0)
 2963                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
 2964                                          1 << t->sched, t->channel << t->sched);
 2965                 break;
 2966         }
 2967         case CHELSIO_GET_EEPROM: {
 2968                 int i;
 2969                 struct ch_eeprom *e = (struct ch_eeprom *)data;
 2970                 uint8_t *buf;
 2971 
 2972                 if (e->offset & 3 || e->offset >= EEPROMSIZE ||
 2973                     e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
 2974                         return (EINVAL);
 2975                 }
 2976 
 2977                 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
 2978                 if (buf == NULL) {
 2979                         return (ENOMEM);
 2980                 }
 2981                 e->magic = EEPROM_MAGIC;
 2982                 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
 2983                         error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
 2984 
 2985                 if (!error)
 2986                         error = copyout(buf + e->offset, e->data, e->len);
 2987 
 2988                 free(buf, M_DEVBUF);
 2989                 break;
 2990         }
 2991         case CHELSIO_CLEAR_STATS: {
 2992                 if (!(sc->flags & FULL_INIT_DONE))
 2993                         return EAGAIN;
 2994 
 2995                 PORT_LOCK(pi);
 2996                 t3_mac_update_stats(&pi->mac);
 2997                 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
 2998                 PORT_UNLOCK(pi);
 2999                 break;
 3000         }
 3001         case CHELSIO_GET_UP_LA: {
 3002                 struct ch_up_la *la = (struct ch_up_la *)data;
 3003                 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
 3004                 if (buf == NULL) {
 3005                         return (ENOMEM);
 3006                 }
 3007                 if (la->bufsize < LA_BUFSIZE)
 3008                         error = ENOBUFS;
 3009 
 3010                 if (!error)
 3011                         error = -t3_get_up_la(sc, &la->stopped, &la->idx,
 3012                                               &la->bufsize, buf);
 3013                 if (!error)
 3014                         error = copyout(buf, la->data, la->bufsize);
 3015 
 3016                 free(buf, M_DEVBUF);
 3017                 break;
 3018         }
 3019         case CHELSIO_GET_UP_IOQS: {
 3020                 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
 3021                 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
 3022                 uint32_t *v;
 3023 
 3024                 if (buf == NULL) {
 3025                         return (ENOMEM);
 3026                 }
 3027                 if (ioqs->bufsize < IOQS_BUFSIZE)
 3028                         error = ENOBUFS;
 3029 
 3030                 if (!error)
 3031                         error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
 3032 
 3033                 if (!error) {
 3034                         v = (uint32_t *)buf;
 3035 
 3036                         ioqs->ioq_rx_enable = *v++;
 3037                         ioqs->ioq_tx_enable = *v++;
 3038                         ioqs->ioq_rx_status = *v++;
 3039                         ioqs->ioq_tx_status = *v++;
 3040 
 3041                         error = copyout(v, ioqs->data, ioqs->bufsize);
 3042                 }
 3043 
 3044                 free(buf, M_DEVBUF);
 3045                 break;
 3046         }
 3047         case CHELSIO_SET_FILTER: {
 3048                 struct ch_filter *f = (struct ch_filter *)data;
 3049                 struct filter_info *p;
 3050                 unsigned int nfilters = sc->params.mc5.nfilters;
 3051 
 3052                 if (!is_offload(sc))
 3053                         return (EOPNOTSUPP);    /* No TCAM */
 3054                 if (!(sc->flags & FULL_INIT_DONE))
 3055                         return (EAGAIN);        /* mc5 not setup yet */
 3056                 if (nfilters == 0)
 3057                         return (EBUSY);         /* TOE will use TCAM */
 3058 
 3059                 /* sanity checks */
 3060                 if (f->filter_id >= nfilters ||
 3061                     (f->val.dip && f->mask.dip != 0xffffffff) ||
 3062                     (f->val.sport && f->mask.sport != 0xffff) ||
 3063                     (f->val.dport && f->mask.dport != 0xffff) ||
 3064                     (f->val.vlan && f->mask.vlan != 0xfff) ||
 3065                     (f->val.vlan_prio &&
 3066                         f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
 3067                     (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
 3068                     f->qset >= SGE_QSETS ||
 3069                     sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
 3070                         return (EINVAL);
 3071 
 3072                 /* Was allocated with M_WAITOK */
 3073                 KASSERT(sc->filters, ("filter table NULL\n"));
 3074 
 3075                 p = &sc->filters[f->filter_id];
 3076                 if (p->locked)
 3077                         return (EPERM);
 3078 
 3079                 bzero(p, sizeof(*p));
 3080                 p->sip = f->val.sip;
 3081                 p->sip_mask = f->mask.sip;
 3082                 p->dip = f->val.dip;
 3083                 p->sport = f->val.sport;
 3084                 p->dport = f->val.dport;
 3085                 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
 3086                 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
 3087                     FILTER_NO_VLAN_PRI;
 3088                 p->mac_hit = f->mac_hit;
 3089                 p->mac_vld = f->mac_addr_idx != 0xffff;
 3090                 p->mac_idx = f->mac_addr_idx;
 3091                 p->pkt_type = f->proto;
 3092                 p->report_filter_id = f->want_filter_id;
 3093                 p->pass = f->pass;
 3094                 p->rss = f->rss;
 3095                 p->qset = f->qset;
 3096 
 3097                 error = set_filter(sc, f->filter_id, p);
 3098                 if (error == 0)
 3099                         p->valid = 1;
 3100                 break;
 3101         }
 3102         case CHELSIO_DEL_FILTER: {
 3103                 struct ch_filter *f = (struct ch_filter *)data;
 3104                 struct filter_info *p;
 3105                 unsigned int nfilters = sc->params.mc5.nfilters;
 3106 
 3107                 if (!is_offload(sc))
 3108                         return (EOPNOTSUPP);
 3109                 if (!(sc->flags & FULL_INIT_DONE))
 3110                         return (EAGAIN);
 3111                 if (nfilters == 0 || sc->filters == NULL)
 3112                         return (EINVAL);
 3113                 if (f->filter_id >= nfilters)
 3114                        return (EINVAL);
 3115 
 3116                 p = &sc->filters[f->filter_id];
 3117                 if (p->locked)
 3118                         return (EPERM);
 3119                 if (!p->valid)
 3120                         return (EFAULT); /* Read "Bad address" as "Bad index" */
 3121 
 3122                 bzero(p, sizeof(*p));
 3123                 p->sip = p->sip_mask = 0xffffffff;
 3124                 p->vlan = 0xfff;
 3125                 p->vlan_prio = FILTER_NO_VLAN_PRI;
 3126                 p->pkt_type = 1;
 3127                 error = set_filter(sc, f->filter_id, p);
 3128                 break;
 3129         }
 3130         case CHELSIO_GET_FILTER: {
 3131                 struct ch_filter *f = (struct ch_filter *)data;
 3132                 struct filter_info *p;
 3133                 unsigned int i, nfilters = sc->params.mc5.nfilters;
 3134 
 3135                 if (!is_offload(sc))
 3136                         return (EOPNOTSUPP);
 3137                 if (!(sc->flags & FULL_INIT_DONE))
 3138                         return (EAGAIN);
 3139                 if (nfilters == 0 || sc->filters == NULL)
 3140                         return (EINVAL);
 3141 
 3142                 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
 3143                 for (; i < nfilters; i++) {
 3144                         p = &sc->filters[i];
 3145                         if (!p->valid)
 3146                                 continue;
 3147 
 3148                         bzero(f, sizeof(*f));
 3149 
 3150                         f->filter_id = i;
 3151                         f->val.sip = p->sip;
 3152                         f->mask.sip = p->sip_mask;
 3153                         f->val.dip = p->dip;
 3154                         f->mask.dip = p->dip ? 0xffffffff : 0;
 3155                         f->val.sport = p->sport;
 3156                         f->mask.sport = p->sport ? 0xffff : 0;
 3157                         f->val.dport = p->dport;
 3158                         f->mask.dport = p->dport ? 0xffff : 0;
 3159                         f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
 3160                         f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
 3161                         f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
 3162                             0 : p->vlan_prio;
 3163                         f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
 3164                             0 : FILTER_NO_VLAN_PRI;
 3165                         f->mac_hit = p->mac_hit;
 3166                         f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
 3167                         f->proto = p->pkt_type;
 3168                         f->want_filter_id = p->report_filter_id;
 3169                         f->pass = p->pass;
 3170                         f->rss = p->rss;
 3171                         f->qset = p->qset;
 3172 
 3173                         break;
 3174                 }
 3175                 
 3176                 if (i == nfilters)
 3177                         f->filter_id = 0xffffffff;
 3178                 break;
 3179         }
 3180         default:
 3181                 return (EOPNOTSUPP);
 3182                 break;
 3183         }
 3184 
 3185         return (error);
 3186 }
 3187 
 3188 static __inline void
 3189 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
 3190     unsigned int end)
 3191 {
 3192         uint32_t *p = (uint32_t *)(buf + start);
 3193 
 3194         for ( ; start <= end; start += sizeof(uint32_t))
 3195                 *p++ = t3_read_reg(ap, start);
 3196 }
 3197 
 3198 #define T3_REGMAP_SIZE (3 * 1024)
 3199 static int
 3200 cxgb_get_regs_len(void)
 3201 {
 3202         return T3_REGMAP_SIZE;
 3203 }
 3204 
 3205 static void
 3206 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
 3207 {           
 3208         
 3209         /*
 3210          * Version scheme:
 3211          * bits 0..9: chip version
 3212          * bits 10..15: chip revision
 3213          * bit 31: set for PCIe cards
 3214          */
 3215         regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
 3216 
 3217         /*
 3218          * We skip the MAC statistics registers because they are clear-on-read.
 3219          * Also reading multi-register stats would need to synchronize with the
 3220          * periodic mac stats accumulation.  Hard to justify the complexity.
 3221          */
 3222         memset(buf, 0, cxgb_get_regs_len());
 3223         reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
 3224         reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
 3225         reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
 3226         reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
 3227         reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
 3228         reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
 3229                        XGM_REG(A_XGM_SERDES_STAT3, 1));
 3230         reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
 3231                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
 3232 }
 3233 
 3234 static int
 3235 alloc_filters(struct adapter *sc)
 3236 {
 3237         struct filter_info *p;
 3238         unsigned int nfilters = sc->params.mc5.nfilters;
 3239 
 3240         if (nfilters == 0)
 3241                 return (0);
 3242 
 3243         p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
 3244         sc->filters = p;
 3245 
 3246         p = &sc->filters[nfilters - 1];
 3247         p->vlan = 0xfff;
 3248         p->vlan_prio = FILTER_NO_VLAN_PRI;
 3249         p->pass = p->rss = p->valid = p->locked = 1;
 3250 
 3251         return (0);
 3252 }
 3253 
 3254 static int
 3255 setup_hw_filters(struct adapter *sc)
 3256 {
 3257         int i, rc;
 3258         unsigned int nfilters = sc->params.mc5.nfilters;
 3259 
 3260         if (!sc->filters)
 3261                 return (0);
 3262 
 3263         t3_enable_filters(sc);
 3264 
 3265         for (i = rc = 0; i < nfilters && !rc; i++) {
 3266                 if (sc->filters[i].locked)
 3267                         rc = set_filter(sc, i, &sc->filters[i]);
 3268         }
 3269 
 3270         return (rc);
 3271 }
 3272 
 3273 static int
 3274 set_filter(struct adapter *sc, int id, const struct filter_info *f)
 3275 {
 3276         int len;
 3277         struct mbuf *m;
 3278         struct ulp_txpkt *txpkt;
 3279         struct work_request_hdr *wr;
 3280         struct cpl_pass_open_req *oreq;
 3281         struct cpl_set_tcb_field *sreq;
 3282 
 3283         len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
 3284         KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
 3285 
 3286         id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
 3287               sc->params.mc5.nfilters;
 3288 
 3289         m = m_gethdr(M_WAITOK, MT_DATA);
 3290         m->m_len = m->m_pkthdr.len = len;
 3291         bzero(mtod(m, char *), len);
 3292 
 3293         wr = mtod(m, struct work_request_hdr *);
 3294         wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
 3295 
 3296         oreq = (struct cpl_pass_open_req *)(wr + 1);
 3297         txpkt = (struct ulp_txpkt *)oreq;
 3298         txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
 3299         txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
 3300         OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
 3301         oreq->local_port = htons(f->dport);
 3302         oreq->peer_port = htons(f->sport);
 3303         oreq->local_ip = htonl(f->dip);
 3304         oreq->peer_ip = htonl(f->sip);
 3305         oreq->peer_netmask = htonl(f->sip_mask);
 3306         oreq->opt0h = 0;
 3307         oreq->opt0l = htonl(F_NO_OFFLOAD);
 3308         oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
 3309                          V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
 3310                          V_VLAN_PRI(f->vlan_prio >> 1) |
 3311                          V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
 3312                          V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
 3313                          V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
 3314 
 3315         sreq = (struct cpl_set_tcb_field *)(oreq + 1);
 3316         set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
 3317                           (f->report_filter_id << 15) | (1 << 23) |
 3318                           ((u64)f->pass << 35) | ((u64)!f->rss << 36));
 3319         set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
 3320         t3_mgmt_tx(sc, m);
 3321 
 3322         if (f->pass && !f->rss) {
 3323                 len = sizeof(*sreq);
 3324                 m = m_gethdr(M_WAITOK, MT_DATA);
 3325                 m->m_len = m->m_pkthdr.len = len;
 3326                 bzero(mtod(m, char *), len);
 3327                 sreq = mtod(m, struct cpl_set_tcb_field *);
 3328                 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 3329                 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
 3330                                  (u64)sc->rrss_map[f->qset] << 19);
 3331                 t3_mgmt_tx(sc, m);
 3332         }
 3333         return 0;
 3334 }
 3335 
 3336 static inline void
 3337 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
 3338     unsigned int word, u64 mask, u64 val)
 3339 {
 3340         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
 3341         req->reply = V_NO_REPLY(1);
 3342         req->cpu_idx = 0;
 3343         req->word = htons(word);
 3344         req->mask = htobe64(mask);
 3345         req->val = htobe64(val);
 3346 }
 3347 
 3348 static inline void
 3349 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
 3350     unsigned int word, u64 mask, u64 val)
 3351 {
 3352         struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
 3353 
 3354         txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
 3355         txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
 3356         mk_set_tcb_field(req, tid, word, mask, val);
 3357 }
 3358 
 3359 void
 3360 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
 3361 {
 3362         struct adapter *sc;
 3363 
 3364         mtx_lock(&t3_list_lock);
 3365         SLIST_FOREACH(sc, &t3_list, link) {
 3366                 /*
 3367                  * func should not make any assumptions about what state sc is
 3368                  * in - the only guarantee is that sc->sc_lock is a valid lock.
 3369                  */
 3370                 func(sc, arg);
 3371         }
 3372         mtx_unlock(&t3_list_lock);
 3373 }
 3374 
 3375 #ifdef TCP_OFFLOAD
 3376 static int
 3377 toe_capability(struct port_info *pi, int enable)
 3378 {
 3379         int rc;
 3380         struct adapter *sc = pi->adapter;
 3381 
 3382         ADAPTER_LOCK_ASSERT_OWNED(sc);
 3383 
 3384         if (!is_offload(sc))
 3385                 return (ENODEV);
 3386 
 3387         if (enable) {
 3388                 if (!(sc->flags & FULL_INIT_DONE)) {
 3389                         log(LOG_WARNING,
 3390                             "You must enable a cxgb interface first\n");
 3391                         return (EAGAIN);
 3392                 }
 3393 
 3394                 if (isset(&sc->offload_map, pi->port_id))
 3395                         return (0);
 3396 
 3397                 if (!(sc->flags & TOM_INIT_DONE)) {
 3398                         rc = t3_activate_uld(sc, ULD_TOM);
 3399                         if (rc == EAGAIN) {
 3400                                 log(LOG_WARNING,
 3401                                     "You must kldload t3_tom.ko before trying "
 3402                                     "to enable TOE on a cxgb interface.\n");
 3403                         }
 3404                         if (rc != 0)
 3405                                 return (rc);
 3406                         KASSERT(sc->tom_softc != NULL,
 3407                             ("%s: TOM activated but softc NULL", __func__));
 3408                         KASSERT(sc->flags & TOM_INIT_DONE,
 3409                             ("%s: TOM activated but flag not set", __func__));
 3410                 }
 3411 
 3412                 setbit(&sc->offload_map, pi->port_id);
 3413 
 3414                 /*
 3415                  * XXX: Temporary code to allow iWARP to be enabled when TOE is
 3416                  * enabled on any port.  Need to figure out how to enable,
 3417                  * disable, load, and unload iWARP cleanly.
 3418                  */
 3419                 if (!isset(&sc->offload_map, MAX_NPORTS) &&
 3420                     t3_activate_uld(sc, ULD_IWARP) == 0)
 3421                         setbit(&sc->offload_map, MAX_NPORTS);
 3422         } else {
 3423                 if (!isset(&sc->offload_map, pi->port_id))
 3424                         return (0);
 3425 
 3426                 KASSERT(sc->flags & TOM_INIT_DONE,
 3427                     ("%s: TOM never initialized?", __func__));
 3428                 clrbit(&sc->offload_map, pi->port_id);
 3429         }
 3430 
 3431         return (0);
 3432 }
 3433 
 3434 /*
 3435  * Add an upper layer driver to the global list.
 3436  */
 3437 int
 3438 t3_register_uld(struct uld_info *ui)
 3439 {
 3440         int rc = 0;
 3441         struct uld_info *u;
 3442 
 3443         mtx_lock(&t3_uld_list_lock);
 3444         SLIST_FOREACH(u, &t3_uld_list, link) {
 3445             if (u->uld_id == ui->uld_id) {
 3446                     rc = EEXIST;
 3447                     goto done;
 3448             }
 3449         }
 3450 
 3451         SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
 3452         ui->refcount = 0;
 3453 done:
 3454         mtx_unlock(&t3_uld_list_lock);
 3455         return (rc);
 3456 }
 3457 
 3458 int
 3459 t3_unregister_uld(struct uld_info *ui)
 3460 {
 3461         int rc = EINVAL;
 3462         struct uld_info *u;
 3463 
 3464         mtx_lock(&t3_uld_list_lock);
 3465 
 3466         SLIST_FOREACH(u, &t3_uld_list, link) {
 3467             if (u == ui) {
 3468                     if (ui->refcount > 0) {
 3469                             rc = EBUSY;
 3470                             goto done;
 3471                     }
 3472 
 3473                     SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
 3474                     rc = 0;
 3475                     goto done;
 3476             }
 3477         }
 3478 done:
 3479         mtx_unlock(&t3_uld_list_lock);
 3480         return (rc);
 3481 }
 3482 
 3483 int
 3484 t3_activate_uld(struct adapter *sc, int id)
 3485 {
 3486         int rc = EAGAIN;
 3487         struct uld_info *ui;
 3488 
 3489         mtx_lock(&t3_uld_list_lock);
 3490 
 3491         SLIST_FOREACH(ui, &t3_uld_list, link) {
 3492                 if (ui->uld_id == id) {
 3493                         rc = ui->activate(sc);
 3494                         if (rc == 0)
 3495                                 ui->refcount++;
 3496                         goto done;
 3497                 }
 3498         }
 3499 done:
 3500         mtx_unlock(&t3_uld_list_lock);
 3501 
 3502         return (rc);
 3503 }
 3504 
 3505 int
 3506 t3_deactivate_uld(struct adapter *sc, int id)
 3507 {
 3508         int rc = EINVAL;
 3509         struct uld_info *ui;
 3510 
 3511         mtx_lock(&t3_uld_list_lock);
 3512 
 3513         SLIST_FOREACH(ui, &t3_uld_list, link) {
 3514                 if (ui->uld_id == id) {
 3515                         rc = ui->deactivate(sc);
 3516                         if (rc == 0)
 3517                                 ui->refcount--;
 3518                         goto done;
 3519                 }
 3520         }
 3521 done:
 3522         mtx_unlock(&t3_uld_list_lock);
 3523 
 3524         return (rc);
 3525 }
 3526 
 3527 static int
 3528 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
 3529     struct mbuf *m)
 3530 {
 3531         m_freem(m);
 3532         return (EDOOFUS);
 3533 }
 3534 
 3535 int
 3536 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
 3537 {
 3538         uintptr_t *loc, new;
 3539 
 3540         if (opcode >= NUM_CPL_HANDLERS)
 3541                 return (EINVAL);
 3542 
 3543         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
 3544         loc = (uintptr_t *) &sc->cpl_handler[opcode];
 3545         atomic_store_rel_ptr(loc, new);
 3546 
 3547         return (0);
 3548 }
 3549 #endif
 3550 
 3551 static int
 3552 cxgbc_mod_event(module_t mod, int cmd, void *arg)
 3553 {
 3554         int rc = 0;
 3555 
 3556         switch (cmd) {
 3557         case MOD_LOAD:
 3558                 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
 3559                 SLIST_INIT(&t3_list);
 3560 #ifdef TCP_OFFLOAD
 3561                 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
 3562                 SLIST_INIT(&t3_uld_list);
 3563 #endif
 3564                 break;
 3565 
 3566         case MOD_UNLOAD:
 3567 #ifdef TCP_OFFLOAD
 3568                 mtx_lock(&t3_uld_list_lock);
 3569                 if (!SLIST_EMPTY(&t3_uld_list)) {
 3570                         rc = EBUSY;
 3571                         mtx_unlock(&t3_uld_list_lock);
 3572                         break;
 3573                 }
 3574                 mtx_unlock(&t3_uld_list_lock);
 3575                 mtx_destroy(&t3_uld_list_lock);
 3576 #endif
 3577                 mtx_lock(&t3_list_lock);
 3578                 if (!SLIST_EMPTY(&t3_list)) {
 3579                         rc = EBUSY;
 3580                         mtx_unlock(&t3_list_lock);
 3581                         break;
 3582                 }
 3583                 mtx_unlock(&t3_list_lock);
 3584                 mtx_destroy(&t3_list_lock);
 3585                 break;
 3586         }
 3587 
 3588         return (rc);
 3589 }
 3590 
 3591 #ifdef NETDUMP
 3592 static void
 3593 cxgb_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize)
 3594 {
 3595         struct port_info *pi;
 3596         adapter_t *adap;
 3597 
 3598         pi = if_getsoftc(ifp);
 3599         adap = pi->adapter;
 3600         ADAPTER_LOCK(adap);
 3601         *nrxr = SGE_QSETS;
 3602         *ncl = adap->sge.qs[0].fl[1].size;
 3603         *clsize = adap->sge.qs[0].fl[1].buf_size;
 3604         ADAPTER_UNLOCK(adap);
 3605 }
 3606 
 3607 static void
 3608 cxgb_netdump_event(struct ifnet *ifp, enum netdump_ev event)
 3609 {
 3610         struct port_info *pi;
 3611         struct sge_qset *qs;
 3612         int i;
 3613 
 3614         pi = if_getsoftc(ifp);
 3615         if (event == NETDUMP_START)
 3616                 for (i = 0; i < SGE_QSETS; i++) {
 3617                         qs = &pi->adapter->sge.qs[i];
 3618 
 3619                         /* Need to reinit after netdump_mbuf_dump(). */
 3620                         qs->fl[0].zone = zone_pack;
 3621                         qs->fl[1].zone = zone_clust;
 3622                         qs->lro.enabled = 0;
 3623                 }
 3624 }
 3625 
 3626 static int
 3627 cxgb_netdump_transmit(struct ifnet *ifp, struct mbuf *m)
 3628 {
 3629         struct port_info *pi;
 3630         struct sge_qset *qs;
 3631 
 3632         pi = if_getsoftc(ifp);
 3633         if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 3634             IFF_DRV_RUNNING)
 3635                 return (ENOENT);
 3636 
 3637         qs = &pi->adapter->sge.qs[pi->first_qset];
 3638         return (cxgb_netdump_encap(qs, &m));
 3639 }
 3640 
 3641 static int
 3642 cxgb_netdump_poll(struct ifnet *ifp, int count)
 3643 {
 3644         struct port_info *pi;
 3645         adapter_t *adap;
 3646         int i;
 3647 
 3648         pi = if_getsoftc(ifp);
 3649         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
 3650                 return (ENOENT);
 3651 
 3652         adap = pi->adapter;
 3653         for (i = 0; i < SGE_QSETS; i++)
 3654                 (void)cxgb_netdump_poll_rx(adap, &adap->sge.qs[i]);
 3655         (void)cxgb_netdump_poll_tx(&adap->sge.qs[pi->first_qset]);
 3656         return (0);
 3657 }
 3658 #endif /* NETDUMP */

Cache object: 25ad933266186b114b46cf84758f12e5


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.