The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/cxgb/cxgb_offload.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 
    2 /**************************************************************************
    3 
    4 Copyright (c) 2007, Chelsio Inc.
    5 All rights reserved.
    6 
    7 Redistribution and use in source and binary forms, with or without
    8 modification, are permitted provided that the following conditions are met:
    9 
   10  1. Redistributions of source code must retain the above copyright notice,
   11     this list of conditions and the following disclaimer.
   12 
   13  2. Neither the name of the Chelsio Corporation nor the names of its
   14     contributors may be used to endorse or promote products derived from
   15     this software without specific prior written permission.
   16 
   17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   27 POSSIBILITY OF SUCH DAMAGE.
   28 
   29 
   30 ***************************************************************************/
   31 
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD: releng/6.4/sys/dev/cxgb/cxgb_offload.c 171884 2007-08-18 09:10:26Z kmacy $");
   35 
   36 #include <sys/param.h>
   37 #include <sys/systm.h>
   38 #include <sys/kernel.h>
   39 #include <sys/bus.h>
   40 #include <sys/module.h>
   41 #include <sys/pciio.h>
   42 #include <sys/conf.h>
   43 #include <machine/bus.h>
   44 #include <machine/resource.h>
   45 #include <sys/bus_dma.h>
   46 #include <sys/rman.h>
   47 #include <sys/ioccom.h>
   48 #include <sys/mbuf.h>
   49 #include <sys/linker.h>
   50 #include <sys/firmware.h>
   51 #include <sys/socket.h>
   52 #include <sys/sockio.h>
   53 #include <sys/smp.h>
   54 #include <sys/sysctl.h>
   55 #include <sys/queue.h>
   56 #include <sys/taskqueue.h>
   57 
   58 #ifdef CONFIG_DEFINED
   59 #include <cxgb_include.h>
   60 #else
   61 #include <dev/cxgb/cxgb_include.h>
   62 #endif
   63 
   64 #include <net/if_vlan_var.h>
   65 #include <net/route.h>
   66 
   67 /*
   68  * XXX 
   69  */
   70 #define LOG_NOTICE 2
   71 #define BUG_ON(...)
   72 #define VALIDATE_TID 0
   73 
   74 
   75 TAILQ_HEAD(, cxgb_client) client_list;
   76 TAILQ_HEAD(, toedev) ofld_dev_list;
   77 TAILQ_HEAD(, adapter) adapter_list;
   78 
   79 static struct mtx cxgb_db_lock;
   80 static struct rwlock adapter_list_lock;
   81 
   82 
   83 static const unsigned int MAX_ATIDS = 64 * 1024;
   84 static const unsigned int ATID_BASE = 0x100000;
   85 static int inited = 0;
   86 
   87 static inline int
   88 offload_activated(struct toedev *tdev)
   89 {
   90         struct adapter *adapter = tdev2adap(tdev);
   91         
   92         return (isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT));
   93 }
   94 
   95 /**
   96  *      cxgb_register_client - register an offload client
   97  *      @client: the client
   98  *
   99  *      Add the client to the client list,
  100  *      and call backs the client for each activated offload device
  101  */
  102 void
  103 cxgb_register_client(struct cxgb_client *client)
  104 {
  105         struct toedev *tdev;
  106 
  107         mtx_lock(&cxgb_db_lock);
  108         TAILQ_INSERT_TAIL(&client_list, client, client_entry);
  109 
  110         if (client->add) {
  111                 TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) {
  112                         if (offload_activated(tdev))
  113                                 client->add(tdev);
  114                 }
  115         }
  116         mtx_unlock(&cxgb_db_lock);
  117 }
  118 
  119 /**
  120  *      cxgb_unregister_client - unregister an offload client
  121  *      @client: the client
  122  *
  123  *      Remove the client to the client list,
  124  *      and call backs the client for each activated offload device.
  125  */
  126 void
  127 cxgb_unregister_client(struct cxgb_client *client)
  128 {
  129         struct toedev *tdev;
  130 
  131         mtx_lock(&cxgb_db_lock);
  132         TAILQ_REMOVE(&client_list, client, client_entry);
  133 
  134         if (client->remove) {
  135                 TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) {
  136                         if (offload_activated(tdev))
  137                                 client->remove(tdev);
  138                 }
  139         }
  140         mtx_unlock(&cxgb_db_lock);
  141 }
  142 
  143 /**
  144  *      cxgb_add_clients - activate register clients for an offload device
  145  *      @tdev: the offload device
  146  *
  147  *      Call backs all registered clients once a offload device is activated 
  148  */
  149 void
  150 cxgb_add_clients(struct toedev *tdev)
  151 {
  152         struct cxgb_client *client;
  153 
  154         mtx_lock(&cxgb_db_lock);
  155         TAILQ_FOREACH(client, &client_list, client_entry) {
  156                 if (client->add)
  157                         client->add(tdev);
  158         }
  159         mtx_unlock(&cxgb_db_lock);
  160 }
  161 
  162 /**
  163  *      cxgb_remove_clients - activate register clients for an offload device
  164  *      @tdev: the offload device
  165  *
  166  *      Call backs all registered clients once a offload device is deactivated 
  167  */
  168 void
  169 cxgb_remove_clients(struct toedev *tdev)
  170 {
  171         struct cxgb_client *client;
  172 
  173         mtx_lock(&cxgb_db_lock);
  174         TAILQ_FOREACH(client, &client_list, client_entry) {
  175                 if (client->remove)
  176                         client->remove(tdev);
  177         }
  178         mtx_unlock(&cxgb_db_lock);
  179 }
  180 
  181 static int
  182 is_offloading(struct ifnet *ifp)
  183 {
  184         struct adapter *adapter;
  185         int port;
  186 
  187         rw_rlock(&adapter_list_lock);
  188         TAILQ_FOREACH(adapter, &adapter_list, adapter_entry) {
  189                 for_each_port(adapter, port) {
  190                         if (ifp == adapter->port[port].ifp) {
  191                                 rw_runlock(&adapter_list_lock);
  192                                 return 1;
  193                         }
  194                 }
  195         }
  196         rw_runlock(&adapter_list_lock);
  197         return 0;
  198 }
  199 
  200 static struct ifnet *
  201 get_iff_from_mac(adapter_t *adapter, const uint8_t *mac, unsigned int vlan)
  202 {
  203 #ifdef notyet   
  204         int i;
  205 
  206         for_each_port(adapter, i) {
  207                 const struct vlan_group *grp;
  208                 const struct port_info *p = &adapter->port[i];
  209                 struct ifnet *ifnet = p->ifp;
  210 
  211                 if (!memcmp(p->hw_addr, mac, ETHER_ADDR_LEN)) {
  212                         if (vlan && vlan != EVL_VLID_MASK) {
  213                                 grp = p->vlan_grp;
  214                                 dev = grp ? grp->vlan_devices[vlan] : NULL;
  215                         } else
  216                                 while (dev->master)
  217                                         dev = dev->master;
  218                         return dev;
  219                 }
  220         }
  221 #endif  
  222         return NULL;
  223 }
  224 
  225 static inline void
  226 failover_fixup(adapter_t *adapter, int port)
  227 {
  228         if (adapter->params.rev == 0) {
  229                 struct ifnet *ifp = adapter->port[port].ifp;
  230                 struct cmac *mac = &adapter->port[port].mac;
  231                 if (!(ifp->if_flags & IFF_UP)) {
  232                         /* Failover triggered by the interface ifdown */
  233                         t3_write_reg(adapter, A_XGM_TX_CTRL + mac->offset,
  234                                      F_TXEN);
  235                         t3_read_reg(adapter, A_XGM_TX_CTRL + mac->offset);
  236                 } else {
  237                         /* Failover triggered by the interface link down */
  238                         t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0);
  239                         t3_read_reg(adapter, A_XGM_RX_CTRL + mac->offset);
  240                         t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset,
  241                                      F_RXEN);
  242                 }
  243         }
  244 }
  245 
  246 static int
  247 cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data)
  248 {
  249         int ret = 0;
  250         struct ulp_iscsi_info *uiip = data;
  251 
  252         switch (req) {
  253         case ULP_ISCSI_GET_PARAMS:
  254                 uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT);
  255                 uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT);
  256                 uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK);
  257                 /*
  258                  * On tx, the iscsi pdu has to be <= tx page size and has to
  259                  * fit into the Tx PM FIFO.
  260                  */
  261                 uiip->max_txsz = min(adapter->params.tp.tx_pg_size,
  262                                      t3_read_reg(adapter, A_PM1_TX_CFG) >> 17);
  263                 /* on rx, the iscsi pdu has to be < rx page size and the
  264                    whole pdu + cpl headers has to fit into one sge buffer */
  265                 uiip->max_rxsz =
  266                     (unsigned int)min(adapter->params.tp.rx_pg_size,
  267                         (adapter->sge.qs[0].fl[1].buf_size -
  268                             sizeof(struct cpl_rx_data) * 2 -
  269                             sizeof(struct cpl_rx_data_ddp)) );
  270                 break;
  271         case ULP_ISCSI_SET_PARAMS:
  272                 t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask);
  273                 break;
  274         default:
  275                 ret = (EOPNOTSUPP);
  276         }
  277         return ret;
  278 }
  279 
  280 /* Response queue used for RDMA events. */
  281 #define ASYNC_NOTIF_RSPQ 0
  282 
  283 static int
  284 cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data)
  285 {
  286         int ret = 0;
  287 
  288         switch (req) {
  289         case RDMA_GET_PARAMS: {
  290                 struct rdma_info *req = data;
  291 
  292                 req->udbell_physbase = rman_get_start(adapter->regs_res);
  293                 req->udbell_len = rman_get_size(adapter->regs_res);
  294                 req->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT);
  295                 req->tpt_top  = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT);
  296                 req->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT);
  297                 req->pbl_top  = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT);
  298                 req->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT);
  299                 req->rqt_top  = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT);
  300                 req->kdb_addr = (void *)(rman_get_start(adapter->regs_res) + A_SG_KDOORBELL);
  301                 break;
  302         }
  303         case RDMA_CQ_OP: {
  304                 struct rdma_cq_op *req = data;
  305 
  306                 /* may be called in any context */
  307                 mtx_lock(&adapter->sge.reg_lock);
  308                 ret = t3_sge_cqcntxt_op(adapter, req->id, req->op,
  309                                         req->credits);
  310                 mtx_unlock(&adapter->sge.reg_lock);
  311                 break;
  312         }
  313         case RDMA_GET_MEM: {
  314                 struct ch_mem_range *t = data;
  315                 struct mc7 *mem;
  316 
  317                 if ((t->addr & 7) || (t->len & 7))
  318                         return (EINVAL);
  319                 if (t->mem_id == MEM_CM)
  320                         mem = &adapter->cm;
  321                 else if (t->mem_id == MEM_PMRX)
  322                         mem = &adapter->pmrx;
  323                 else if (t->mem_id == MEM_PMTX)
  324                         mem = &adapter->pmtx;
  325                 else
  326                         return (EINVAL);
  327 
  328                 ret = t3_mc7_bd_read(mem, t->addr/8, t->len/8, (u64 *)t->buf);
  329                 if (ret)
  330                         return (ret);
  331                 break;
  332         }
  333         case RDMA_CQ_SETUP: {
  334                 struct rdma_cq_setup *req = data;
  335 
  336                 mtx_lock(&adapter->sge.reg_lock);
  337                 ret = t3_sge_init_cqcntxt(adapter, req->id, req->base_addr,
  338                                           req->size, ASYNC_NOTIF_RSPQ,
  339                                           req->ovfl_mode, req->credits,
  340                                           req->credit_thres);
  341                 mtx_unlock(&adapter->sge.reg_lock);
  342                 break;
  343         }
  344         case RDMA_CQ_DISABLE:
  345                 mtx_lock(&adapter->sge.reg_lock);
  346                 ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data);
  347                 mtx_unlock(&adapter->sge.reg_lock);
  348                 break;
  349         case RDMA_CTRL_QP_SETUP: {
  350                 struct rdma_ctrlqp_setup *req = data;
  351 
  352                 mtx_lock(&adapter->sge.reg_lock);
  353                 ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0,
  354                                          SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ,
  355                                          req->base_addr, req->size,
  356                                          FW_RI_TID_START, 1, 0);
  357                 mtx_unlock(&adapter->sge.reg_lock);
  358                 break;
  359         }
  360         default:
  361                 ret = EOPNOTSUPP;
  362         }
  363         return (ret);
  364 }
  365 
  366 static int
  367 cxgb_offload_ctl(struct toedev *tdev, unsigned int req, void *data)
  368 {
  369         struct adapter *adapter = tdev2adap(tdev);
  370         struct tid_range *tid;
  371         struct mtutab *mtup;
  372         struct iff_mac *iffmacp;
  373         struct ddp_params *ddpp;
  374         struct adap_ports *ports;
  375         int port;
  376 
  377         switch (req) {
  378         case GET_MAX_OUTSTANDING_WR:
  379                 *(unsigned int *)data = FW_WR_NUM;
  380                 break;
  381         case GET_WR_LEN:
  382                 *(unsigned int *)data = WR_FLITS;
  383                 break;
  384         case GET_TX_MAX_CHUNK:
  385                 *(unsigned int *)data = 1 << 20;  /* 1MB */
  386                 break;
  387         case GET_TID_RANGE:
  388                 tid = data;
  389                 tid->num = t3_mc5_size(&adapter->mc5) -
  390                         adapter->params.mc5.nroutes -
  391                         adapter->params.mc5.nfilters -
  392                         adapter->params.mc5.nservers;
  393                 tid->base = 0;
  394                 break;
  395         case GET_STID_RANGE:
  396                 tid = data;
  397                 tid->num = adapter->params.mc5.nservers;
  398                 tid->base = t3_mc5_size(&adapter->mc5) - tid->num -
  399                         adapter->params.mc5.nfilters -
  400                         adapter->params.mc5.nroutes;
  401                 break;
  402         case GET_L2T_CAPACITY:
  403                 *(unsigned int *)data = 2048;
  404                 break;
  405         case GET_MTUS:
  406                 mtup = data;
  407                 mtup->size = NMTUS;
  408                 mtup->mtus = adapter->params.mtus;
  409                 break;
  410         case GET_IFF_FROM_MAC:
  411                 iffmacp = data;
  412                 iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr,
  413                                           iffmacp->vlan_tag & EVL_VLID_MASK);
  414                 break;
  415         case GET_DDP_PARAMS:
  416                 ddpp = data;
  417                 ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT);
  418                 ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT);
  419                 ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK);
  420                 break;
  421         case GET_PORTS:
  422                 ports = data;
  423                 ports->nports   = adapter->params.nports;
  424                 for_each_port(adapter, port)
  425                         ports->lldevs[port] = adapter->port[port].ifp;
  426                 break;
  427         case FAILOVER:
  428                 port = *(int *)data;
  429                 t3_port_failover(adapter, port);
  430                 failover_fixup(adapter, port);
  431                 break;
  432         case FAILOVER_DONE:
  433                 port = *(int *)data;
  434                 t3_failover_done(adapter, port);
  435                 break;
  436         case FAILOVER_CLEAR:
  437                 t3_failover_clear(adapter);
  438                 break;
  439         case ULP_ISCSI_GET_PARAMS:
  440         case ULP_ISCSI_SET_PARAMS:
  441                 if (!offload_running(adapter))
  442                         return (EAGAIN);
  443                 return cxgb_ulp_iscsi_ctl(adapter, req, data);
  444         case RDMA_GET_PARAMS:
  445         case RDMA_CQ_OP:
  446         case RDMA_CQ_SETUP:
  447         case RDMA_CQ_DISABLE:
  448         case RDMA_CTRL_QP_SETUP:
  449         case RDMA_GET_MEM:
  450                 if (!offload_running(adapter))
  451                         return (EAGAIN);
  452                 return cxgb_rdma_ctl(adapter, req, data);
  453         default:
  454                 return (EOPNOTSUPP);
  455         }
  456         return 0;
  457 }
  458 
  459 /*
  460  * Dummy handler for Rx offload packets in case we get an offload packet before
  461  * proper processing is setup.  This complains and drops the packet as it isn't
  462  * normal to get offload packets at this stage.
  463  */
  464 static int
  465 rx_offload_blackhole(struct toedev *dev, struct mbuf **m, int n)
  466 {
  467         CH_ERR(tdev2adap(dev), "%d unexpected offload packets, first data 0x%x\n",
  468             n, *mtod(m[0], uint32_t *));
  469         while (n--)
  470                 m_freem(m[n]);
  471         return 0;
  472 }
  473 
  474 static void
  475 dummy_neigh_update(struct toedev *dev, struct rtentry *neigh)
  476 {
  477 }
  478 
  479 void
  480 cxgb_set_dummy_ops(struct toedev *dev)
  481 {
  482         dev->recv         = rx_offload_blackhole;
  483         dev->neigh_update = dummy_neigh_update;
  484 }
  485 
  486 /*
  487  * Free an active-open TID.
  488  */
  489 void *
  490 cxgb_free_atid(struct toedev *tdev, int atid)
  491 {
  492         struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
  493         union active_open_entry *p = atid2entry(t, atid);
  494         void *ctx = p->toe_tid.ctx;
  495 
  496         mtx_lock(&t->atid_lock);
  497         p->next = t->afree;
  498         t->afree = p;
  499         t->atids_in_use--;
  500         mtx_lock(&t->atid_lock);
  501 
  502         return ctx;
  503 }
  504 
  505 /*
  506  * Free a server TID and return it to the free pool.
  507  */
  508 void
  509 cxgb_free_stid(struct toedev *tdev, int stid)
  510 {
  511         struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
  512         union listen_entry *p = stid2entry(t, stid);
  513 
  514         mtx_lock(&t->stid_lock);
  515         p->next = t->sfree;
  516         t->sfree = p;
  517         t->stids_in_use--;
  518         mtx_unlock(&t->stid_lock);
  519 }
  520 
  521 void
  522 cxgb_insert_tid(struct toedev *tdev, struct cxgb_client *client,
  523         void *ctx, unsigned int tid)
  524 {
  525         struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
  526 
  527         t->tid_tab[tid].client = client;
  528         t->tid_tab[tid].ctx = ctx;
  529         atomic_add_int(&t->tids_in_use, 1);
  530 }
  531 
  532 /*
  533  * Populate a TID_RELEASE WR.  The mbuf must be already propely sized.
  534  */
  535 static inline void
  536 mk_tid_release(struct mbuf *m, unsigned int tid)
  537 {
  538         struct cpl_tid_release *req;
  539 
  540         m_set_priority(m, CPL_PRIORITY_SETUP);
  541         req = mtod(m, struct cpl_tid_release *);
  542         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
  543         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
  544 }
  545 
  546 static void
  547 t3_process_tid_release_list(void *data, int pending)
  548 {
  549         struct mbuf *m;
  550         struct toedev *tdev = data;
  551         struct toe_data *td = TOE_DATA(tdev);
  552 
  553         mtx_lock(&td->tid_release_lock);
  554         while (td->tid_release_list) {
  555                 struct toe_tid_entry *p = td->tid_release_list;
  556 
  557                 td->tid_release_list = (struct toe_tid_entry *)p->ctx;
  558                 mtx_unlock(&td->tid_release_lock);
  559                 m = m_get(M_WAIT, MT_DATA);
  560                 mk_tid_release(m, p - td->tid_maps.tid_tab);
  561                 cxgb_ofld_send(tdev, m);
  562                 p->ctx = NULL;
  563                 mtx_lock(&td->tid_release_lock);
  564         }
  565         mtx_unlock(&td->tid_release_lock);
  566 }
  567 
  568 /* use ctx as a next pointer in the tid release list */
  569 void
  570 cxgb_queue_tid_release(struct toedev *tdev, unsigned int tid)
  571 {
  572         struct toe_data *td = TOE_DATA(tdev);
  573         struct toe_tid_entry *p = &td->tid_maps.tid_tab[tid];
  574 
  575         mtx_lock(&td->tid_release_lock);
  576         p->ctx = td->tid_release_list;
  577         td->tid_release_list = p;
  578 
  579         if (!p->ctx)
  580                 taskqueue_enqueue(tdev->adapter->tq, &td->tid_release_task);
  581 
  582         mtx_unlock(&td->tid_release_lock);
  583 }
  584 
  585 /*
  586  * Remove a tid from the TID table.  A client may defer processing its last
  587  * CPL message if it is locked at the time it arrives, and while the message
  588  * sits in the client's backlog the TID may be reused for another connection.
  589  * To handle this we atomically switch the TID association if it still points
  590  * to the original client context.
  591  */
  592 void
  593 cxgb_remove_tid(struct toedev *tdev, void *ctx, unsigned int tid)
  594 {
  595         struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
  596 
  597         BUG_ON(tid >= t->ntids);
  598         if (tdev->type == T3A)
  599                 atomic_cmpset_ptr((uintptr_t *)&t->tid_tab[tid].ctx, (long)NULL, (long)ctx);
  600         else {
  601                 struct mbuf *m;
  602 
  603                 m = m_get(M_NOWAIT, MT_DATA);
  604                 if (__predict_true(m != NULL)) {
  605                         mk_tid_release(m, tid);
  606                         cxgb_ofld_send(tdev, m);
  607                         t->tid_tab[tid].ctx = NULL;
  608                 } else
  609                         cxgb_queue_tid_release(tdev, tid);
  610         }
  611         atomic_add_int(&t->tids_in_use, -1);
  612 }
  613 
  614 int
  615 cxgb_alloc_atid(struct toedev *tdev, struct cxgb_client *client,
  616                      void *ctx)
  617 {
  618         int atid = -1;
  619         struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
  620 
  621         mtx_lock(&t->atid_lock);
  622         if (t->afree) {
  623                 union active_open_entry *p = t->afree;
  624 
  625                 atid = (p - t->atid_tab) + t->atid_base;
  626                 t->afree = p->next;
  627                 p->toe_tid.ctx = ctx;
  628                 p->toe_tid.client = client;
  629                 t->atids_in_use++;
  630         }
  631         mtx_unlock(&t->atid_lock);
  632         return atid;
  633 }
  634 
  635 int
  636 cxgb_alloc_stid(struct toedev *tdev, struct cxgb_client *client,
  637                      void *ctx)
  638 {
  639         int stid = -1;
  640         struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
  641 
  642         mtx_lock(&t->stid_lock);
  643         if (t->sfree) {
  644                 union listen_entry *p = t->sfree;
  645 
  646                 stid = (p - t->stid_tab) + t->stid_base;
  647                 t->sfree = p->next;
  648                 p->toe_tid.ctx = ctx;
  649                 p->toe_tid.client = client;
  650                 t->stids_in_use++;
  651         }
  652         mtx_unlock(&t->stid_lock);
  653         return stid;
  654 }
  655 
  656 static int
  657 do_smt_write_rpl(struct toedev *dev, struct mbuf *m)
  658 {
  659         struct cpl_smt_write_rpl *rpl = cplhdr(m);
  660 
  661         if (rpl->status != CPL_ERR_NONE)
  662                 log(LOG_ERR,
  663                        "Unexpected SMT_WRITE_RPL status %u for entry %u\n",
  664                        rpl->status, GET_TID(rpl));
  665 
  666         return CPL_RET_BUF_DONE;
  667 }
  668 
  669 static int
  670 do_l2t_write_rpl(struct toedev *dev, struct mbuf *m)
  671 {
  672         struct cpl_l2t_write_rpl *rpl = cplhdr(m);
  673 
  674         if (rpl->status != CPL_ERR_NONE)
  675                 log(LOG_ERR,
  676                        "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
  677                        rpl->status, GET_TID(rpl));
  678 
  679         return CPL_RET_BUF_DONE;
  680 }
  681 
  682 static int
  683 do_act_open_rpl(struct toedev *dev, struct mbuf *m)
  684 {
  685         struct cpl_act_open_rpl *rpl = cplhdr(m);
  686         unsigned int atid = G_TID(ntohl(rpl->atid));
  687         struct toe_tid_entry *toe_tid;
  688 
  689         toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid);
  690         if (toe_tid->ctx && toe_tid->client && toe_tid->client->handlers &&
  691                 toe_tid->client->handlers[CPL_ACT_OPEN_RPL]) {
  692                 return toe_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, m,
  693                         toe_tid->ctx);
  694         } else {
  695                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
  696                         dev->name, CPL_ACT_OPEN_RPL);
  697                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
  698         }
  699 }
  700 
  701 static int
  702 do_stid_rpl(struct toedev *dev, struct mbuf *m)
  703 {
  704         union opcode_tid *p = cplhdr(m);
  705         unsigned int stid = G_TID(ntohl(p->opcode_tid));
  706         struct toe_tid_entry *toe_tid;
  707 
  708         toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid);
  709         if (toe_tid->ctx && toe_tid->client->handlers &&
  710                 toe_tid->client->handlers[p->opcode]) {
  711                 return toe_tid->client->handlers[p->opcode] (dev, m, toe_tid->ctx);
  712         } else {
  713                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
  714                         dev->name, p->opcode);
  715                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
  716         }
  717 }
  718 
  719 static int
  720 do_hwtid_rpl(struct toedev *dev, struct mbuf *m)
  721 {
  722         union opcode_tid *p = cplhdr(m);
  723         unsigned int hwtid;
  724         struct toe_tid_entry *toe_tid;
  725         
  726         printf("do_hwtid_rpl m=%p\n", m);
  727         return (0);
  728         
  729         
  730         hwtid = G_TID(ntohl(p->opcode_tid));
  731 
  732         toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid);
  733         if (toe_tid->ctx && toe_tid->client->handlers &&
  734                 toe_tid->client->handlers[p->opcode]) {
  735                 return toe_tid->client->handlers[p->opcode]
  736                                                 (dev, m, toe_tid->ctx);
  737         } else {
  738                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
  739                         dev->name, p->opcode);
  740                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
  741         }
  742 }
  743 
  744 static int
  745 do_cr(struct toedev *dev, struct mbuf *m)
  746 {
  747         struct cpl_pass_accept_req *req = cplhdr(m);
  748         unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
  749         struct toe_tid_entry *toe_tid;
  750 
  751         toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid);
  752         if (toe_tid->ctx && toe_tid->client->handlers &&
  753                 toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) {
  754                 return toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]
  755                                                 (dev, m, toe_tid->ctx);
  756         } else {
  757                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
  758                         dev->name, CPL_PASS_ACCEPT_REQ);
  759                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
  760         }
  761 }
  762 
  763 static int
  764 do_abort_req_rss(struct toedev *dev, struct mbuf *m)
  765 {
  766         union opcode_tid *p = cplhdr(m);
  767         unsigned int hwtid = G_TID(ntohl(p->opcode_tid));
  768         struct toe_tid_entry *toe_tid;
  769 
  770         toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid);
  771         if (toe_tid->ctx && toe_tid->client->handlers &&
  772                 toe_tid->client->handlers[p->opcode]) {
  773                 return toe_tid->client->handlers[p->opcode]
  774                                                 (dev, m, toe_tid->ctx);
  775         } else {
  776                 struct cpl_abort_req_rss *req = cplhdr(m);
  777                 struct cpl_abort_rpl *rpl;
  778                 
  779                 struct mbuf *m = m_get(M_NOWAIT, MT_DATA);
  780                 if (!m) {
  781                         log(LOG_NOTICE, "do_abort_req_rss: couldn't get mbuf!\n");
  782                         goto out;
  783                 }
  784 
  785                 m_set_priority(m, CPL_PRIORITY_DATA);
  786 #if 0   
  787                 __skb_put(skb, sizeof(struct cpl_abort_rpl));
  788 #endif          
  789                 rpl = cplhdr(m);
  790                 rpl->wr.wr_hi = 
  791                         htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
  792                 rpl->wr.wr_lo = htonl(V_WR_TID(GET_TID(req)));
  793                 OPCODE_TID(rpl) =
  794                         htonl(MK_OPCODE_TID(CPL_ABORT_RPL, GET_TID(req)));
  795                 rpl->cmd = req->status;
  796                 cxgb_ofld_send(dev, m);
  797  out:
  798                 return CPL_RET_BUF_DONE;
  799         }
  800 }
  801 
  802 static int
  803 do_act_establish(struct toedev *dev, struct mbuf *m)
  804 {
  805         struct cpl_act_establish *req = cplhdr(m);
  806         unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
  807         struct toe_tid_entry *toe_tid;
  808 
  809         toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid);
  810         if (toe_tid->ctx && toe_tid->client->handlers &&
  811                 toe_tid->client->handlers[CPL_ACT_ESTABLISH]) {
  812                 return toe_tid->client->handlers[CPL_ACT_ESTABLISH]
  813                                                 (dev, m, toe_tid->ctx);
  814         } else {
  815                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
  816                         dev->name, CPL_PASS_ACCEPT_REQ);
  817                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
  818         }
  819 }
  820 
  821 static int
  822 do_set_tcb_rpl(struct toedev *dev, struct mbuf *m)
  823 {
  824         struct cpl_set_tcb_rpl *rpl = cplhdr(m);
  825 
  826         if (rpl->status != CPL_ERR_NONE)
  827                 log(LOG_ERR,
  828                     "Unexpected SET_TCB_RPL status %u for tid %u\n",
  829                         rpl->status, GET_TID(rpl));
  830         return CPL_RET_BUF_DONE;
  831 }
  832 
  833 static int
  834 do_trace(struct toedev *dev, struct mbuf *m)
  835 {
  836 #if 0
  837         struct cpl_trace_pkt *p = cplhdr(m);
  838 
  839 
  840         skb->protocol = 0xffff;
  841         skb->dev = dev->lldev;
  842         skb_pull(skb, sizeof(*p));
  843         skb->mac.raw = mtod(m, (char *));
  844         netif_receive_skb(skb);
  845 #endif  
  846         return 0;
  847 }
  848 
  849 static int
  850 do_term(struct toedev *dev, struct mbuf *m)
  851 {
  852         unsigned int hwtid = ntohl(m_get_priority(m)) >> 8 & 0xfffff;
  853         unsigned int opcode = G_OPCODE(ntohl(m->m_pkthdr.csum_data));
  854         struct toe_tid_entry *toe_tid;
  855 
  856         toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid);
  857         if (toe_tid->ctx && toe_tid->client->handlers &&
  858                 toe_tid->client->handlers[opcode]) {
  859                 return toe_tid->client->handlers[opcode](dev, m, toe_tid->ctx);
  860         } else {
  861                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
  862                         dev->name, opcode);
  863                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
  864         }
  865         return (0);
  866 }
  867 
  868 #if defined(FOO)
  869 #include <linux/config.h> 
  870 #include <linux/kallsyms.h> 
  871 #include <linux/kprobes.h> 
  872 #include <net/arp.h> 
  873 
  874 static int (*orig_arp_constructor)(struct ifnet *);
  875 
  876 static void
  877 neigh_suspect(struct ifnet *neigh)
  878 {
  879         struct hh_cache *hh;
  880 
  881         neigh->output = neigh->ops->output;
  882 
  883         for (hh = neigh->hh; hh; hh = hh->hh_next)
  884                 hh->hh_output = neigh->ops->output;
  885 }
  886 
  887 static void
  888 neigh_connect(struct ifnet *neigh)
  889 {
  890         struct hh_cache *hh;
  891 
  892         neigh->output = neigh->ops->connected_output;
  893 
  894         for (hh = neigh->hh; hh; hh = hh->hh_next)
  895                 hh->hh_output = neigh->ops->hh_output;
  896 }
  897 
  898 static inline int
  899 neigh_max_probes(const struct neighbour *n)
  900 {
  901         const struct neigh_parms *p = n->parms;
  902         return (n->nud_state & NUD_PROBE ?
  903                 p->ucast_probes :
  904                 p->ucast_probes + p->app_probes + p->mcast_probes);
  905 }
  906 
  907 static void
  908 neigh_timer_handler_offload(unsigned long arg)
  909 {
  910         unsigned long now, next;
  911         struct neighbour *neigh = (struct neighbour *)arg;
  912         unsigned state;
  913         int notify = 0;
  914 
  915         write_lock(&neigh->lock);
  916 
  917         state = neigh->nud_state;
  918         now = jiffies;
  919         next = now + HZ;
  920 
  921         if (!(state & NUD_IN_TIMER)) {
  922 #ifndef CONFIG_SMP
  923                 log(LOG_WARNING, "neigh: timer & !nud_in_timer\n");
  924 #endif
  925                 goto out;
  926         }
  927 
  928         if (state & NUD_REACHABLE) {
  929                 if (time_before_eq(now,
  930                                    neigh->confirmed + 
  931                                    neigh->parms->reachable_time)) {
  932                         next = neigh->confirmed + neigh->parms->reachable_time;
  933                 } else if (time_before_eq(now,
  934                                           neigh->used + 
  935                                           neigh->parms->delay_probe_time)) {
  936                         neigh->nud_state = NUD_DELAY;
  937                         neigh->updated = jiffies;
  938                         neigh_suspect(neigh);
  939                         next = now + neigh->parms->delay_probe_time;
  940                 } else {
  941                         neigh->nud_state = NUD_STALE;
  942                         neigh->updated = jiffies;
  943                         neigh_suspect(neigh);
  944                         cxgb_neigh_update(neigh);
  945                 }
  946         } else if (state & NUD_DELAY) {
  947                 if (time_before_eq(now,
  948                                    neigh->confirmed +
  949                                    neigh->parms->delay_probe_time)) {
  950                         neigh->nud_state = NUD_REACHABLE;
  951                         neigh->updated = jiffies;
  952                         neigh_connect(neigh);
  953                         cxgb_neigh_update(neigh);
  954                         next = neigh->confirmed + neigh->parms->reachable_time;
  955                 } else {
  956                         neigh->nud_state = NUD_PROBE;
  957                         neigh->updated = jiffies;
  958                         atomic_set_int(&neigh->probes, 0);
  959                         next = now + neigh->parms->retrans_time;
  960                 }
  961         } else {
  962                 /* NUD_PROBE|NUD_INCOMPLETE */
  963                 next = now + neigh->parms->retrans_time;
  964         }
  965         /*
  966          * Needed for read of probes
  967          */
  968         mb();
  969         if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) &&
  970             neigh->probes >= neigh_max_probes(neigh)) {
  971                 struct mbuf *m;
  972 
  973                 neigh->nud_state = NUD_FAILED;
  974                 neigh->updated = jiffies;
  975                 notify = 1;
  976                 cxgb_neigh_update(neigh);
  977                 NEIGH_CACHE_STAT_INC(neigh->tbl, res_failed);
  978 
  979                 /* It is very thin place. report_unreachable is very 
  980                    complicated routine. Particularly, it can hit the same 
  981                    neighbour entry!
  982                    So that, we try to be accurate and avoid dead loop. --ANK
  983                  */
  984                 while (neigh->nud_state == NUD_FAILED &&
  985                        (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
  986                         write_unlock(&neigh->lock);
  987                         neigh->ops->error_report(neigh, skb);
  988                         write_lock(&neigh->lock);
  989                 }
  990                 skb_queue_purge(&neigh->arp_queue);
  991         }
  992 
  993         if (neigh->nud_state & NUD_IN_TIMER) {
  994                 if (time_before(next, jiffies + HZ/2))
  995                         next = jiffies + HZ/2;
  996                 if (!mod_timer(&neigh->timer, next))
  997                         neigh_hold(neigh);
  998         }
  999         if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) {
 1000                 struct mbuf *m = skb_peek(&neigh->arp_queue);
 1001 
 1002                 write_unlock(&neigh->lock);
 1003                 neigh->ops->solicit(neigh, skb);
 1004                 atomic_add_int(&neigh->probes, 1);
 1005                 if (m)
 1006                         m_free(m);
 1007         } else {
 1008 out:
 1009                 write_unlock(&neigh->lock);
 1010         }
 1011 
 1012 #ifdef CONFIG_ARPD
 1013         if (notify && neigh->parms->app_probes)
 1014                 neigh_app_notify(neigh);
 1015 #endif
 1016         neigh_release(neigh);
 1017 }
 1018 
 1019 static int
 1020 arp_constructor_offload(struct neighbour *neigh)
 1021 {
 1022         if (neigh->ifp && is_offloading(neigh->ifp))
 1023                 neigh->timer.function = neigh_timer_handler_offload;
 1024         return orig_arp_constructor(neigh);
 1025 }
 1026 
 1027 /*
 1028  * This must match exactly the signature of neigh_update for jprobes to work.
 1029  * It runs from a trap handler with interrupts off so don't disable BH.
 1030  */
 1031 static int
 1032 neigh_update_offload(struct neighbour *neigh, const u8 *lladdr,
 1033                                 u8 new, u32 flags)
 1034 {
 1035         write_lock(&neigh->lock);
 1036         cxgb_neigh_update(neigh);
 1037         write_unlock(&neigh->lock);
 1038         jprobe_return();
 1039         /* NOTREACHED */
 1040         return 0;
 1041 }
 1042 
 1043 static struct jprobe neigh_update_jprobe = {
 1044         .entry = (kprobe_opcode_t *) neigh_update_offload,
 1045         .kp.addr = (kprobe_opcode_t *) neigh_update
 1046 };
 1047 
 1048 #ifdef MODULE_SUPPORT
 1049 static int
 1050 prepare_arp_with_t3core(void)
 1051 {
 1052         int err;
 1053 
 1054         err = register_jprobe(&neigh_update_jprobe);
 1055         if (err) {
 1056                 log(LOG_ERR, "Could not install neigh_update jprobe, "
 1057                                 "error %d\n", err);
 1058                 return err;
 1059         }
 1060 
 1061         orig_arp_constructor = arp_tbl.constructor;
 1062         arp_tbl.constructor  = arp_constructor_offload;
 1063 
 1064         return 0;
 1065 }
 1066 
 1067 static void
 1068 restore_arp_sans_t3core(void)
 1069 {
 1070         arp_tbl.constructor = orig_arp_constructor;
 1071         unregister_jprobe(&neigh_update_jprobe);
 1072 }
 1073 
 1074 #else /* Module suport */
 1075 static inline int
 1076 prepare_arp_with_t3core(void)
 1077 {
 1078         return 0;
 1079 }
 1080 
 1081 static inline void
 1082 restore_arp_sans_t3core(void)
 1083 {}
 1084 #endif
 1085 #endif
 1086 /*
 1087  * Process a received packet with an unknown/unexpected CPL opcode.
 1088  */
 1089 static int
 1090 do_bad_cpl(struct toedev *dev, struct mbuf *m)
 1091 {
 1092         log(LOG_ERR, "%s: received bad CPL command 0x%x\n", dev->name,
 1093             *mtod(m, uint32_t *));
 1094         return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
 1095 }
 1096 
 1097 /*
 1098  * Handlers for each CPL opcode
 1099  */
 1100 static cpl_handler_func cpl_handlers[NUM_CPL_CMDS];
 1101 
 1102 /*
 1103  * Add a new handler to the CPL dispatch table.  A NULL handler may be supplied
 1104  * to unregister an existing handler.
 1105  */
 1106 void
 1107 t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h)
 1108 {
 1109         if (opcode < NUM_CPL_CMDS)
 1110                 cpl_handlers[opcode] = h ? h : do_bad_cpl;
 1111         else
 1112                 log(LOG_ERR, "T3C: handler registration for "
 1113                        "opcode %x failed\n", opcode);
 1114 }
 1115 
 1116 /*
 1117  * TOEDEV's receive method.
 1118  */
 1119 int
 1120 process_rx(struct toedev *dev, struct mbuf **m, int n)
 1121 {
 1122         while (n--) {
 1123                 struct mbuf *m0 = *m++;
 1124                 unsigned int opcode = G_OPCODE(ntohl(m0->m_pkthdr.csum_data));          
 1125                 int ret = cpl_handlers[opcode] (dev, m0);
 1126 
 1127 #if VALIDATE_TID
 1128                 if (ret & CPL_RET_UNKNOWN_TID) {
 1129                         union opcode_tid *p = cplhdr(m0);
 1130 
 1131                         log(LOG_ERR, "%s: CPL message (opcode %u) had "
 1132                                "unknown TID %u\n", dev->name, opcode,
 1133                                G_TID(ntohl(p->opcode_tid)));
 1134                 }
 1135 #endif
 1136                 if (ret & CPL_RET_BUF_DONE)
 1137                         m_freem(m0);
 1138         }
 1139         return 0;
 1140 }
 1141 
 1142 /*
 1143  * Sends an sk_buff to a T3C driver after dealing with any active network taps.
 1144  */
 1145 int
 1146 cxgb_ofld_send(struct toedev *dev, struct mbuf *m)
 1147 {
 1148         int r;
 1149 
 1150         critical_enter();
 1151         r = dev->send(dev, m);
 1152         critical_exit();
 1153         return r;
 1154 }
 1155 
 1156 
 1157 /**
 1158  * cxgb_ofld_recv - process n received offload packets
 1159  * @dev: the offload device
 1160  * @m: an array of offload packets
 1161  * @n: the number of offload packets
 1162  *
 1163  * Process an array of ingress offload packets.  Each packet is forwarded
 1164  * to any active network taps and then passed to the offload device's receive
 1165  * method.  We optimize passing packets to the receive method by passing
 1166  * it the whole array at once except when there are active taps.
 1167  */
 1168 int
 1169 cxgb_ofld_recv(struct toedev *dev, struct mbuf **m, int n)
 1170 {
 1171 
 1172 #if defined(CONFIG_CHELSIO_T3)
 1173         if (likely(!netdev_nit))
 1174                 return dev->recv(dev, skb, n);
 1175 
 1176         for ( ; n; n--, skb++) {
 1177                 skb[0]->dev = dev->lldev;
 1178                 dev_queue_xmit_nit(skb[0], dev->lldev);
 1179                 skb[0]->dev = NULL;
 1180                 dev->recv(dev, skb, 1);
 1181         }
 1182         return 0;
 1183 #else
 1184         return dev->recv(dev, m, n);
 1185 #endif
 1186 }
 1187 
 1188 void
 1189 cxgb_neigh_update(struct rtentry *rt)
 1190 {
 1191 
 1192         if (is_offloading(rt->rt_ifp)) {
 1193                 struct toedev *tdev = TOEDEV(rt->rt_ifp);
 1194 
 1195                 BUG_ON(!tdev);
 1196                 t3_l2t_update(tdev, rt);
 1197         }
 1198 }
 1199 
 1200 static void
 1201 set_l2t_ix(struct toedev *tdev, u32 tid, struct l2t_entry *e)
 1202 {
 1203         struct mbuf *m;
 1204         struct cpl_set_tcb_field *req;
 1205 
 1206         m = m_gethdr(M_NOWAIT, MT_DATA);
 1207         if (!m) {
 1208                 log(LOG_ERR, "%s: cannot allocate mbuf!\n", __FUNCTION__);
 1209                 return;
 1210         }
 1211         
 1212         m_set_priority(m, CPL_PRIORITY_CONTROL);
 1213         req = mtod(m, struct cpl_set_tcb_field *);
 1214         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 1215         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
 1216         req->reply = 0;
 1217         req->cpu_idx = 0;
 1218         req->word = htons(W_TCB_L2T_IX);
 1219         req->mask = htobe64(V_TCB_L2T_IX(M_TCB_L2T_IX));
 1220         req->val = htobe64(V_TCB_L2T_IX(e->idx));
 1221         tdev->send(tdev, m);
 1222 }
 1223 
 1224 void
 1225 cxgb_redirect(struct rtentry *old, struct rtentry *new)
 1226 {
 1227         struct ifnet *olddev, *newdev;
 1228         struct tid_info *ti;
 1229         struct toedev *tdev;
 1230         u32 tid;
 1231         int update_tcb;
 1232         struct l2t_entry *e;
 1233         struct toe_tid_entry *te;
 1234 
 1235         olddev = old->rt_ifp;
 1236         newdev = new->rt_ifp;
 1237         if (!is_offloading(olddev))
 1238                 return;
 1239         if (!is_offloading(newdev)) {
 1240                 log(LOG_WARNING, "%s: Redirect to non-offload"
 1241                     "device ignored.\n", __FUNCTION__);
 1242                 return;
 1243         }
 1244         tdev = TOEDEV(olddev);
 1245         BUG_ON(!tdev);
 1246         if (tdev != TOEDEV(newdev)) {
 1247                 log(LOG_WARNING, "%s: Redirect to different "
 1248                     "offload device ignored.\n", __FUNCTION__);
 1249                 return;
 1250         }
 1251 
 1252         /* Add new L2T entry */
 1253         e = t3_l2t_get(tdev, new, ((struct port_info *)new->rt_ifp->if_softc)->port_id);
 1254         if (!e) {
 1255                 log(LOG_ERR, "%s: couldn't allocate new l2t entry!\n",
 1256                        __FUNCTION__);
 1257                 return;
 1258         }
 1259 
 1260         /* Walk tid table and notify clients of dst change. */
 1261         ti = &(TOE_DATA(tdev))->tid_maps;
 1262         for (tid=0; tid < ti->ntids; tid++) {
 1263                 te = lookup_tid(ti, tid);
 1264                 BUG_ON(!te);
 1265                 if (te->ctx && te->client && te->client->redirect) {
 1266                         update_tcb = te->client->redirect(te->ctx, old, new,
 1267                                                           e);
 1268                         if (update_tcb)  {
 1269                                 l2t_hold(L2DATA(tdev), e);
 1270                                 set_l2t_ix(tdev, tid, e);
 1271                         }
 1272                 }
 1273         }
 1274         l2t_release(L2DATA(tdev), e);
 1275 }
 1276 
 1277 /*
 1278  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
 1279  * The allocated memory is cleared.
 1280  */
 1281 void *
 1282 cxgb_alloc_mem(unsigned long size)
 1283 {
 1284 
 1285         return malloc(size, M_DEVBUF, M_ZERO);
 1286 }
 1287 
 1288 /*
 1289  * Free memory allocated through t3_alloc_mem().
 1290  */
 1291 void
 1292 cxgb_free_mem(void *addr)
 1293 {
 1294         free(addr, M_DEVBUF);
 1295 }
 1296 
 1297 
 1298 /*
 1299  * Allocate and initialize the TID tables.  Returns 0 on success.
 1300  */
 1301 static int
 1302 init_tid_tabs(struct tid_info *t, unsigned int ntids,
 1303                          unsigned int natids, unsigned int nstids,
 1304                          unsigned int atid_base, unsigned int stid_base)
 1305 {
 1306         unsigned long size = ntids * sizeof(*t->tid_tab) +
 1307             natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab);
 1308 
 1309         t->tid_tab = cxgb_alloc_mem(size);
 1310         if (!t->tid_tab)
 1311                 return (ENOMEM);
 1312 
 1313         t->stid_tab = (union listen_entry *)&t->tid_tab[ntids];
 1314         t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids];
 1315         t->ntids = ntids;
 1316         t->nstids = nstids;
 1317         t->stid_base = stid_base;
 1318         t->sfree = NULL;
 1319         t->natids = natids;
 1320         t->atid_base = atid_base;
 1321         t->afree = NULL;
 1322         t->stids_in_use = t->atids_in_use = 0;
 1323         atomic_set_int(&t->tids_in_use, 0);
 1324         mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF);
 1325         mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF);
 1326 
 1327         /*
 1328          * Setup the free lists for stid_tab and atid_tab.
 1329          */
 1330         if (nstids) {
 1331                 while (--nstids)
 1332                         t->stid_tab[nstids - 1].next = &t->stid_tab[nstids];
 1333                 t->sfree = t->stid_tab;
 1334         }
 1335         if (natids) {
 1336                 while (--natids)
 1337                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
 1338                 t->afree = t->atid_tab;
 1339         }
 1340         return 0;
 1341 }
 1342 
 1343 static void
 1344 free_tid_maps(struct tid_info *t)
 1345 {
 1346         cxgb_free_mem(t->tid_tab);
 1347 }
 1348 
 1349 static inline void
 1350 add_adapter(adapter_t *adap)
 1351 {
 1352         rw_wlock(&adapter_list_lock);
 1353         TAILQ_INSERT_TAIL(&adapter_list, adap, adapter_entry);
 1354         rw_wunlock(&adapter_list_lock);
 1355 }
 1356 
 1357 static inline void
 1358 remove_adapter(adapter_t *adap)
 1359 {
 1360         rw_wlock(&adapter_list_lock);
 1361         TAILQ_REMOVE(&adapter_list, adap, adapter_entry);
 1362         rw_wunlock(&adapter_list_lock);
 1363 }
 1364 
 1365 /*
 1366  * XXX
 1367  */
 1368 #define t3_free_l2t(...)
 1369 
 1370 int
 1371 cxgb_offload_activate(struct adapter *adapter)
 1372 {
 1373         struct toedev *dev = &adapter->tdev;
 1374         int natids, err;
 1375         struct toe_data *t;
 1376         struct tid_range stid_range, tid_range;
 1377         struct mtutab mtutab;
 1378         unsigned int l2t_capacity;
 1379 
 1380         t = malloc(sizeof(*t), M_DEVBUF, M_WAITOK);
 1381         if (!t)
 1382                 return (ENOMEM);
 1383 
 1384         err = (EOPNOTSUPP);
 1385         if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 ||
 1386             dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 ||
 1387             dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 ||
 1388             dev->ctl(dev, GET_MTUS, &mtutab) < 0 ||
 1389             dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 ||
 1390             dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0)
 1391                 goto out_free;
 1392 
 1393         err = (ENOMEM);
 1394         L2DATA(dev) = t3_init_l2t(l2t_capacity);
 1395         if (!L2DATA(dev))
 1396                 goto out_free;
 1397 
 1398         natids = min(tid_range.num / 2, MAX_ATIDS);
 1399         err = init_tid_tabs(&t->tid_maps, tid_range.num, natids,
 1400                             stid_range.num, ATID_BASE, stid_range.base);
 1401         if (err)
 1402                 goto out_free_l2t;
 1403 
 1404         t->mtus = mtutab.mtus;
 1405         t->nmtus = mtutab.size;
 1406 
 1407         TASK_INIT(&t->tid_release_task, 0 /* XXX? */, t3_process_tid_release_list, dev);
 1408         mtx_init(&t->tid_release_lock, "tid release", NULL, MTX_DEF);
 1409         t->dev = dev;
 1410 
 1411         TOE_DATA(dev) = t;
 1412         dev->recv = process_rx;
 1413         dev->neigh_update = t3_l2t_update;
 1414 #if 0
 1415         offload_proc_dev_setup(dev);
 1416 #endif  
 1417         /* Register netevent handler once */
 1418         if (TAILQ_EMPTY(&adapter_list)) {
 1419 #if defined(CONFIG_CHELSIO_T3_MODULE)
 1420                 if (prepare_arp_with_t3core())
 1421                         log(LOG_ERR, "Unable to set offload capabilities\n");
 1422 #endif
 1423         }
 1424         add_adapter(adapter);
 1425         return 0;
 1426 
 1427 out_free_l2t:
 1428         t3_free_l2t(L2DATA(dev));
 1429         L2DATA(dev) = NULL;
 1430 out_free:
 1431         free(t, M_DEVBUF);
 1432         return err;
 1433 
 1434 }
 1435 
 1436 void
 1437 cxgb_offload_deactivate(struct adapter *adapter)
 1438 {
 1439         struct toedev *tdev = &adapter->tdev;
 1440         struct toe_data *t = TOE_DATA(tdev);
 1441 
 1442         remove_adapter(adapter);
 1443         if (TAILQ_EMPTY(&adapter_list)) {
 1444 #if defined(CONFIG_CHELSIO_T3_MODULE)
 1445                 restore_arp_sans_t3core();
 1446 #endif
 1447         }
 1448         free_tid_maps(&t->tid_maps);
 1449         TOE_DATA(tdev) = NULL;
 1450         t3_free_l2t(L2DATA(tdev));
 1451         L2DATA(tdev) = NULL;
 1452         free(t, M_DEVBUF);
 1453 }
 1454 
 1455 
 1456 static inline void
 1457 register_tdev(struct toedev *tdev)
 1458 {
 1459         static int unit;
 1460 
 1461         mtx_lock(&cxgb_db_lock);
 1462         snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++);
 1463         TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, ofld_entry);
 1464         mtx_unlock(&cxgb_db_lock);
 1465 }
 1466 
 1467 static inline void
 1468 unregister_tdev(struct toedev *tdev)
 1469 {
 1470         mtx_lock(&cxgb_db_lock);
 1471         TAILQ_REMOVE(&ofld_dev_list, tdev, ofld_entry);
 1472         mtx_unlock(&cxgb_db_lock);      
 1473 }
 1474 
 1475 void
 1476 cxgb_adapter_ofld(struct adapter *adapter)
 1477 {
 1478         struct toedev *tdev = &adapter->tdev;
 1479 
 1480         cxgb_set_dummy_ops(tdev);
 1481         tdev->send = t3_offload_tx;
 1482         tdev->ctl = cxgb_offload_ctl;
 1483         tdev->type = adapter->params.rev == 0 ?
 1484                      T3A : T3B;
 1485 
 1486         register_tdev(tdev);
 1487 #if 0   
 1488         offload_proc_dev_init(tdev);
 1489 #endif  
 1490 }
 1491 
 1492 void
 1493 cxgb_adapter_unofld(struct adapter *adapter)
 1494 {
 1495         struct toedev *tdev = &adapter->tdev;
 1496 #if 0
 1497         offload_proc_dev_cleanup(tdev);
 1498         offload_proc_dev_exit(tdev);
 1499 #endif  
 1500         tdev->recv = NULL;
 1501         tdev->neigh_update = NULL;
 1502 
 1503         unregister_tdev(tdev);
 1504 }
 1505 
 1506 void
 1507 cxgb_offload_init(void)
 1508 {
 1509         int i;
 1510 
 1511         if (inited)
 1512                 return;
 1513         else
 1514                 inited = 1;
 1515         
 1516         mtx_init(&cxgb_db_lock, "ofld db", NULL, MTX_DEF);
 1517         rw_init(&adapter_list_lock, "ofld adap list");
 1518         TAILQ_INIT(&client_list);
 1519         TAILQ_INIT(&ofld_dev_list);
 1520         TAILQ_INIT(&adapter_list);
 1521         
 1522         for (i = 0; i < NUM_CPL_CMDS; ++i)
 1523                 cpl_handlers[i] = do_bad_cpl;
 1524         
 1525         t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl);
 1526         t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
 1527         t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl);
 1528         t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl);
 1529         t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr);
 1530         t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl);
 1531         t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl);
 1532         t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl);
 1533         t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl);
 1534         t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl);
 1535         t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl);
 1536         t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl);
 1537         t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl);
 1538         t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl);
 1539         t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl);
 1540         t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss);
 1541         t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
 1542         t3_register_cpl_handler(CPL_SET_TCB_RPL, do_set_tcb_rpl);
 1543         t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term);
 1544         t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl);
 1545         t3_register_cpl_handler(CPL_TRACE_PKT, do_trace);
 1546         t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl);
 1547         t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl);
 1548         t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl);
 1549 #if 0
 1550        if (offload_proc_init())
 1551                log(LOG_WARNING, "Unable to create /proc/net/cxgb3 dir\n");
 1552 #endif
 1553 }
 1554 
 1555 void 
 1556 cxgb_offload_exit(void)
 1557 {
 1558         static int deinited = 0;
 1559 
 1560         if (deinited)
 1561                 return;
 1562 
 1563         deinited = 1;
 1564         mtx_destroy(&cxgb_db_lock);
 1565         rw_destroy(&adapter_list_lock);
 1566 #if 0   
 1567         offload_proc_cleanup();
 1568 #endif  
 1569 }
 1570 
 1571 #if 0
 1572 static int
 1573 offload_info_read_proc(char *buf, char **start, off_t offset,
 1574                                   int length, int *eof, void *data)
 1575 {
 1576         struct toe_data *d = data;
 1577         struct tid_info *t = &d->tid_maps;
 1578         int len;
 1579 
 1580         len = sprintf(buf, "TID range: 0..%d, in use: %u\n"
 1581                       "STID range: %d..%d, in use: %u\n"
 1582                       "ATID range: %d..%d, in use: %u\n"
 1583                       "MSS: %u\n",
 1584                       t->ntids - 1, atomic_read(&t->tids_in_use), t->stid_base,
 1585                       t->stid_base + t->nstids - 1, t->stids_in_use,
 1586                       t->atid_base, t->atid_base + t->natids - 1,
 1587                       t->atids_in_use, d->tx_max_chunk);
 1588         if (len > length)
 1589                 len = length;
 1590         *eof = 1;
 1591         return len;
 1592 }
 1593 
 1594 static int
 1595 offload_info_proc_setup(struct proc_dir_entry *dir,
 1596                                    struct toe_data *d)
 1597 {
 1598         struct proc_dir_entry *p;
 1599 
 1600         if (!dir)
 1601                 return (EINVAL);
 1602 
 1603         p = create_proc_read_entry("info", 0, dir, offload_info_read_proc, d);
 1604         if (!p)
 1605                 return (ENOMEM);
 1606 
 1607         p->owner = THIS_MODULE;
 1608         return 0;
 1609 }
 1610 
 1611 
 1612 static int
 1613 offload_devices_read_proc(char *buf, char **start, off_t offset,
 1614                                      int length, int *eof, void *data)
 1615 {
 1616         int len;
 1617         struct toedev *dev;
 1618         struct net_device *ndev;
 1619         
 1620         len = sprintf(buf, "Device           Interfaces\n");
 1621         
 1622         mtx_lock(&cxgb_db_lock);
 1623         TAILQ_FOREACH(dev, &ofld_dev_list, ofld_entry) {        
 1624                 len += sprintf(buf + len, "%-16s", dev->name);
 1625                 read_lock(&dev_base_lock);
 1626                 for (ndev = dev_base; ndev; ndev = ndev->next) {
 1627                         if (TOEDEV(ndev) == dev)
 1628                                 len += sprintf(buf + len, " %s", ndev->name);
 1629                 }
 1630                 read_unlock(&dev_base_lock);
 1631                 len += sprintf(buf + len, "\n");
 1632                 if (len >= length)
 1633                         break;
 1634         }
 1635         mtx_unlock(&cxgb_db_lock);
 1636         
 1637         if (len > length)
 1638                 len = length;
 1639         *eof = 1;
 1640         return len;
 1641 }
 1642 
 1643 #endif
 1644 

Cache object: 27bbd3088de2cb8bcddc663aecfb8a5e


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.