mlx5_srq.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2013-2017, Mellanox Technologies, Ltd.  All rights reserved.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. Redistributions in binary form must reproduce the above copyright
   10  *    notice, this list of conditions and the following disclaimer in the
   11  *    documentation and/or other materials provided with the distribution.
   12  *
   13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
   14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
   17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   23  * SUCH DAMAGE.
   24  *
   25  * $FreeBSD$
   26  */
   27 
   28 #include "opt_rss.h"
   29 #include "opt_ratelimit.h"
   30 
   31 #include <linux/kernel.h>
   32 #include <linux/module.h>
   33 #include <dev/mlx5/driver.h>
   34 #include <dev/mlx5/srq.h>
   35 #include <rdma/ib_verbs.h>
   36 #include <dev/mlx5/mlx5_core/mlx5_core.h>
   37 #include <dev/mlx5/mlx5_core/transobj.h>
   38 
   39 void mlx5_srq_event(struct mlx5_core_dev *dev, u32 srqn, int event_type)
   40 {
   41         struct mlx5_srq_table *table = &dev->priv.srq_table;
   42         struct mlx5_core_srq *srq;
   43 
   44         spin_lock(&table->lock);
   45 
   46         srq = radix_tree_lookup(&table->tree, srqn);
   47         if (srq)
   48                 atomic_inc(&srq->refcount);
   49 
   50         spin_unlock(&table->lock);
   51 
   52         if (!srq) {
   53                 mlx5_core_warn(dev, "Async event for bogus SRQ 0x%08x\n", srqn);
   54                 return;
   55         }
   56 
   57         srq->event(srq, event_type);
   58 
   59         if (atomic_dec_and_test(&srq->refcount))
   60                 complete(&srq->free);
   61 }
   62 
   63 static void set_wq(void *wq, struct mlx5_srq_attr *in)
   64 {
   65         MLX5_SET(wq,   wq, wq_signature,  !!(in->flags & MLX5_SRQ_FLAG_WQ_SIG));
   66         MLX5_SET(wq,   wq, log_wq_pg_sz,  in->log_page_size);
   67         MLX5_SET(wq,   wq, log_wq_stride, in->wqe_shift + 4);
   68         MLX5_SET(wq,   wq, log_wq_sz,     in->log_size);
   69         MLX5_SET(wq,   wq, page_offset,   in->page_offset);
   70         MLX5_SET(wq,   wq, lwm,           in->lwm);
   71         MLX5_SET(wq,   wq, pd,            in->pd);
   72         MLX5_SET64(wq, wq, dbr_addr,      in->db_record);
   73 }
   74 
   75 static void set_srqc(void *srqc, struct mlx5_srq_attr *in)
   76 {
   77         MLX5_SET(srqc,   srqc, wq_signature,  !!(in->flags & MLX5_SRQ_FLAG_WQ_SIG));
   78         MLX5_SET(srqc,   srqc, log_page_size, in->log_page_size);
   79         MLX5_SET(srqc,   srqc, log_rq_stride, in->wqe_shift);
   80         MLX5_SET(srqc,   srqc, log_srq_size,  in->log_size);
   81         MLX5_SET(srqc,   srqc, page_offset,   in->page_offset);
   82         MLX5_SET(srqc,   srqc, lwm,           in->lwm);
   83         MLX5_SET(srqc,   srqc, pd,            in->pd);
   84         MLX5_SET64(srqc, srqc, dbr_addr,      in->db_record);
   85         MLX5_SET(srqc,   srqc, xrcd,          in->xrcd);
   86         MLX5_SET(srqc,   srqc, cqn,           in->cqn);
   87 }
   88 
   89 static void get_wq(void *wq, struct mlx5_srq_attr *in)
   90 {
   91         if (MLX5_GET(wq, wq, wq_signature))
   92                 in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
   93         in->log_page_size = MLX5_GET(wq,   wq, log_wq_pg_sz);
   94         in->wqe_shift     = MLX5_GET(wq,   wq, log_wq_stride) - 4;
   95         in->log_size      = MLX5_GET(wq,   wq, log_wq_sz);
   96         in->page_offset   = MLX5_GET(wq,   wq, page_offset);
   97         in->lwm           = MLX5_GET(wq,   wq, lwm);
   98         in->pd            = MLX5_GET(wq,   wq, pd);
   99         in->db_record     = MLX5_GET64(wq, wq, dbr_addr);
  100 }
  101 
  102 static void get_srqc(void *srqc, struct mlx5_srq_attr *in)
  103 {
  104         if (MLX5_GET(srqc, srqc, wq_signature))
  105                 in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
  106         in->log_page_size = MLX5_GET(srqc,   srqc, log_page_size);
  107         in->wqe_shift     = MLX5_GET(srqc,   srqc, log_rq_stride);
  108         in->log_size      = MLX5_GET(srqc,   srqc, log_srq_size);
  109         in->page_offset   = MLX5_GET(srqc,   srqc, page_offset);
  110         in->lwm           = MLX5_GET(srqc,   srqc, lwm);
  111         in->pd            = MLX5_GET(srqc,   srqc, pd);
  112         in->db_record     = MLX5_GET64(srqc, srqc, dbr_addr);
  113 }
  114 
  115 struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn)
  116 {
  117         struct mlx5_srq_table *table = &dev->priv.srq_table;
  118         struct mlx5_core_srq *srq;
  119 
  120         spin_lock(&table->lock);
  121 
  122         srq = radix_tree_lookup(&table->tree, srqn);
  123         if (srq)
  124                 atomic_inc(&srq->refcount);
  125 
  126         spin_unlock(&table->lock);
  127 
  128         return srq;
  129 }
  130 EXPORT_SYMBOL(mlx5_core_get_srq);
  131 
  132 static int get_pas_size(struct mlx5_srq_attr *in)
  133 {
  134         u32 log_page_size = in->log_page_size + 12;
  135         u32 log_srq_size  = in->log_size;
  136         u32 log_rq_stride = in->wqe_shift;
  137         u32 page_offset   = in->page_offset;
  138         u32 po_quanta     = 1 << (log_page_size - 6);
  139         u32 rq_sz         = 1 << (log_srq_size + 4 + log_rq_stride);
  140         u32 page_size     = 1 << log_page_size;
  141         u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);
  142         u32 rq_num_pas    = (rq_sz_po + page_size - 1) / page_size;
  143 
  144         return rq_num_pas * sizeof(u64);
  145 
  146 }
  147 
  148 static int create_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  149                           struct mlx5_srq_attr *in)
  150 {
  151         void *create_in;
  152         void *rmpc;
  153         void *wq;
  154         int pas_size;
  155         int inlen;
  156         int err;
  157 
  158         pas_size = get_pas_size(in);
  159         inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
  160         create_in = mlx5_vzalloc(inlen);
  161         if (!create_in)
  162                 return -ENOMEM;
  163 
  164         rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);
  165         wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
  166 
  167         MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
  168         set_wq(wq, in);
  169         memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);
  170 
  171         err = mlx5_core_create_rmp(dev, create_in, inlen, &srq->srqn);
  172 
  173         kvfree(create_in);
  174         return err;
  175 }
  176 
  177 static int destroy_rmp_cmd(struct mlx5_core_dev *dev,
  178                             struct mlx5_core_srq *srq)
  179 {
  180         return mlx5_core_destroy_rmp(dev, srq->srqn);
  181 }
  182 
  183 static int query_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  184                          struct mlx5_srq_attr *out)
  185 {
  186         u32 *rmp_out;
  187         void *rmpc;
  188         int err;
  189 
  190         rmp_out =  mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_rmp_out));
  191         if (!rmp_out)
  192                 return -ENOMEM;
  193 
  194         err = mlx5_core_query_rmp(dev, srq->srqn, rmp_out);
  195         if (err)
  196                 goto out;
  197 
  198         rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);
  199         get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out);
  200         if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY)
  201                 out->flags |= MLX5_SRQ_FLAG_ERR;
  202 
  203 out:
  204         kvfree(rmp_out);
  205         return 0;
  206 }
  207 
  208 static int arm_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, u16 lwm)
  209 {
  210         return mlx5_core_arm_rmp(dev, srq->srqn, lwm);
  211 }
  212 
  213 static int create_xrc_srq_cmd(struct mlx5_core_dev *dev,
  214                               struct mlx5_core_srq *srq,
  215                               struct mlx5_srq_attr *in)
  216 {
  217         void *create_in;
  218         void *xrc_srqc;
  219         void *pas;
  220         int pas_size;
  221         int inlen;
  222         int err;
  223 
  224         pas_size  = get_pas_size(in);
  225         inlen     = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
  226         create_in = mlx5_vzalloc(inlen);
  227         if (!create_in)
  228                 return -ENOMEM;
  229 
  230         xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in, xrc_srq_context_entry);
  231         pas      = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);
  232 
  233         set_srqc(xrc_srqc, in);
  234         MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
  235         memcpy(pas, in->pas, pas_size);
  236 
  237         err = mlx5_core_create_xsrq(dev, create_in, inlen, &srq->srqn);
  238         if (err)
  239                 goto out;
  240 
  241 out:
  242         kvfree(create_in);
  243         return err;
  244 }
  245 
  246 static int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev,
  247                                struct mlx5_core_srq *srq)
  248 {
  249         return mlx5_core_destroy_xsrq(dev, srq->srqn);
  250 }
  251 
  252 static int query_xrc_srq_cmd(struct mlx5_core_dev *dev,
  253                              struct mlx5_core_srq *srq,
  254                              struct mlx5_srq_attr *out)
  255 {
  256         u32 *xrcsrq_out;
  257         void *xrc_srqc;
  258         int err;
  259 
  260         xrcsrq_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out));
  261         if (!xrcsrq_out)
  262                 return -ENOMEM;
  263 
  264         err = mlx5_core_query_xsrq(dev, srq->srqn, xrcsrq_out);
  265         if (err)
  266                 goto out;
  267 
  268         xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out,
  269                                 xrc_srq_context_entry);
  270         get_srqc(xrc_srqc, out);
  271         if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD)
  272                 out->flags |= MLX5_SRQ_FLAG_ERR;
  273 
  274 out:
  275         kvfree(xrcsrq_out);
  276         return err;
  277 }
  278 
  279 static int arm_xrc_srq_cmd(struct mlx5_core_dev *dev,
  280                            struct mlx5_core_srq *srq, u16 lwm)
  281 {
  282         return mlx5_core_arm_xsrq(dev, srq->srqn, lwm);
  283 }
  284 
  285 static int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  286                           struct mlx5_srq_attr *in)
  287 {
  288         u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0};
  289         void *create_in;
  290         void *srqc;
  291         void *pas;
  292         int pas_size;
  293         int inlen;
  294         int err;
  295 
  296         pas_size  = get_pas_size(in);
  297         inlen     = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
  298         create_in = mlx5_vzalloc(inlen);
  299         if (!create_in)
  300                 return -ENOMEM;
  301 
  302         srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry);
  303         pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
  304 
  305         set_srqc(srqc, in);
  306         memcpy(pas, in->pas, pas_size);
  307 
  308         MLX5_SET(create_srq_in, create_in, opcode, MLX5_CMD_OP_CREATE_SRQ);
  309         err = mlx5_cmd_exec(dev, create_in, inlen, create_out, sizeof(create_out));
  310         kvfree(create_in);
  311         if (!err)
  312                 srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);
  313 
  314         return err;
  315 }
  316 
  317 static int destroy_srq_cmd(struct mlx5_core_dev *dev,
  318                            struct mlx5_core_srq *srq)
  319 {
  320         u32 srq_out[MLX5_ST_SZ_DW(destroy_srq_out)] = {0};
  321         u32 srq_in[MLX5_ST_SZ_DW(destroy_srq_in)] = {0};
  322 
  323         MLX5_SET(destroy_srq_in, srq_in, opcode, MLX5_CMD_OP_DESTROY_SRQ);
  324         MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn);
  325 
  326         return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in), srq_out, sizeof(srq_out));
  327 }
  328 
  329 static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  330                          struct mlx5_srq_attr *out)
  331 {
  332         u32 srq_in[MLX5_ST_SZ_DW(query_srq_in)] = {0};
  333         u32 *srq_out;
  334         void *srqc;
  335         int outlen = MLX5_ST_SZ_BYTES(query_srq_out);
  336         int err;
  337 
  338         srq_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_srq_out));
  339         if (!srq_out)
  340                 return -ENOMEM;
  341 
  342         MLX5_SET(query_srq_in, srq_in, opcode, MLX5_CMD_OP_QUERY_SRQ);
  343         MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn);
  344         err =  mlx5_cmd_exec(dev, srq_in, sizeof(srq_in), srq_out, outlen);
  345         if (err)
  346                 goto out;
  347 
  348         srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry);
  349         get_srqc(srqc, out);
  350         if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD)
  351                 out->flags |= MLX5_SRQ_FLAG_ERR;
  352 out:
  353         kvfree(srq_out);
  354         return err;
  355 }
  356 
  357 static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  358                        u16 lwm, int is_srq)
  359 {
  360         /* arm_srq structs missing using identical xrc ones */
  361         u32 srq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0};
  362         u32 srq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
  363 
  364         MLX5_SET(arm_xrc_srq_in, srq_in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);
  365         MLX5_SET(arm_xrc_srq_in, srq_in, xrc_srqn, srq->srqn);
  366         MLX5_SET(arm_xrc_srq_in, srq_in, lwm,      lwm);
  367 
  368         return  mlx5_cmd_exec(dev, srq_in, sizeof(srq_in), srq_out, sizeof(srq_out));
  369 }
  370 
  371 static int create_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  372                             struct mlx5_srq_attr *in)
  373 {
  374         if (!dev->issi)
  375                 return create_srq_cmd(dev, srq, in);
  376         else if (srq->common.res == MLX5_RES_XSRQ)
  377                 return create_xrc_srq_cmd(dev, srq, in);
  378         else
  379                 return create_rmp_cmd(dev, srq, in);
  380 }
  381 
  382 static int destroy_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
  383 {
  384         if (!dev->issi)
  385                 return destroy_srq_cmd(dev, srq);
  386         else if (srq->common.res == MLX5_RES_XSRQ)
  387                 return destroy_xrc_srq_cmd(dev, srq);
  388         else
  389                 return destroy_rmp_cmd(dev, srq);
  390 }
  391 
  392 int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  393                          struct mlx5_srq_attr *in)
  394 {
  395         int err;
  396         struct mlx5_srq_table *table = &dev->priv.srq_table;
  397 
  398         if (in->type == IB_SRQT_XRC)
  399                 srq->common.res = MLX5_RES_XSRQ;
  400         else
  401                 srq->common.res = MLX5_RES_SRQ;
  402 
  403         err = create_srq_split(dev, srq, in);
  404         if (err)
  405                 return err;
  406 
  407         atomic_set(&srq->refcount, 1);
  408         init_completion(&srq->free);
  409 
  410         spin_lock_irq(&table->lock);
  411         err = radix_tree_insert(&table->tree, srq->srqn, srq);
  412         spin_unlock_irq(&table->lock);
  413         if (err) {
  414                 mlx5_core_warn(dev, "err %d, srqn 0x%x\n", err, srq->srqn);
  415                 goto err_destroy_srq_split;
  416         }
  417 
  418         return 0;
  419 
  420 err_destroy_srq_split:
  421         destroy_srq_split(dev, srq);
  422 
  423         return err;
  424 }
  425 EXPORT_SYMBOL(mlx5_core_create_srq);
  426 
  427 int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
  428 {
  429         struct mlx5_srq_table *table = &dev->priv.srq_table;
  430         struct mlx5_core_srq *tmp;
  431         int err;
  432 
  433         spin_lock_irq(&table->lock);
  434         tmp = radix_tree_delete(&table->tree, srq->srqn);
  435         spin_unlock_irq(&table->lock);
  436         if (!tmp) {
  437                 mlx5_core_warn(dev, "srq 0x%x not found in tree\n", srq->srqn);
  438                 return -EINVAL;
  439         }
  440         if (tmp != srq) {
  441                 mlx5_core_warn(dev, "corruption on srqn 0x%x\n", srq->srqn);
  442                 return -EINVAL;
  443         }
  444 
  445         err = destroy_srq_split(dev, srq);
  446         if (err)
  447                 return err;
  448 
  449         if (atomic_dec_and_test(&srq->refcount))
  450                 complete(&srq->free);
  451         wait_for_completion(&srq->free);
  452 
  453         return 0;
  454 }
  455 EXPORT_SYMBOL(mlx5_core_destroy_srq);
  456 
  457 int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  458                         struct mlx5_srq_attr *out)
  459 {
  460         if (!dev->issi)
  461                 return query_srq_cmd(dev, srq, out);
  462         else if (srq->common.res == MLX5_RES_XSRQ)
  463                 return query_xrc_srq_cmd(dev, srq, out);
  464         else
  465                 return query_rmp_cmd(dev, srq, out);
  466 }
  467 EXPORT_SYMBOL(mlx5_core_query_srq);
  468 
  469 int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
  470                       u16 lwm, int is_srq)
  471 {
  472         if (!dev->issi)
  473                 return arm_srq_cmd(dev, srq, lwm, is_srq);
  474         else if (srq->common.res == MLX5_RES_XSRQ)
  475                 return arm_xrc_srq_cmd(dev, srq, lwm);
  476         else
  477                 return arm_rmp_cmd(dev, srq, lwm);
  478 }
  479 EXPORT_SYMBOL(mlx5_core_arm_srq);
  480 
  481 void mlx5_init_srq_table(struct mlx5_core_dev *dev)
  482 {
  483         struct mlx5_srq_table *table = &dev->priv.srq_table;
  484 
  485         memset(table, 0, sizeof(*table));
  486         spin_lock_init(&table->lock);
  487         INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
  488 }
  489 
  490 void mlx5_cleanup_srq_table(struct mlx5_core_dev *dev)
  491 {
  492         /* nothing */
  493 }
Cache object: 6d5b0a0fecb4570b4daacca4075d0277
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/mlx5/mlx5_core/mlx5_srq.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/mlx5/mlx5_core/mlx5_srq.c
