mlx4_cq.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
    3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
    4  * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
    5  * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved.
    6  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
    7  *
    8  * This software is available to you under a choice of one of two
    9  * licenses.  You may choose to be licensed under the terms of the GNU
   10  * General Public License (GPL) Version 2, available from the file
   11  * COPYING in the main directory of this source tree, or the
   12  * OpenIB.org BSD license below:
   13  *
   14  *     Redistribution and use in source and binary forms, with or
   15  *     without modification, are permitted provided that the following
   16  *     conditions are met:
   17  *
   18  *      - Redistributions of source code must retain the above
   19  *        copyright notice, this list of conditions and the following
   20  *        disclaimer.
   21  *
   22  *      - Redistributions in binary form must reproduce the above
   23  *        copyright notice, this list of conditions and the following
   24  *        disclaimer in the documentation and/or other materials
   25  *        provided with the distribution.
   26  *
   27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
   30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
   31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
   32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
   33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   34  * SOFTWARE.
   35  */
   36 
   37 #include <linux/hardirq.h>
   38 #include <linux/module.h>
   39 #include <dev/mlx4/cmd.h>
   40 #include <dev/mlx4/cq.h>
   41 
   42 #include "mlx4.h"
   43 #include "icm.h"
   44 
   45 #define MLX4_CQ_STATUS_OK               ( 0 << 28)
   46 #define MLX4_CQ_STATUS_OVERFLOW         ( 9 << 28)
   47 #define MLX4_CQ_STATUS_WRITE_FAIL       (10 << 28)
   48 #define MLX4_CQ_FLAG_CC                 ( 1 << 18)
   49 #define MLX4_CQ_FLAG_OI                 ( 1 << 17)
   50 #define MLX4_CQ_STATE_ARMED             ( 9 <<  8)
   51 #define MLX4_CQ_STATE_ARMED_SOL         ( 6 <<  8)
   52 #define MLX4_EQ_STATE_FIRED             (10 <<  8)
   53 
   54 void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn)
   55 {
   56         struct mlx4_cq *cq;
   57 
   58         cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree,
   59                                cqn & (dev->caps.num_cqs - 1));
   60         if (!cq) {
   61                 mlx4_dbg(dev, "Completion event for bogus CQ %08x\n", cqn);
   62                 return;
   63         }
   64 
   65         ++cq->arm_sn;
   66 
   67         cq->comp(cq);
   68 }
   69 
   70 void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type)
   71 {
   72         struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
   73         struct mlx4_cq *cq;
   74 
   75         spin_lock(&cq_table->lock);
   76 
   77         cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1));
   78         if (cq)
   79                 atomic_inc(&cq->refcount);
   80 
   81         spin_unlock(&cq_table->lock);
   82 
   83         if (!cq) {
   84                 mlx4_warn(dev, "Async event for bogus CQ %08x\n", cqn);
   85                 return;
   86         }
   87 
   88         cq->event(cq, event_type);
   89 
   90         if (atomic_dec_and_test(&cq->refcount))
   91                 complete(&cq->free);
   92 }
   93 
   94 static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
   95                          int cq_num)
   96 {
   97         return mlx4_cmd(dev, mailbox->dma, cq_num, 0,
   98                         MLX4_CMD_SW2HW_CQ, MLX4_CMD_TIME_CLASS_A,
   99                         MLX4_CMD_WRAPPED);
  100 }
  101 
  102 static int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
  103                          int cq_num, u32 opmod)
  104 {
  105         return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ,
  106                         MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
  107 }
  108 
  109 static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
  110                          int cq_num)
  111 {
  112         return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0,
  113                             cq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ,
  114                             MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
  115 }
  116 
  117 int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq,
  118                    u16 count, u16 period)
  119 {
  120         struct mlx4_cmd_mailbox *mailbox;
  121         struct mlx4_cq_context *cq_context;
  122         int err;
  123 
  124         mailbox = mlx4_alloc_cmd_mailbox(dev);
  125         if (IS_ERR(mailbox))
  126                 return PTR_ERR(mailbox);
  127 
  128         cq_context = mailbox->buf;
  129         cq_context->cq_max_count = cpu_to_be16(count);
  130         cq_context->cq_period    = cpu_to_be16(period);
  131 
  132         err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1);
  133 
  134         mlx4_free_cmd_mailbox(dev, mailbox);
  135         return err;
  136 }
  137 EXPORT_SYMBOL_GPL(mlx4_cq_modify);
  138 
  139 int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,
  140                    int entries, struct mlx4_mtt *mtt)
  141 {
  142         struct mlx4_cmd_mailbox *mailbox;
  143         struct mlx4_cq_context *cq_context;
  144         u64 mtt_addr;
  145         int err;
  146 
  147         mailbox = mlx4_alloc_cmd_mailbox(dev);
  148         if (IS_ERR(mailbox))
  149                 return PTR_ERR(mailbox);
  150 
  151         cq_context = mailbox->buf;
  152         cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24);
  153         cq_context->log_page_size   = mtt->page_shift - 12;
  154         mtt_addr = mlx4_mtt_addr(dev, mtt);
  155         cq_context->mtt_base_addr_h = mtt_addr >> 32;
  156         cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
  157 
  158         err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0);
  159 
  160         mlx4_free_cmd_mailbox(dev, mailbox);
  161         return err;
  162 }
  163 EXPORT_SYMBOL_GPL(mlx4_cq_resize);
  164 
  165 int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
  166 {
  167         struct mlx4_priv *priv = mlx4_priv(dev);
  168         struct mlx4_cq_table *cq_table = &priv->cq_table;
  169         int err;
  170 
  171         *cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
  172         if (*cqn == -1)
  173                 return -ENOMEM;
  174 
  175         err = mlx4_table_get(dev, &cq_table->table, *cqn, GFP_KERNEL);
  176         if (err)
  177                 goto err_out;
  178 
  179         err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn, GFP_KERNEL);
  180         if (err)
  181                 goto err_put;
  182         return 0;
  183 
  184 err_put:
  185         mlx4_table_put(dev, &cq_table->table, *cqn);
  186 
  187 err_out:
  188         mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR);
  189         return err;
  190 }
  191 
  192 static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
  193 {
  194         u64 out_param;
  195         int err;
  196 
  197         if (mlx4_is_mfunc(dev)) {
  198                 err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ,
  199                                    RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,
  200                                    MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
  201                 if (err)
  202                         return err;
  203                 else {
  204                         *cqn = get_param_l(&out_param);
  205                         return 0;
  206                 }
  207         }
  208         return __mlx4_cq_alloc_icm(dev, cqn);
  209 }
  210 
  211 void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
  212 {
  213         struct mlx4_priv *priv = mlx4_priv(dev);
  214         struct mlx4_cq_table *cq_table = &priv->cq_table;
  215 
  216         mlx4_table_put(dev, &cq_table->cmpt_table, cqn);
  217         mlx4_table_put(dev, &cq_table->table, cqn);
  218         mlx4_bitmap_free(&cq_table->bitmap, cqn, MLX4_NO_RR);
  219 }
  220 
  221 static void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
  222 {
  223         u64 in_param = 0;
  224         int err;
  225 
  226         if (mlx4_is_mfunc(dev)) {
  227                 set_param_l(&in_param, cqn);
  228                 err = mlx4_cmd(dev, in_param, RES_CQ, RES_OP_RESERVE_AND_MAP,
  229                                MLX4_CMD_FREE_RES,
  230                                MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
  231                 if (err)
  232                         mlx4_warn(dev, "Failed freeing cq:%d\n", cqn);
  233         } else
  234                 __mlx4_cq_free_icm(dev, cqn);
  235 }
  236 
  237 int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
  238                   struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec,
  239                   struct mlx4_cq *cq, unsigned vector, int collapsed,
  240                   int timestamp_en)
  241 {
  242         struct mlx4_priv *priv = mlx4_priv(dev);
  243         struct mlx4_cq_table *cq_table = &priv->cq_table;
  244         struct mlx4_cmd_mailbox *mailbox;
  245         struct mlx4_cq_context *cq_context;
  246         u64 mtt_addr;
  247         int err;
  248 
  249         if (vector >= dev->caps.num_comp_vectors)
  250                 return -EINVAL;
  251 
  252         cq->vector = vector;
  253 
  254         err = mlx4_cq_alloc_icm(dev, &cq->cqn);
  255         if (err)
  256                 return err;
  257 
  258         spin_lock_irq(&cq_table->lock);
  259         err = radix_tree_insert(&cq_table->tree, cq->cqn, cq);
  260         spin_unlock_irq(&cq_table->lock);
  261         if (err)
  262                 goto err_icm;
  263 
  264         mailbox = mlx4_alloc_cmd_mailbox(dev);
  265         if (IS_ERR(mailbox)) {
  266                 err = PTR_ERR(mailbox);
  267                 goto err_radix;
  268         }
  269 
  270         cq_context = mailbox->buf;
  271         cq_context->flags           = cpu_to_be32(!!collapsed << 18);
  272         if (timestamp_en)
  273                 cq_context->flags  |= cpu_to_be32(1 << 19);
  274 
  275         cq_context->logsize_usrpage =
  276                 cpu_to_be32((ilog2(nent) << 24) |
  277                             mlx4_to_hw_uar_index(dev, uar->index));
  278         cq_context->comp_eqn        = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].eqn;
  279         cq_context->log_page_size   = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
  280 
  281         mtt_addr = mlx4_mtt_addr(dev, mtt);
  282         cq_context->mtt_base_addr_h = mtt_addr >> 32;
  283         cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
  284         cq_context->db_rec_addr     = cpu_to_be64(db_rec);
  285 
  286         err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn);
  287         mlx4_free_cmd_mailbox(dev, mailbox);
  288         if (err)
  289                 goto err_radix;
  290 
  291         cq->cons_index = 0;
  292         cq->arm_sn     = 1;
  293         cq->uar        = uar;
  294         atomic_set(&cq->refcount, 1);
  295         init_completion(&cq->free);
  296 
  297         cq->irq = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].irq;
  298         return 0;
  299 
  300 err_radix:
  301         spin_lock_irq(&cq_table->lock);
  302         radix_tree_delete(&cq_table->tree, cq->cqn);
  303         spin_unlock_irq(&cq_table->lock);
  304 
  305 err_icm:
  306         mlx4_cq_free_icm(dev, cq->cqn);
  307 
  308         return err;
  309 }
  310 EXPORT_SYMBOL_GPL(mlx4_cq_alloc);
  311 
  312 void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
  313 {
  314         struct mlx4_priv *priv = mlx4_priv(dev);
  315         struct mlx4_cq_table *cq_table = &priv->cq_table;
  316         int err;
  317 
  318         err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn);
  319         if (err)
  320                 mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn);
  321 
  322         synchronize_irq(priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq);
  323         if (priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq !=
  324             priv->eq_table.eq[MLX4_EQ_ASYNC].irq)
  325                 synchronize_irq(priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
  326 
  327         spin_lock_irq(&cq_table->lock);
  328         radix_tree_delete(&cq_table->tree, cq->cqn);
  329         spin_unlock_irq(&cq_table->lock);
  330 
  331         if (atomic_dec_and_test(&cq->refcount))
  332                 complete(&cq->free);
  333         wait_for_completion(&cq->free);
  334 
  335         mlx4_cq_free_icm(dev, cq->cqn);
  336 }
  337 EXPORT_SYMBOL_GPL(mlx4_cq_free);
  338 
  339 int mlx4_init_cq_table(struct mlx4_dev *dev)
  340 {
  341         struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
  342         int err;
  343 
  344         spin_lock_init(&cq_table->lock);
  345         INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
  346         if (mlx4_is_slave(dev))
  347                 return 0;
  348 
  349         err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs,
  350                                dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0);
  351         if (err)
  352                 return err;
  353 
  354         return 0;
  355 }
  356 
  357 void mlx4_cleanup_cq_table(struct mlx4_dev *dev)
  358 {
  359         if (mlx4_is_slave(dev))
  360                 return;
  361         /* Nothing to do to clean up radix_tree */
  362         mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap);
  363 }
Cache object: 96dc22427595e103ba18f96fed048973
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/mlx4/mlx4_core/mlx4_cq.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/mlx4/mlx4_core/mlx4_cq.c
