iser_memory.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10

    1 /* $FreeBSD$ */
    2 /*-
    3  * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #include "icl_iser.h"
   28 
   29 static struct fast_reg_descriptor *
   30 iser_reg_desc_get(struct ib_conn *ib_conn)
   31 {
   32         struct fast_reg_descriptor *desc;
   33 
   34         mtx_lock(&ib_conn->lock);
   35         desc = list_first_entry(&ib_conn->fastreg.pool,
   36                                 struct fast_reg_descriptor, list);
   37         list_del(&desc->list);
   38         mtx_unlock(&ib_conn->lock);
   39 
   40         return (desc);
   41 }
   42 
   43 static void
   44 iser_reg_desc_put(struct ib_conn *ib_conn,
   45                   struct fast_reg_descriptor *desc)
   46 {
   47         mtx_lock(&ib_conn->lock);
   48         list_add(&desc->list, &ib_conn->fastreg.pool);
   49         mtx_unlock(&ib_conn->lock);
   50 }
   51 
   52 #define IS_4K_ALIGNED(addr)     ((((unsigned long)addr) & ~MASK_4K) == 0)
   53 
   54 /**
   55  * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
   56  * for RDMA sub-list of a scatter-gather list of memory buffers, and  returns
   57  * the number of entries which are aligned correctly. Supports the case where
   58  * consecutive SG elements are actually fragments of the same physcial page.
   59  */
   60 static int
   61 iser_data_buf_aligned_len(struct iser_data_buf *data, struct ib_device *ibdev)
   62 {
   63         struct scatterlist *sg, *sgl, *next_sg = NULL;
   64         u64 start_addr, end_addr;
   65         int i, ret_len, start_check = 0;
   66 
   67         if (data->dma_nents == 1)
   68                 return (1);
   69 
   70         sgl = data->sgl;
   71         start_addr  = ib_sg_dma_address(ibdev, sgl);
   72 
   73         for_each_sg(sgl, sg, data->dma_nents, i) {
   74                 if (start_check && !IS_4K_ALIGNED(start_addr))
   75                         break;
   76 
   77                 next_sg = sg_next(sg);
   78                 if (!next_sg)
   79                         break;
   80 
   81                 end_addr    = start_addr + ib_sg_dma_len(ibdev, sg);
   82                 start_addr  = ib_sg_dma_address(ibdev, next_sg);
   83 
   84                 if (end_addr == start_addr) {
   85                         start_check = 0;
   86                         continue;
   87                 } else
   88                         start_check = 1;
   89 
   90                 if (!IS_4K_ALIGNED(end_addr))
   91                         break;
   92         }
   93         ret_len = (next_sg) ? i : i+1;
   94 
   95         return (ret_len);
   96 }
   97 
   98 void
   99 iser_dma_unmap_task_data(struct icl_iser_pdu *iser_pdu,
  100                          struct iser_data_buf *data,
  101                          enum dma_data_direction dir)
  102 {
  103         struct ib_device *dev;
  104 
  105         dev = iser_pdu->iser_conn->ib_conn.device->ib_device;
  106         ib_dma_unmap_sg(dev, data->sgl, data->size, dir);
  107 }
  108 
  109 static int
  110 iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
  111              struct iser_mem_reg *reg)
  112 {
  113         struct scatterlist *sg = mem->sgl;
  114 
  115         reg->sge.lkey = device->mr->lkey;
  116         reg->rkey = device->mr->rkey;
  117         reg->sge.length = ib_sg_dma_len(device->ib_device, &sg[0]);
  118         reg->sge.addr = ib_sg_dma_address(device->ib_device, &sg[0]);
  119 
  120         return (0);
  121 }
  122 
  123 /**
  124  * TODO: This should be a verb
  125  * iser_ib_inc_rkey - increments the key portion of the given rkey. Can be used
  126  * for calculating a new rkey for type 2 memory windows.
  127  * @rkey - the rkey to increment.
  128  */
  129 static inline u32
  130 iser_ib_inc_rkey(u32 rkey)
  131 {
  132         const u32 mask = 0x000000ff;
  133 
  134         return (((rkey + 1) & mask) | (rkey & ~mask));
  135 }
  136 
  137 static void
  138 iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr)
  139 {
  140         u32 rkey;
  141 
  142         memset(inv_wr, 0, sizeof(*inv_wr));
  143         inv_wr->opcode = IB_WR_LOCAL_INV;
  144         inv_wr->wr_id = ISER_FASTREG_LI_WRID;
  145         inv_wr->ex.invalidate_rkey = mr->rkey;
  146 
  147         rkey = iser_ib_inc_rkey(mr->rkey);
  148         ib_update_fast_reg_key(mr, rkey);
  149 }
  150 
  151 static int
  152 iser_fast_reg_mr(struct icl_iser_pdu *iser_pdu,
  153                  struct iser_data_buf *mem,
  154                  struct iser_reg_resources *rsc,
  155                  struct iser_mem_reg *reg)
  156 {
  157         struct ib_conn *ib_conn = &iser_pdu->iser_conn->ib_conn;
  158         struct iser_device *device = ib_conn->device;
  159         struct ib_mr *mr = rsc->mr;
  160         struct ib_reg_wr fastreg_wr;
  161         struct ib_send_wr inv_wr;
  162         const struct ib_send_wr *bad_wr;
  163         struct ib_send_wr *wr = NULL;
  164         int ret, n;
  165 
  166         /* if there a single dma entry, dma mr suffices */
  167         if (mem->dma_nents == 1)
  168                 return iser_reg_dma(device, mem, reg);
  169 
  170         if (!rsc->mr_valid) {
  171                 iser_inv_rkey(&inv_wr, mr);
  172                 wr = &inv_wr;
  173         }
  174 
  175         n = ib_map_mr_sg(mr, mem->sg, mem->size, NULL, SIZE_4K);
  176         if (unlikely(n != mem->size)) {
  177                 ISER_ERR("failed to map sg (%d/%d)\n", n, mem->size);
  178                 return n < 0 ? n : -EINVAL;
  179         }
  180         /* Prepare FASTREG WR */
  181         memset(&fastreg_wr, 0, sizeof(fastreg_wr));
  182         fastreg_wr.wr.opcode = IB_WR_REG_MR;
  183         fastreg_wr.wr.wr_id = ISER_FASTREG_LI_WRID;
  184         fastreg_wr.wr.num_sge = 0;
  185         fastreg_wr.mr = mr;
  186         fastreg_wr.key = mr->rkey;
  187         fastreg_wr.access = IB_ACCESS_LOCAL_WRITE  |
  188                             IB_ACCESS_REMOTE_WRITE |
  189                             IB_ACCESS_REMOTE_READ;
  190 
  191         if (!wr)
  192                 wr = &fastreg_wr.wr;
  193         else
  194                 wr->next = &fastreg_wr.wr;
  195 
  196         ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
  197         if (ret) {
  198                 ISER_ERR("fast registration failed, ret:%d", ret);
  199                 return (ret);
  200         }
  201         rsc->mr_valid = 0;
  202 
  203         reg->sge.lkey = mr->lkey;
  204         reg->rkey = mr->rkey;
  205         reg->sge.addr = mr->iova;
  206         reg->sge.length = mr->length;
  207 
  208         return (ret);
  209 }
  210 
  211 /**
  212  * iser_reg_rdma_mem - Registers memory intended for RDMA,
  213  * using Fast Registration WR (if possible) obtaining rkey and va
  214  *
  215  * returns 0 on success, errno code on failure
  216  */
  217 int
  218 iser_reg_rdma_mem(struct icl_iser_pdu *iser_pdu,
  219                   enum iser_data_dir cmd_dir)
  220 {
  221         struct ib_conn *ib_conn = &iser_pdu->iser_conn->ib_conn;
  222         struct iser_device   *device = ib_conn->device;
  223         struct ib_device     *ibdev = device->ib_device;
  224         struct iser_data_buf *mem = &iser_pdu->data[cmd_dir];
  225         struct iser_mem_reg *mem_reg = &iser_pdu->rdma_reg[cmd_dir];
  226         struct fast_reg_descriptor *desc = NULL;
  227         int err, aligned_len;
  228 
  229         aligned_len = iser_data_buf_aligned_len(mem, ibdev);
  230         if (aligned_len != mem->dma_nents) {
  231                 ISER_ERR("bounce buffer is not supported");
  232                 return 1;
  233         }
  234 
  235         if (mem->dma_nents != 1) {
  236                 desc = iser_reg_desc_get(ib_conn);
  237                 mem_reg->mem_h = desc;
  238         }
  239 
  240         err = iser_fast_reg_mr(iser_pdu, mem, desc ? &desc->rsc : NULL,
  241                                        mem_reg);
  242         if (err)
  243                 goto err_reg;
  244 
  245         return (0);
  246 
  247 err_reg:
  248         if (desc)
  249                 iser_reg_desc_put(ib_conn, desc);
  250 
  251         return (err);
  252 }
  253 
  254 void
  255 iser_unreg_rdma_mem(struct icl_iser_pdu *iser_pdu,
  256                     enum iser_data_dir cmd_dir)
  257 {
  258         struct iser_mem_reg *reg = &iser_pdu->rdma_reg[cmd_dir];
  259 
  260         if (!reg->mem_h)
  261                 return;
  262 
  263         iser_reg_desc_put(&iser_pdu->iser_conn->ib_conn,
  264                           reg->mem_h);
  265         reg->mem_h = NULL;
  266 }
  267 
  268 int
  269 iser_dma_map_task_data(struct icl_iser_pdu *iser_pdu,
  270                        struct iser_data_buf *data,
  271                        enum iser_data_dir iser_dir,
  272                        enum dma_data_direction dma_dir)
  273 {
  274         struct ib_device *dev;
  275 
  276         iser_pdu->dir[iser_dir] = 1;
  277         dev = iser_pdu->iser_conn->ib_conn.device->ib_device;
  278 
  279         data->dma_nents = ib_dma_map_sg(dev, data->sgl, data->size, dma_dir);
  280         if (data->dma_nents == 0) {
  281                 ISER_ERR("dma_map_sg failed");
  282                 return (EINVAL);
  283         }
  284 
  285         return (0);
  286 }

Cache object: a1c0dc1734dc6cc244cf891525976744

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.

FreeBSD/Linux Kernel Cross Reference sys/dev/iser/iser_memory.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/iser/iser_memory.c