xdma.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause
    3  *
    4  * Copyright (c) 2016-2019 Ruslan Bukin <br@bsdpad.com>
    5  *
    6  * This software was developed by SRI International and the University of
    7  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
    8  * ("CTSRD"), as part of the DARPA CRASH research programme.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   29  * SUCH DAMAGE.
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 
   35 #include "opt_platform.h"
   36 #include <sys/param.h>
   37 #include <sys/conf.h>
   38 #include <sys/bus.h>
   39 #include <sys/epoch.h>
   40 #include <sys/kernel.h>
   41 #include <sys/queue.h>
   42 #include <sys/kobj.h>
   43 #include <sys/malloc.h>
   44 #include <sys/limits.h>
   45 #include <sys/lock.h>
   46 #include <sys/mutex.h>
   47 #include <sys/sysctl.h>
   48 #include <sys/systm.h>
   49 
   50 #include <machine/bus.h>
   51 
   52 #ifdef FDT
   53 #include <dev/fdt/fdt_common.h>
   54 #include <dev/ofw/ofw_bus.h>
   55 #include <dev/ofw/ofw_bus_subr.h>
   56 #endif
   57 
   58 #include <dev/xdma/xdma.h>
   59 
   60 #include <xdma_if.h>
   61 
   62 /*
   63  * Multiple xDMA controllers may work with single DMA device,
   64  * so we have global lock for physical channel management.
   65  */
   66 static struct mtx xdma_mtx;
   67 
   68 #define XDMA_LOCK()                     mtx_lock(&xdma_mtx)
   69 #define XDMA_UNLOCK()                   mtx_unlock(&xdma_mtx)
   70 #define XDMA_ASSERT_LOCKED()            mtx_assert(&xdma_mtx, MA_OWNED)
   71 
   72 #define FDT_REG_CELLS   4
   73 
   74 #ifdef FDT
   75 static int
   76 xdma_get_iommu_fdt(xdma_controller_t *xdma, xdma_channel_t *xchan)
   77 {
   78         struct xdma_iommu *xio;
   79         phandle_t node;
   80         pcell_t prop;
   81         size_t len;
   82 
   83         node = ofw_bus_get_node(xdma->dma_dev);
   84         if (OF_getproplen(node, "xdma,iommu") <= 0)
   85                 return (0);
   86 
   87         len = OF_getencprop(node, "xdma,iommu", &prop, sizeof(prop));
   88         if (len != sizeof(prop)) {
   89                 device_printf(xdma->dev,
   90                     "%s: Can't get iommu device node\n", __func__);
   91                 return (0);
   92         }
   93 
   94         xio = &xchan->xio;
   95         xio->dev = OF_device_from_xref(prop);
   96         if (xio->dev == NULL) {
   97                 device_printf(xdma->dev,
   98                     "%s: Can't get iommu device\n", __func__);
   99                 return (0);
  100         }
  101 
  102         /* Found */
  103         return (1);
  104 }
  105 #endif
  106 
  107 /*
  108  * Allocate virtual xDMA channel.
  109  */
  110 xdma_channel_t *
  111 xdma_channel_alloc(xdma_controller_t *xdma, uint32_t caps)
  112 {
  113         xdma_channel_t *xchan;
  114         int ret;
  115 
  116         xchan = malloc(sizeof(xdma_channel_t), M_XDMA, M_WAITOK | M_ZERO);
  117         xchan->xdma = xdma;
  118 
  119 #ifdef FDT
  120         /* Check if this DMA controller supports IOMMU. */
  121         if (xdma_get_iommu_fdt(xdma, xchan))
  122                 caps |= XCHAN_CAP_IOMMU | XCHAN_CAP_NOSEG;
  123 #endif
  124 
  125         xchan->caps = caps;
  126 
  127         XDMA_LOCK();
  128 
  129         /* Request a real channel from hardware driver. */
  130         ret = XDMA_CHANNEL_ALLOC(xdma->dma_dev, xchan);
  131         if (ret != 0) {
  132                 device_printf(xdma->dev,
  133                     "%s: Can't request hardware channel.\n", __func__);
  134                 XDMA_UNLOCK();
  135                 free(xchan, M_XDMA);
  136 
  137                 return (NULL);
  138         }
  139 
  140         TAILQ_INIT(&xchan->ie_handlers);
  141 
  142         mtx_init(&xchan->mtx_lock, "xDMA chan", NULL, MTX_DEF);
  143         mtx_init(&xchan->mtx_qin_lock, "xDMA qin", NULL, MTX_DEF);
  144         mtx_init(&xchan->mtx_qout_lock, "xDMA qout", NULL, MTX_DEF);
  145         mtx_init(&xchan->mtx_bank_lock, "xDMA bank", NULL, MTX_DEF);
  146         mtx_init(&xchan->mtx_proc_lock, "xDMA proc", NULL, MTX_DEF);
  147 
  148         TAILQ_INIT(&xchan->bank);
  149         TAILQ_INIT(&xchan->queue_in);
  150         TAILQ_INIT(&xchan->queue_out);
  151         TAILQ_INIT(&xchan->processing);
  152 
  153         if (xchan->caps & XCHAN_CAP_IOMMU)
  154                 xdma_iommu_init(&xchan->xio);
  155 
  156         TAILQ_INSERT_TAIL(&xdma->channels, xchan, xchan_next);
  157 
  158         XDMA_UNLOCK();
  159 
  160         return (xchan);
  161 }
  162 
  163 int
  164 xdma_channel_free(xdma_channel_t *xchan)
  165 {
  166         xdma_controller_t *xdma;
  167         int err;
  168 
  169         xdma = xchan->xdma;
  170         KASSERT(xdma != NULL, ("xdma is NULL"));
  171 
  172         XDMA_LOCK();
  173 
  174         /* Free the real DMA channel. */
  175         err = XDMA_CHANNEL_FREE(xdma->dma_dev, xchan);
  176         if (err != 0) {
  177                 device_printf(xdma->dev,
  178                     "%s: Can't free real hw channel.\n", __func__);
  179                 XDMA_UNLOCK();
  180                 return (-1);
  181         }
  182 
  183         if (xchan->flags & XCHAN_TYPE_SG)
  184                 xdma_channel_free_sg(xchan);
  185 
  186         if (xchan->caps & XCHAN_CAP_IOMMU)
  187                 xdma_iommu_release(&xchan->xio);
  188 
  189         xdma_teardown_all_intr(xchan);
  190 
  191         mtx_destroy(&xchan->mtx_lock);
  192         mtx_destroy(&xchan->mtx_qin_lock);
  193         mtx_destroy(&xchan->mtx_qout_lock);
  194         mtx_destroy(&xchan->mtx_bank_lock);
  195         mtx_destroy(&xchan->mtx_proc_lock);
  196 
  197         TAILQ_REMOVE(&xdma->channels, xchan, xchan_next);
  198 
  199         free(xchan, M_XDMA);
  200 
  201         XDMA_UNLOCK();
  202 
  203         return (0);
  204 }
  205 
  206 int
  207 xdma_setup_intr(xdma_channel_t *xchan, int flags,
  208     int (*cb)(void *, xdma_transfer_status_t *),
  209     void *arg, void **ihandler)
  210 {
  211         struct xdma_intr_handler *ih;
  212         xdma_controller_t *xdma;
  213 
  214         xdma = xchan->xdma;
  215         KASSERT(xdma != NULL, ("xdma is NULL"));
  216 
  217         /* Sanity check. */
  218         if (cb == NULL) {
  219                 device_printf(xdma->dev,
  220                     "%s: Can't setup interrupt handler.\n",
  221                     __func__);
  222 
  223                 return (-1);
  224         }
  225 
  226         ih = malloc(sizeof(struct xdma_intr_handler),
  227             M_XDMA, M_WAITOK | M_ZERO);
  228         ih->flags = flags;
  229         ih->cb = cb;
  230         ih->cb_user = arg;
  231 
  232         XCHAN_LOCK(xchan);
  233         TAILQ_INSERT_TAIL(&xchan->ie_handlers, ih, ih_next);
  234         XCHAN_UNLOCK(xchan);
  235 
  236         if (ihandler != NULL)
  237                 *ihandler = ih;
  238 
  239         return (0);
  240 }
  241 
  242 int
  243 xdma_teardown_intr(xdma_channel_t *xchan, struct xdma_intr_handler *ih)
  244 {
  245         xdma_controller_t *xdma;
  246 
  247         xdma = xchan->xdma;
  248         KASSERT(xdma != NULL, ("xdma is NULL"));
  249 
  250         /* Sanity check. */
  251         if (ih == NULL) {
  252                 device_printf(xdma->dev,
  253                     "%s: Can't teardown interrupt.\n", __func__);
  254                 return (-1);
  255         }
  256 
  257         TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next);
  258         free(ih, M_XDMA);
  259 
  260         return (0);
  261 }
  262 
  263 int
  264 xdma_teardown_all_intr(xdma_channel_t *xchan)
  265 {
  266         struct xdma_intr_handler *ih_tmp;
  267         struct xdma_intr_handler *ih;
  268 
  269         KASSERT(xchan->xdma != NULL, ("xdma is NULL"));
  270 
  271         TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) {
  272                 TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next);
  273                 free(ih, M_XDMA);
  274         }
  275 
  276         return (0);
  277 }
  278 
  279 int
  280 xdma_request(xdma_channel_t *xchan, struct xdma_request *req)
  281 {
  282         xdma_controller_t *xdma;
  283         int ret;
  284 
  285         xdma = xchan->xdma;
  286 
  287         KASSERT(xdma != NULL, ("xdma is NULL"));
  288 
  289         XCHAN_LOCK(xchan);
  290         ret = XDMA_CHANNEL_REQUEST(xdma->dma_dev, xchan, req);
  291         if (ret != 0) {
  292                 device_printf(xdma->dev,
  293                     "%s: Can't request a transfer.\n", __func__);
  294                 XCHAN_UNLOCK(xchan);
  295 
  296                 return (-1);
  297         }
  298         XCHAN_UNLOCK(xchan);
  299 
  300         return (0);
  301 }
  302 
  303 int
  304 xdma_control(xdma_channel_t *xchan, enum xdma_command cmd)
  305 {
  306         xdma_controller_t *xdma;
  307         int ret;
  308 
  309         xdma = xchan->xdma;
  310         KASSERT(xdma != NULL, ("xdma is NULL"));
  311 
  312         ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, cmd);
  313         if (ret != 0) {
  314                 device_printf(xdma->dev,
  315                     "%s: Can't process command.\n", __func__);
  316                 return (-1);
  317         }
  318 
  319         return (0);
  320 }
  321 
  322 void
  323 xdma_callback(xdma_channel_t *xchan, xdma_transfer_status_t *status)
  324 {
  325         struct xdma_intr_handler *ih_tmp;
  326         struct xdma_intr_handler *ih;
  327         struct epoch_tracker et;
  328 
  329         KASSERT(xchan->xdma != NULL, ("xdma is NULL"));
  330 
  331         TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) {
  332                 if (ih->cb != NULL) {
  333                         if (ih->flags & XDMA_INTR_NET)
  334                                 NET_EPOCH_ENTER(et);
  335                         ih->cb(ih->cb_user, status);
  336                         if (ih->flags & XDMA_INTR_NET)
  337                                 NET_EPOCH_EXIT(et);
  338                 }
  339         }
  340 
  341         if (xchan->flags & XCHAN_TYPE_SG)
  342                 xdma_queue_submit(xchan);
  343 }
  344 
  345 #ifdef FDT
  346 /*
  347  * Notify the DMA driver we have machine-dependent data in FDT.
  348  */
  349 static int
  350 xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cells, int ncells)
  351 {
  352         uint32_t ret;
  353 
  354         ret = XDMA_OFW_MD_DATA(xdma->dma_dev,
  355             cells, ncells, (void **)&xdma->data);
  356 
  357         return (ret);
  358 }
  359 
  360 int
  361 xdma_handle_mem_node(vmem_t *vmem, phandle_t memory)
  362 {
  363         pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS];
  364         pcell_t *regp;
  365         int addr_cells, size_cells;
  366         int i, reg_len, ret, tuple_size, tuples;
  367         u_long mem_start, mem_size;
  368 
  369         if ((ret = fdt_addrsize_cells(OF_parent(memory), &addr_cells,
  370             &size_cells)) != 0)
  371                 return (ret);
  372 
  373         if (addr_cells > 2)
  374                 return (ERANGE);
  375 
  376         tuple_size = sizeof(pcell_t) * (addr_cells + size_cells);
  377         reg_len = OF_getproplen(memory, "reg");
  378         if (reg_len <= 0 || reg_len > sizeof(reg))
  379                 return (ERANGE);
  380 
  381         if (OF_getprop(memory, "reg", reg, reg_len) <= 0)
  382                 return (ENXIO);
  383 
  384         tuples = reg_len / tuple_size;
  385         regp = (pcell_t *)&reg;
  386         for (i = 0; i < tuples; i++) {
  387                 ret = fdt_data_to_res(regp, addr_cells, size_cells,
  388                     &mem_start, &mem_size);
  389                 if (ret != 0)
  390                         return (ret);
  391 
  392                 vmem_add(vmem, mem_start, mem_size, 0);
  393                 regp += addr_cells + size_cells;
  394         }
  395 
  396         return (0);
  397 }
  398 
  399 vmem_t *
  400 xdma_get_memory(device_t dev)
  401 {
  402         phandle_t mem_node, node;
  403         pcell_t mem_handle;
  404         vmem_t *vmem;
  405 
  406         node = ofw_bus_get_node(dev);
  407         if (node <= 0) {
  408                 device_printf(dev,
  409                     "%s called on not ofw based device.\n", __func__);
  410                 return (NULL);
  411         }
  412 
  413         if (!OF_hasprop(node, "memory-region"))
  414                 return (NULL);
  415 
  416         if (OF_getencprop(node, "memory-region", (void *)&mem_handle,
  417             sizeof(mem_handle)) <= 0)
  418                 return (NULL);
  419 
  420         vmem = vmem_create("xDMA vmem", 0, 0, PAGE_SIZE,
  421             PAGE_SIZE, M_BESTFIT | M_WAITOK);
  422         if (vmem == NULL)
  423                 return (NULL);
  424 
  425         mem_node = OF_node_from_xref(mem_handle);
  426         if (xdma_handle_mem_node(vmem, mem_node) != 0) {
  427                 vmem_destroy(vmem);
  428                 return (NULL);
  429         }
  430 
  431         return (vmem);
  432 }
  433 
  434 void
  435 xdma_put_memory(vmem_t *vmem)
  436 {
  437 
  438         vmem_destroy(vmem);
  439 }
  440 
  441 void
  442 xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem)
  443 {
  444 
  445         xchan->vmem = vmem;
  446 }
  447 
  448 /*
  449  * Allocate xdma controller.
  450  */
  451 xdma_controller_t *
  452 xdma_ofw_get(device_t dev, const char *prop)
  453 {
  454         phandle_t node, parent;
  455         xdma_controller_t *xdma;
  456         device_t dma_dev;
  457         pcell_t *cells;
  458         int ncells;
  459         int error;
  460         int ndmas;
  461         int idx;
  462 
  463         node = ofw_bus_get_node(dev);
  464         if (node <= 0)
  465                 device_printf(dev,
  466                     "%s called on not ofw based device.\n", __func__);
  467 
  468         error = ofw_bus_parse_xref_list_get_length(node,
  469             "dmas", "#dma-cells", &ndmas);
  470         if (error) {
  471                 device_printf(dev,
  472                     "%s can't get dmas list.\n", __func__);
  473                 return (NULL);
  474         }
  475 
  476         if (ndmas == 0) {
  477                 device_printf(dev,
  478                     "%s dmas list is empty.\n", __func__);
  479                 return (NULL);
  480         }
  481 
  482         error = ofw_bus_find_string_index(node, "dma-names", prop, &idx);
  483         if (error != 0) {
  484                 device_printf(dev,
  485                     "%s can't find string index.\n", __func__);
  486                 return (NULL);
  487         }
  488 
  489         error = ofw_bus_parse_xref_list_alloc(node, "dmas", "#dma-cells",
  490             idx, &parent, &ncells, &cells);
  491         if (error != 0) {
  492                 device_printf(dev,
  493                     "%s can't get dma device xref.\n", __func__);
  494                 return (NULL);
  495         }
  496 
  497         dma_dev = OF_device_from_xref(parent);
  498         if (dma_dev == NULL) {
  499                 device_printf(dev,
  500                     "%s can't get dma device.\n", __func__);
  501                 return (NULL);
  502         }
  503 
  504         xdma = malloc(sizeof(struct xdma_controller),
  505             M_XDMA, M_WAITOK | M_ZERO);
  506         xdma->dev = dev;
  507         xdma->dma_dev = dma_dev;
  508 
  509         TAILQ_INIT(&xdma->channels);
  510 
  511         xdma_ofw_md_data(xdma, cells, ncells);
  512         free(cells, M_OFWPROP);
  513 
  514         return (xdma);
  515 }
  516 #endif
  517 
  518 /*
  519  * Allocate xdma controller.
  520  */
  521 xdma_controller_t *
  522 xdma_get(device_t dev, device_t dma_dev)
  523 {
  524         xdma_controller_t *xdma;
  525 
  526         xdma = malloc(sizeof(struct xdma_controller),
  527             M_XDMA, M_WAITOK | M_ZERO);
  528         xdma->dev = dev;
  529         xdma->dma_dev = dma_dev;
  530 
  531         TAILQ_INIT(&xdma->channels);
  532 
  533         return (xdma);
  534 }
  535 
  536 /*
  537  * Free xDMA controller object.
  538  */
  539 int
  540 xdma_put(xdma_controller_t *xdma)
  541 {
  542 
  543         XDMA_LOCK();
  544 
  545         /* Ensure no channels allocated. */
  546         if (!TAILQ_EMPTY(&xdma->channels)) {
  547                 device_printf(xdma->dev, "%s: Can't free xDMA\n", __func__);
  548                 return (-1);
  549         }
  550 
  551         free(xdma->data, M_DEVBUF);
  552         free(xdma, M_XDMA);
  553 
  554         XDMA_UNLOCK();
  555 
  556         return (0);
  557 }
  558 
  559 static void
  560 xdma_init(void)
  561 {
  562 
  563         mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF);
  564 }
  565 
  566 SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL);
Cache object: 8712736c5e944dc3851400dfc868cb22
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/xdma/xdma.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/xdma/xdma.c
