The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/arm/arm/busdma_machdep.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2012 Ian Lepore
    3  * Copyright (c) 2004 Olivier Houchard
    4  * Copyright (c) 2002 Peter Grehan
    5  * Copyright (c) 1997, 1998 Justin T. Gibbs.
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions, and the following disclaimer,
   13  *    without modification, immediately at the beginning of the file.
   14  * 2. The name of the author may not be used to endorse or promote products
   15  *    derived from this software without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
   21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  *
   29  *   From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD: releng/10.2/sys/arm/arm/busdma_machdep.c 282506 2015-05-05 19:47:17Z hselasky $");
   34 
   35 /*
   36  * ARM bus dma support routines.
   37  *
   38  * XXX Things to investigate / fix some day...
   39  *  - What is the earliest that this API can be called?  Could there be any
   40  *    fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM?
   41  *  - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the
   42  *    bus_dmamap_load() function.  This code has historically (and still does)
   43  *    honor it in bus_dmamem_alloc().  If we got rid of that we could lose some
   44  *    error checking because some resource management calls would become WAITOK
   45  *    and thus "cannot fail."
   46  *  - The decisions made by _bus_dma_can_bounce() should be made once, at tag
   47  *    creation time, and the result stored in the tag.
   48  *  - It should be possible to take some shortcuts when mapping a buffer we know
   49  *    came from the uma(9) allocators based on what we know about such buffers
   50  *    (aligned, contiguous, etc).
   51  *  - The allocation of bounce pages could probably be cleaned up, then we could
   52  *    retire arm_remap_nocache().
   53  */
   54 
   55 #define _ARM32_BUS_DMA_PRIVATE
   56 #include <sys/param.h>
   57 #include <sys/systm.h>
   58 #include <sys/malloc.h>
   59 #include <sys/bus.h>
   60 #include <sys/busdma_bufalloc.h>
   61 #include <sys/interrupt.h>
   62 #include <sys/lock.h>
   63 #include <sys/proc.h>
   64 #include <sys/memdesc.h>
   65 #include <sys/mutex.h>
   66 #include <sys/ktr.h>
   67 #include <sys/kernel.h>
   68 #include <sys/sysctl.h>
   69 #include <sys/uio.h>
   70 
   71 #include <vm/uma.h>
   72 #include <vm/vm.h>
   73 #include <vm/vm_extern.h>
   74 #include <vm/vm_kern.h>
   75 #include <vm/vm_page.h>
   76 #include <vm/vm_map.h>
   77 
   78 #include <machine/atomic.h>
   79 #include <machine/bus.h>
   80 #include <machine/cpufunc.h>
   81 #include <machine/md_var.h>
   82 
   83 #define MAX_BPAGES 64
   84 #define BUS_DMA_COULD_BOUNCE    BUS_DMA_BUS3
   85 #define BUS_DMA_MIN_ALLOC_COMP  BUS_DMA_BUS4
   86 
   87 struct bounce_zone;
   88 
   89 struct bus_dma_tag {
   90         bus_dma_tag_t           parent;
   91         bus_size_t              alignment;
   92         bus_addr_t              boundary;
   93         bus_addr_t              lowaddr;
   94         bus_addr_t              highaddr;
   95         bus_dma_filter_t        *filter;
   96         void                    *filterarg;
   97         bus_size_t              maxsize;
   98         u_int                   nsegments;
   99         bus_size_t              maxsegsz;
  100         int                     flags;
  101         int                     ref_count;
  102         int                     map_count;
  103         bus_dma_lock_t          *lockfunc;
  104         void                    *lockfuncarg;
  105         /*
  106          * DMA range for this tag.  If the page doesn't fall within
  107          * one of these ranges, an error is returned.  The caller
  108          * may then decide what to do with the transfer.  If the
  109          * range pointer is NULL, it is ignored.
  110          */
  111         struct arm32_dma_range  *ranges;
  112         int                     _nranges;
  113         struct bounce_zone *bounce_zone;
  114         /*
  115          * Most tags need one or two segments, and can use the local tagsegs
  116          * array.  For tags with a larger limit, we'll allocate a bigger array
  117          * on first use.
  118          */
  119         bus_dma_segment_t       *segments;
  120         bus_dma_segment_t       tagsegs[2];
  121 };
  122 
  123 struct bounce_page {
  124         vm_offset_t     vaddr;          /* kva of bounce buffer */
  125         bus_addr_t      busaddr;        /* Physical address */
  126         vm_offset_t     datavaddr;      /* kva of client data */
  127         bus_addr_t      dataaddr;       /* client physical address */
  128         bus_size_t      datacount;      /* client data count */
  129         STAILQ_ENTRY(bounce_page) links;
  130 };
  131 
  132 struct sync_list {
  133         vm_offset_t     vaddr;          /* kva of bounce buffer */
  134         bus_addr_t      busaddr;        /* Physical address */
  135         bus_size_t      datacount;      /* client data count */
  136 };
  137 
  138 int busdma_swi_pending;
  139 
  140 struct bounce_zone {
  141         STAILQ_ENTRY(bounce_zone) links;
  142         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
  143         int             total_bpages;
  144         int             free_bpages;
  145         int             reserved_bpages;
  146         int             active_bpages;
  147         int             total_bounced;
  148         int             total_deferred;
  149         int             map_count;
  150         bus_size_t      alignment;
  151         bus_addr_t      lowaddr;
  152         char            zoneid[8];
  153         char            lowaddrid[20];
  154         struct sysctl_ctx_list sysctl_tree;
  155         struct sysctl_oid *sysctl_tree_top;
  156 };
  157 
  158 static struct mtx bounce_lock;
  159 static int total_bpages;
  160 static int busdma_zonecount;
  161 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
  162 
  163 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
  164 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
  165            "Total bounce pages");
  166 
  167 #define DMAMAP_COHERENT         0x8
  168 #define DMAMAP_CACHE_ALIGNED    0x10
  169 
  170 struct bus_dmamap {
  171         struct bp_list  bpages;
  172         int             pagesneeded;
  173         int             pagesreserved;
  174         bus_dma_tag_t   dmat;
  175         struct memdesc  mem;
  176         int             flags;
  177         STAILQ_ENTRY(bus_dmamap) links;
  178         bus_dmamap_callback_t *callback;
  179         void                  *callback_arg;
  180         int                    sync_count;
  181         struct sync_list       *slist;
  182 };
  183 
  184 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
  185 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
  186 
  187 static struct mtx busdma_mtx;
  188 
  189 MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF);
  190 
  191 static void init_bounce_pages(void *dummy);
  192 static int alloc_bounce_zone(bus_dma_tag_t dmat);
  193 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
  194 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  195                                 int commit);
  196 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
  197                                   vm_offset_t vaddr, bus_addr_t addr,
  198                                   bus_size_t size);
  199 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
  200 
  201 /* Default tag, as most drivers provide no parent tag. */
  202 bus_dma_tag_t arm_root_dma_tag;
  203 
  204 /*
  205  * ----------------------------------------------------------------------------
  206  * Begin block of code useful to transplant to other implementations.
  207  */
  208 
  209 static uma_zone_t dmamap_zone;  /* Cache of struct bus_dmamap items */
  210 
  211 static busdma_bufalloc_t coherent_allocator;    /* Cache of coherent buffers */
  212 static busdma_bufalloc_t standard_allocator;    /* Cache of standard buffers */
  213 
  214 /*
  215  * This is the ctor function passed to uma_zcreate() for the pool of dma maps.
  216  * It'll need platform-specific changes if this code is copied.
  217  */
  218 static int
  219 dmamap_ctor(void *mem, int size, void *arg, int flags)
  220 {
  221         bus_dmamap_t map;
  222         bus_dma_tag_t dmat;
  223 
  224         map = (bus_dmamap_t)mem;
  225         dmat = (bus_dma_tag_t)arg;
  226 
  227         dmat->map_count++;
  228 
  229         map->dmat = dmat;
  230         map->flags = 0;
  231         STAILQ_INIT(&map->bpages);
  232 
  233         return (0);
  234 }
  235 
  236 /*
  237  * This is the dtor function passed to uma_zcreate() for the pool of dma maps.
  238  * It may need platform-specific changes if this code is copied              .
  239  */
  240 static void 
  241 dmamap_dtor(void *mem, int size, void *arg)
  242 {
  243         bus_dmamap_t map;
  244 
  245         map = (bus_dmamap_t)mem;
  246 
  247         map->dmat->map_count--;
  248 }
  249 
  250 static void
  251 busdma_init(void *dummy)
  252 {
  253 
  254         /* Create a cache of maps for bus_dmamap_create(). */
  255         dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap),
  256             dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
  257 
  258         /* Create a cache of buffers in standard (cacheable) memory. */
  259         standard_allocator = busdma_bufalloc_create("buffer", 
  260             arm_dcache_align,   /* minimum_alignment */
  261             NULL,               /* uma_alloc func */ 
  262             NULL,               /* uma_free func */
  263             0);                 /* uma_zcreate_flags */
  264 
  265         /*
  266          * Create a cache of buffers in uncacheable memory, to implement the
  267          * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag.
  268          */
  269         coherent_allocator = busdma_bufalloc_create("coherent",
  270             arm_dcache_align,   /* minimum_alignment */
  271             busdma_bufalloc_alloc_uncacheable, 
  272             busdma_bufalloc_free_uncacheable, 
  273             0);                 /* uma_zcreate_flags */
  274 }
  275 
  276 /*
  277  * This init historically used SI_SUB_VM, but now the init code requires
  278  * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by
  279  * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using
  280  * SI_SUB_KMEM and SI_ORDER_THIRD.
  281  */
  282 SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL);
  283 
  284 /*
  285  * End block of code useful to transplant to other implementations.
  286  * ----------------------------------------------------------------------------
  287  */
  288 
  289 /*
  290  * Return true if a match is made.
  291  *
  292  * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
  293  *
  294  * If paddr is within the bounds of the dma tag then call the filter callback
  295  * to check for a match, if there is no filter callback then assume a match.
  296  */
  297 static int
  298 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
  299 {
  300         int retval;
  301 
  302         retval = 0;
  303 
  304         do {
  305                 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
  306                  || ((paddr & (dmat->alignment - 1)) != 0))
  307                  && (dmat->filter == NULL
  308                   || (*dmat->filter)(dmat->filterarg, paddr) != 0))
  309                         retval = 1;
  310 
  311                 dmat = dmat->parent;            
  312         } while (retval == 0 && dmat != NULL);
  313         return (retval);
  314 }
  315 
  316 /*
  317  * This routine checks the exclusion zone constraints from a tag against the
  318  * physical RAM available on the machine.  If a tag specifies an exclusion zone
  319  * but there's no RAM in that zone, then we avoid allocating resources to bounce
  320  * a request, and we can use any memory allocator (as opposed to needing
  321  * kmem_alloc_contig() just because it can allocate pages in an address range).
  322  *
  323  * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the
  324  * same value on 32-bit architectures) as their lowaddr constraint, and we can't
  325  * possibly have RAM at an address higher than the highest address we can
  326  * express, so we take a fast out.
  327  */
  328 static __inline int
  329 _bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr)
  330 {
  331         int i;
  332 
  333         if (lowaddr >= BUS_SPACE_MAXADDR)
  334                 return (0);
  335 
  336         for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) {
  337                 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1])
  338                     || (lowaddr < phys_avail[i] &&
  339                     highaddr > phys_avail[i]))
  340                         return (1);
  341         }
  342         return (0);
  343 }
  344 
  345 static __inline struct arm32_dma_range *
  346 _bus_dma_inrange(struct arm32_dma_range *ranges, int nranges,
  347     bus_addr_t curaddr)
  348 {
  349         struct arm32_dma_range *dr;
  350         int i;
  351 
  352         for (i = 0, dr = ranges; i < nranges; i++, dr++) {
  353                 if (curaddr >= dr->dr_sysbase &&
  354                     round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len))
  355                         return (dr);
  356         }
  357 
  358         return (NULL);
  359 }
  360 /*
  361  * Convenience function for manipulating driver locks from busdma (during
  362  * busdma_swi, for example).  Drivers that don't provide their own locks
  363  * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
  364  * non-mutex locking scheme don't have to use this at all.
  365  */
  366 void
  367 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
  368 {
  369         struct mtx *dmtx;
  370 
  371         dmtx = (struct mtx *)arg;
  372         switch (op) {
  373         case BUS_DMA_LOCK:
  374                 mtx_lock(dmtx);
  375                 break;
  376         case BUS_DMA_UNLOCK:
  377                 mtx_unlock(dmtx);
  378                 break;
  379         default:
  380                 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
  381         }
  382 }
  383 
  384 /*
  385  * dflt_lock should never get called.  It gets put into the dma tag when
  386  * lockfunc == NULL, which is only valid if the maps that are associated
  387  * with the tag are meant to never be defered.
  388  * XXX Should have a way to identify which driver is responsible here.
  389  */
  390 static void
  391 dflt_lock(void *arg, bus_dma_lock_op_t op)
  392 {
  393 #ifdef INVARIANTS
  394         panic("driver error: busdma dflt_lock called");
  395 #else
  396         printf("DRIVER_ERROR: busdma dflt_lock called\n");
  397 #endif
  398 }
  399 
  400 /*
  401  * Allocate a device specific dma_tag.
  402  */
  403 #define SEG_NB 1024
  404 
  405 int
  406 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  407                    bus_addr_t boundary, bus_addr_t lowaddr,
  408                    bus_addr_t highaddr, bus_dma_filter_t *filter,
  409                    void *filterarg, bus_size_t maxsize, int nsegments,
  410                    bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  411                    void *lockfuncarg, bus_dma_tag_t *dmat)
  412 {
  413         bus_dma_tag_t newtag;
  414         int error = 0;
  415         /* Return a NULL tag on failure */
  416         *dmat = NULL;
  417         if (!parent)
  418                 parent = arm_root_dma_tag;
  419 
  420         newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
  421         if (newtag == NULL) {
  422                 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  423                     __func__, newtag, 0, error);
  424                 return (ENOMEM);
  425         }
  426 
  427         newtag->parent = parent;
  428         newtag->alignment = alignment ? alignment : 1;
  429         newtag->boundary = boundary;
  430         newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1);
  431         newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1);
  432         newtag->filter = filter;
  433         newtag->filterarg = filterarg;
  434         newtag->maxsize = maxsize;
  435         newtag->nsegments = nsegments;
  436         newtag->maxsegsz = maxsegsz;
  437         newtag->flags = flags;
  438         newtag->ref_count = 1; /* Count ourself */
  439         newtag->map_count = 0;
  440         newtag->ranges = bus_dma_get_range();
  441         newtag->_nranges = bus_dma_get_range_nb();
  442         if (lockfunc != NULL) {
  443                 newtag->lockfunc = lockfunc;
  444                 newtag->lockfuncarg = lockfuncarg;
  445         } else {
  446                 newtag->lockfunc = dflt_lock;
  447                 newtag->lockfuncarg = NULL;
  448         }
  449         /*
  450          * If all the segments we need fit into the local tagsegs array, set the
  451          * pointer now.  Otherwise NULL the pointer and an array of segments
  452          * will be allocated later, on first use.  We don't pre-allocate now
  453          * because some tags exist just to pass contraints to children in the
  454          * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we
  455          * sure don't want to try to allocate an array for that.
  456          */
  457         if (newtag->nsegments <= nitems(newtag->tagsegs))
  458                 newtag->segments = newtag->tagsegs;
  459         else
  460                 newtag->segments = NULL;
  461         /*
  462          * Take into account any restrictions imposed by our parent tag
  463          */
  464         if (parent != NULL) {
  465                 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
  466                 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
  467                 if (newtag->boundary == 0)
  468                         newtag->boundary = parent->boundary;
  469                 else if (parent->boundary != 0)
  470                         newtag->boundary = MIN(parent->boundary,
  471                                                newtag->boundary);
  472                 if ((newtag->filter != NULL) ||
  473                     ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
  474                         newtag->flags |= BUS_DMA_COULD_BOUNCE;
  475                 if (newtag->filter == NULL) {
  476                         /*
  477                          * Short circuit looking at our parent directly
  478                          * since we have encapsulated all of its information
  479                          */
  480                         newtag->filter = parent->filter;
  481                         newtag->filterarg = parent->filterarg;
  482                         newtag->parent = parent->parent;
  483                 }
  484                 if (newtag->parent != NULL)
  485                         atomic_add_int(&parent->ref_count, 1);
  486         }
  487         if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr)
  488          || newtag->alignment > 1)
  489                 newtag->flags |= BUS_DMA_COULD_BOUNCE;
  490 
  491         if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
  492             (flags & BUS_DMA_ALLOCNOW) != 0) {
  493                 struct bounce_zone *bz;
  494 
  495                 /* Must bounce */
  496 
  497                 if ((error = alloc_bounce_zone(newtag)) != 0) {
  498                         free(newtag, M_DEVBUF);
  499                         return (error);
  500                 }
  501                 bz = newtag->bounce_zone;
  502 
  503                 if (ptoa(bz->total_bpages) < maxsize) {
  504                         int pages;
  505 
  506                         pages = atop(maxsize) - bz->total_bpages;
  507 
  508                         /* Add pages to our bounce pool */
  509                         if (alloc_bounce_pages(newtag, pages) < pages)
  510                                 error = ENOMEM;
  511                 }
  512                 /* Performed initial allocation */
  513                 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
  514         } else
  515                 newtag->bounce_zone = NULL;
  516         if (error != 0)
  517                 free(newtag, M_DEVBUF);
  518         else
  519                 *dmat = newtag;
  520         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  521             __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
  522 
  523         return (error);
  524 }
  525 
  526 int
  527 bus_dma_tag_destroy(bus_dma_tag_t dmat)
  528 {
  529 #ifdef KTR
  530         bus_dma_tag_t dmat_copy = dmat;
  531 #endif
  532 
  533         if (dmat != NULL) {
  534                 
  535                 if (dmat->map_count != 0)
  536                         return (EBUSY);
  537                 
  538                 while (dmat != NULL) {
  539                         bus_dma_tag_t parent;
  540                         
  541                         parent = dmat->parent;
  542                         atomic_subtract_int(&dmat->ref_count, 1);
  543                         if (dmat->ref_count == 0) {
  544                                 if (dmat->segments != NULL &&
  545                                     dmat->segments != dmat->tagsegs)
  546                                         free(dmat->segments, M_DEVBUF);
  547                                 free(dmat, M_DEVBUF);
  548                                 /*
  549                                  * Last reference count, so
  550                                  * release our reference
  551                                  * count on our parent.
  552                                  */
  553                                 dmat = parent;
  554                         } else
  555                                 dmat = NULL;
  556                 }
  557         }
  558         CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy);
  559 
  560         return (0);
  561 }
  562 
  563 #include <sys/kdb.h>
  564 /*
  565  * Allocate a handle for mapping from kva/uva/physical
  566  * address space into bus device space.
  567  */
  568 int
  569 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  570 {
  571         struct sync_list *slist;
  572         bus_dmamap_t map;
  573         int error = 0;
  574 
  575         slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
  576         if (slist == NULL)
  577                 return (ENOMEM);
  578 
  579         map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT);
  580         *mapp = map;
  581         if (map == NULL) {
  582                 free(slist, M_DEVBUF);
  583                 return (ENOMEM);
  584         }
  585 
  586         /*
  587          * If the tag's segments haven't been allocated yet we need to do it
  588          * now, because we can't sleep for resources at map load time.
  589          */
  590         if (dmat->segments == NULL) {
  591                 dmat->segments = malloc(dmat->nsegments * 
  592                     sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT);
  593                 if (dmat->segments == NULL) {
  594                         free(slist, M_DEVBUF);
  595                         uma_zfree(dmamap_zone, map);
  596                         *mapp = NULL;
  597                         return (ENOMEM);
  598                 }
  599         }
  600 
  601         /*
  602          * Bouncing might be required if the driver asks for an active
  603          * exclusion region, a data alignment that is stricter than 1, and/or
  604          * an active address boundary.
  605          */
  606         if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
  607 
  608                 /* Must bounce */
  609                 struct bounce_zone *bz;
  610                 int maxpages;
  611 
  612                 if (dmat->bounce_zone == NULL) {
  613                         if ((error = alloc_bounce_zone(dmat)) != 0) {
  614                                 free(slist, M_DEVBUF);
  615                                 uma_zfree(dmamap_zone, map);
  616                                 *mapp = NULL;
  617                                 return (error);
  618                         }
  619                 }
  620                 bz = dmat->bounce_zone;
  621 
  622                 /* Initialize the new map */
  623                 STAILQ_INIT(&((*mapp)->bpages));
  624 
  625                 /*
  626                  * Attempt to add pages to our pool on a per-instance
  627                  * basis up to a sane limit.
  628                  */
  629                 maxpages = MAX_BPAGES;
  630                 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
  631                  || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
  632                         int pages;
  633 
  634                         pages = MAX(atop(dmat->maxsize), 1);
  635                         pages = MIN(maxpages - bz->total_bpages, pages);
  636                         pages = MAX(pages, 1);
  637                         if (alloc_bounce_pages(dmat, pages) < pages)
  638                                 error = ENOMEM;
  639 
  640                         if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
  641                                 if (error == 0)
  642                                         dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
  643                         } else {
  644                                 error = 0;
  645                         }
  646                 }
  647                 bz->map_count++;
  648         }
  649         map->sync_count = 0;
  650         map->slist = slist;
  651         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  652             __func__, dmat, dmat->flags, error);
  653 
  654         return (0);
  655 }
  656 
  657 /*
  658  * Destroy a handle for mapping from kva/uva/physical
  659  * address space into bus device space.
  660  */
  661 int
  662 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
  663 {
  664 
  665         if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
  666                 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  667                     __func__, dmat, EBUSY);
  668                 return (EBUSY);
  669         }
  670         free(map->slist, M_DEVBUF);
  671         uma_zfree(dmamap_zone, map);
  672         if (dmat->bounce_zone)
  673                 dmat->bounce_zone->map_count--;
  674         CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
  675         return (0);
  676 }
  677 
  678 /*
  679  * Allocate a piece of memory that can be efficiently mapped into bus device
  680  * space based on the constraints listed in the dma tag.  Returns a pointer to
  681  * the allocated memory, and a pointer to an associated bus_dmamap.
  682  */
  683 int
  684 bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags,
  685                  bus_dmamap_t *mapp)
  686 {
  687         struct sync_list *slist;
  688         void * vaddr;
  689         struct busdma_bufzone *bufzone;
  690         busdma_bufalloc_t ba;
  691         bus_dmamap_t map;
  692         int mflags;
  693         vm_memattr_t memattr;
  694 
  695         if (flags & BUS_DMA_NOWAIT)
  696                 mflags = M_NOWAIT;
  697         else
  698                 mflags = M_WAITOK;
  699         /*
  700          * If the tag's segments haven't been allocated yet we need to do it
  701          * now, because we can't sleep for resources at map load time.
  702          */
  703         if (dmat->segments == NULL)
  704                 dmat->segments = malloc(dmat->nsegments * 
  705                    sizeof(*dmat->segments), M_DEVBUF, mflags);
  706 
  707         slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
  708         if (slist == NULL)
  709                 return (ENOMEM);
  710         map = uma_zalloc_arg(dmamap_zone, dmat, mflags);
  711         if (map == NULL) {
  712                 free(slist, M_DEVBUF);
  713                 return (ENOMEM);
  714         }
  715         if (flags & BUS_DMA_COHERENT) {
  716                 memattr = VM_MEMATTR_UNCACHEABLE;
  717                 ba = coherent_allocator;
  718                 map->flags |= DMAMAP_COHERENT;
  719         } else {
  720                 memattr = VM_MEMATTR_DEFAULT;
  721                 ba = standard_allocator;
  722         }
  723         /* All buffers we allocate are cache-aligned. */
  724         map->flags |= DMAMAP_CACHE_ALIGNED;
  725 
  726         if (flags & BUS_DMA_ZERO)
  727                 mflags |= M_ZERO;
  728 
  729         /*
  730          * Try to find a bufzone in the allocator that holds a cache of buffers
  731          * of the right size for this request.  If the buffer is too big to be
  732          * held in the allocator cache, this returns NULL.
  733          */
  734         bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
  735 
  736         /*
  737          * Allocate the buffer from the uma(9) allocator if...
  738          *  - It's small enough to be in the allocator (bufzone not NULL).
  739          *  - The alignment constraint isn't larger than the allocation size
  740          *    (the allocator aligns buffers to their size boundaries).
  741          *  - There's no need to handle lowaddr/highaddr exclusion zones.
  742          * else allocate non-contiguous pages if...
  743          *  - The page count that could get allocated doesn't exceed nsegments.
  744          *  - The alignment constraint isn't larger than a page boundary.
  745          *  - There are no boundary-crossing constraints.
  746          * else allocate a block of contiguous pages because one or more of the
  747          * constraints is something that only the contig allocator can fulfill.
  748          */
  749         if (bufzone != NULL && dmat->alignment <= bufzone->size &&
  750             !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) {
  751                 vaddr = uma_zalloc(bufzone->umazone, mflags);
  752         } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
  753             dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) {
  754                 vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
  755                     mflags, 0, dmat->lowaddr, memattr);
  756         } else {
  757                 vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
  758                     mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary,
  759                     memattr);
  760         }
  761         if (vaddr == NULL) {
  762                 free(slist, M_DEVBUF);
  763                 uma_zfree(dmamap_zone, map);
  764                 map = NULL;
  765         } else {
  766                 map->slist = slist;
  767                 map->sync_count = 0;
  768         }
  769         *vaddrp = vaddr;
  770         *mapp = map;
  771 
  772         return (vaddr == NULL ? ENOMEM : 0);
  773 }
  774 
  775 /*
  776  * Free a piece of memory that was allocated via bus_dmamem_alloc, along with
  777  * its associated map.
  778  */
  779 void
  780 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
  781 {
  782         struct busdma_bufzone *bufzone;
  783         busdma_bufalloc_t ba;
  784 
  785         if (map->flags & DMAMAP_COHERENT)
  786                 ba = coherent_allocator;
  787         else
  788                 ba = standard_allocator;
  789         uma_zfree(dmamap_zone, map);
  790 
  791         free(map->slist, M_DEVBUF);
  792         /* Be careful not to access map from here on. */
  793 
  794         bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
  795 
  796         if (bufzone != NULL && dmat->alignment <= bufzone->size &&
  797             !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
  798                 uma_zfree(bufzone->umazone, vaddr);
  799         else
  800                 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
  801 }
  802 
  803 static void
  804 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
  805     bus_size_t buflen, int flags)
  806 {
  807         bus_addr_t curaddr;
  808         bus_size_t sgsize;
  809 
  810         if (map->pagesneeded == 0) {
  811                 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
  812                     dmat->lowaddr, dmat->boundary, dmat->alignment);
  813                 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
  814                     map, map->pagesneeded);
  815                 /*
  816                  * Count the number of bounce pages
  817                  * needed in order to complete this transfer
  818                  */
  819                 curaddr = buf;
  820                 while (buflen != 0) {
  821                         sgsize = MIN(buflen, dmat->maxsegsz);
  822                         if (run_filter(dmat, curaddr) != 0) {
  823                                 sgsize = MIN(sgsize, PAGE_SIZE);
  824                                 map->pagesneeded++;
  825                         }
  826                         curaddr += sgsize;
  827                         buflen -= sgsize;
  828                 }
  829                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
  830         }
  831 }
  832 
  833 static void
  834 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
  835     void *buf, bus_size_t buflen, int flags)
  836 {
  837         vm_offset_t vaddr;
  838         vm_offset_t vendaddr;
  839         bus_addr_t paddr;
  840 
  841         if (map->pagesneeded == 0) {
  842                 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
  843                     dmat->lowaddr, dmat->boundary, dmat->alignment);
  844                 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
  845                     map, map->pagesneeded);
  846                 /*
  847                  * Count the number of bounce pages
  848                  * needed in order to complete this transfer
  849                  */
  850                 vaddr = trunc_page((vm_offset_t)buf);
  851                 vendaddr = (vm_offset_t)buf + buflen;
  852 
  853                 while (vaddr < vendaddr) {
  854                         if (__predict_true(pmap == kernel_pmap))
  855                                 paddr = pmap_kextract(vaddr);
  856                         else
  857                                 paddr = pmap_extract(pmap, vaddr);
  858                         if (run_filter(dmat, paddr) != 0)
  859                                 map->pagesneeded++;
  860                         vaddr += PAGE_SIZE;
  861                 }
  862                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
  863         }
  864 }
  865 
  866 static int
  867 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
  868 {
  869 
  870         /* Reserve Necessary Bounce Pages */
  871         mtx_lock(&bounce_lock);
  872         if (flags & BUS_DMA_NOWAIT) {
  873                 if (reserve_bounce_pages(dmat, map, 0) != 0) {
  874                         mtx_unlock(&bounce_lock);
  875                         return (ENOMEM);
  876                 }
  877         } else {
  878                 if (reserve_bounce_pages(dmat, map, 1) != 0) {
  879                         /* Queue us for resources */
  880                         STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
  881                         mtx_unlock(&bounce_lock);
  882                         return (EINPROGRESS);
  883                 }
  884         }
  885         mtx_unlock(&bounce_lock);
  886 
  887         return (0);
  888 }
  889 
  890 /*
  891  * Add a single contiguous physical range to the segment list.
  892  */
  893 static int
  894 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
  895     bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
  896 {
  897         bus_addr_t baddr, bmask;
  898         int seg;
  899 
  900         /*
  901          * Make sure we don't cross any boundaries.
  902          */
  903         bmask = ~(dmat->boundary - 1);
  904         if (dmat->boundary > 0) {
  905                 baddr = (curaddr + dmat->boundary) & bmask;
  906                 if (sgsize > (baddr - curaddr))
  907                         sgsize = (baddr - curaddr);
  908         }
  909         if (dmat->ranges) {
  910                 struct arm32_dma_range *dr;
  911 
  912                 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
  913                     curaddr);
  914                 if (dr == NULL)
  915                         return (0);
  916                 /*
  917                  * In a valid DMA range.  Translate the physical
  918                  * memory address to an address in the DMA window.
  919                  */
  920                 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
  921                                         
  922         }
  923 
  924         seg = *segp;
  925         /*
  926          * Insert chunk into a segment, coalescing with
  927          * the previous segment if possible.
  928          */
  929         if (seg >= 0 &&
  930             curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
  931             (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
  932             (dmat->boundary == 0 ||
  933              (segs[seg].ds_addr & bmask) == (curaddr & bmask))) {
  934                 segs[seg].ds_len += sgsize;
  935         } else {
  936                 if (++seg >= dmat->nsegments)
  937                         return (0);
  938                 segs[seg].ds_addr = curaddr;
  939                 segs[seg].ds_len = sgsize;
  940         }
  941         *segp = seg;
  942         return (sgsize);
  943 }
  944 
  945 /*
  946  * Utility function to load a physical buffer.  segp contains
  947  * the starting segment on entrace, and the ending segment on exit.
  948  */
  949 int
  950 _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
  951     bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
  952 {
  953         bus_size_t sgsize;
  954         bus_addr_t curaddr;
  955         int error;
  956 
  957         if (segs == NULL)
  958                 segs = dmat->segments;
  959 
  960         if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
  961                 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
  962                 if (map->pagesneeded != 0) {
  963                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
  964                         if (error)
  965                                 return (error);
  966                 }
  967         }
  968 
  969         while (buflen > 0) {
  970                 curaddr = buf;
  971                 sgsize = MIN(buflen, dmat->maxsegsz);
  972                 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
  973                     map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
  974                         sgsize = MIN(sgsize, PAGE_SIZE);
  975                         curaddr = add_bounce_page(dmat, map, 0, curaddr,
  976                             sgsize);
  977                 }
  978                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
  979                     segp);
  980                 if (sgsize == 0)
  981                         break;
  982                 buf += sgsize;
  983                 buflen -= sgsize;
  984         }
  985 
  986         /*
  987          * Did we fit?
  988          */
  989         if (buflen != 0) {
  990                 _bus_dmamap_unload(dmat, map);
  991                 return (EFBIG); /* XXX better return value here? */
  992         }
  993         return (0);
  994 }
  995 
  996 int
  997 _bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
  998     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
  999     bus_dma_segment_t *segs, int *segp)
 1000 {
 1001 
 1002         return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
 1003             segs, segp));
 1004 }
 1005 
 1006 /*
 1007  * Utility function to load a linear buffer.  segp contains
 1008  * the starting segment on entrance, and the ending segment on exit.
 1009  */
 1010 int
 1011 _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 1012     bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,
 1013     int *segp)
 1014 {
 1015         bus_size_t sgsize;
 1016         bus_addr_t curaddr;
 1017         struct sync_list *sl;
 1018         vm_offset_t vaddr = (vm_offset_t)buf;
 1019         int error = 0;
 1020 
 1021         if (segs == NULL)
 1022                 segs = dmat->segments;
 1023         if ((flags & BUS_DMA_LOAD_MBUF) != 0)
 1024                 map->flags |= DMAMAP_CACHE_ALIGNED;
 1025 
 1026         if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
 1027                 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
 1028                 if (map->pagesneeded != 0) {
 1029                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
 1030                         if (error)
 1031                                 return (error);
 1032                 }
 1033         }
 1034         CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
 1035             "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
 1036 
 1037         while (buflen > 0) {
 1038                 /*
 1039                  * Get the physical address for this segment.
 1040                  */
 1041                 if (__predict_true(pmap == kernel_pmap)) {
 1042                         curaddr = pmap_kextract(vaddr);
 1043                 } else {
 1044                         curaddr = pmap_extract(pmap, vaddr);
 1045                         map->flags &= ~DMAMAP_COHERENT;
 1046                 }
 1047 
 1048                 /*
 1049                  * Compute the segment size, and adjust counts.
 1050                  */
 1051                 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
 1052                 if (sgsize > dmat->maxsegsz)
 1053                         sgsize = dmat->maxsegsz;
 1054                 if (buflen < sgsize)
 1055                         sgsize = buflen;
 1056 
 1057                 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
 1058                     map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
 1059                         curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
 1060                             sgsize);
 1061                 } else {
 1062                         sl = &map->slist[map->sync_count - 1];
 1063                         if (map->sync_count == 0 ||
 1064                             vaddr != sl->vaddr + sl->datacount) {
 1065                                 if (++map->sync_count > dmat->nsegments)
 1066                                         goto cleanup;
 1067                                 sl++;
 1068                                 sl->vaddr = vaddr;
 1069                                 sl->datacount = sgsize;
 1070                                 sl->busaddr = curaddr;
 1071                         } else
 1072                                 sl->datacount += sgsize;
 1073                 }
 1074                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 1075                     segp);
 1076                 if (sgsize == 0)
 1077                         break;
 1078                 vaddr += sgsize;
 1079                 buflen -= sgsize;
 1080         }
 1081 
 1082 cleanup:
 1083         /*
 1084          * Did we fit?
 1085          */
 1086         if (buflen != 0) {
 1087                 _bus_dmamap_unload(dmat, map);
 1088                 return (EFBIG); /* XXX better return value here? */
 1089         }
 1090         return (0);
 1091 }
 1092 
 1093 void
 1094 __bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
 1095                     struct memdesc *mem, bus_dmamap_callback_t *callback,
 1096                     void *callback_arg)
 1097 {
 1098 
 1099         KASSERT(dmat != NULL, ("dmatag is NULL"));
 1100         KASSERT(map != NULL, ("dmamap is NULL"));
 1101         map->mem = *mem;
 1102         map->callback = callback;
 1103         map->callback_arg = callback_arg;
 1104 }
 1105 
 1106 bus_dma_segment_t *
 1107 _bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
 1108                      bus_dma_segment_t *segs, int nsegs, int error)
 1109 {
 1110 
 1111         if (segs == NULL)
 1112                 segs = dmat->segments;
 1113         return (segs);
 1114 }
 1115 
 1116 /*
 1117  * Release the mapping held by map.
 1118  */
 1119 void
 1120 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
 1121 {
 1122         struct bounce_page *bpage;
 1123 
 1124         while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
 1125                 STAILQ_REMOVE_HEAD(&map->bpages, links);
 1126                 free_bounce_page(dmat, bpage);
 1127         }
 1128         map->sync_count = 0;
 1129         return;
 1130 }
 1131 
 1132 static void
 1133 bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op,
 1134     int bufaligned)
 1135 {
 1136         char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
 1137         register_t s;
 1138         int partial;
 1139 
 1140         if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) {
 1141                 cpu_dcache_wb_range(buf, len);
 1142                 cpu_l2cache_wb_range(buf, len);
 1143         }
 1144 
 1145         /*
 1146          * If the caller promises the buffer is properly aligned to a cache line
 1147          * (even if the call parms make it look like it isn't) we can avoid
 1148          * attempting to preserve the non-DMA part of the cache line in the
 1149          * POSTREAD case, but we MUST still do a writeback in the PREREAD case.
 1150          *
 1151          * This covers the case of mbufs, where we know how they're aligned and
 1152          * know the CPU doesn't touch the header in front of the DMA data area
 1153          * during the IO, but it may have touched it right before invoking the
 1154          * sync, so a PREREAD writeback is required.
 1155          *
 1156          * It also handles buffers we created in bus_dmamem_alloc(), which are
 1157          * always aligned and padded to cache line size even if the IO length
 1158          * isn't a multiple of cache line size.  In this case the PREREAD
 1159          * writeback probably isn't required, but it's harmless.
 1160          */
 1161         partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask;
 1162 
 1163         if (op & BUS_DMASYNC_PREREAD) {
 1164                 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) {
 1165                         cpu_dcache_inv_range(buf, len);
 1166                         cpu_l2cache_inv_range(buf, len);
 1167                 } else {
 1168                         cpu_dcache_wbinv_range(buf, len);
 1169                         cpu_l2cache_wbinv_range(buf, len);
 1170                 }
 1171         }
 1172         if (op & BUS_DMASYNC_POSTREAD) {
 1173                 if (partial && !bufaligned) {
 1174                         s = intr_disable();
 1175                         if (buf & arm_dcache_align_mask)
 1176                                 memcpy(_tmp_cl, (void *)(buf &
 1177                                     ~arm_dcache_align_mask),
 1178                                     buf & arm_dcache_align_mask);
 1179                         if ((buf + len) & arm_dcache_align_mask)
 1180                                 memcpy(_tmp_clend,
 1181                                     (void *)(buf + len),
 1182                                     arm_dcache_align -
 1183                                     ((buf + len) & arm_dcache_align_mask));
 1184                 }
 1185                 cpu_dcache_inv_range(buf, len);
 1186                 cpu_l2cache_inv_range(buf, len);
 1187                 if (partial && !bufaligned) {
 1188                         if (buf & arm_dcache_align_mask)
 1189                                 memcpy((void *)(buf &
 1190                                     ~arm_dcache_align_mask), _tmp_cl,
 1191                                     buf & arm_dcache_align_mask);
 1192                         if ((buf + len) & arm_dcache_align_mask)
 1193                                 memcpy((void *)(buf + len),
 1194                                     _tmp_clend, arm_dcache_align -
 1195                                     ((buf + len) & arm_dcache_align_mask));
 1196                         intr_restore(s);
 1197                 }
 1198         }
 1199 }
 1200 
 1201 static void
 1202 _bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 1203 {
 1204         struct bounce_page *bpage;
 1205 
 1206         STAILQ_FOREACH(bpage, &map->bpages, links) {
 1207                 if (op & BUS_DMASYNC_PREWRITE) {
 1208                         if (bpage->datavaddr != 0)
 1209                                 bcopy((void *)bpage->datavaddr, 
 1210                                     (void *)bpage->vaddr, bpage->datacount);
 1211                         else
 1212                                 physcopyout(bpage->dataaddr,
 1213                                     (void *)bpage->vaddr,bpage->datacount);
 1214                         cpu_dcache_wb_range(bpage->vaddr, bpage->datacount);
 1215                         cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount);
 1216                         dmat->bounce_zone->total_bounced++;
 1217                 }
 1218                 if (op & BUS_DMASYNC_POSTREAD) {
 1219                         cpu_dcache_inv_range(bpage->vaddr, bpage->datacount);
 1220                         cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount);
 1221                         if (bpage->datavaddr != 0)
 1222                                 bcopy((void *)bpage->vaddr,
 1223                                     (void *)bpage->datavaddr, bpage->datacount);
 1224                         else
 1225                                 physcopyin((void *)bpage->vaddr,
 1226                                     bpage->dataaddr, bpage->datacount);
 1227                         dmat->bounce_zone->total_bounced++;
 1228                 }
 1229         }
 1230 }
 1231 
 1232 void
 1233 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 1234 {
 1235         struct sync_list *sl, *end;
 1236         int bufaligned;
 1237 
 1238         if (op == BUS_DMASYNC_POSTWRITE)
 1239                 return;
 1240         if (map->flags & DMAMAP_COHERENT)
 1241                 goto drain;
 1242         if (STAILQ_FIRST(&map->bpages))
 1243                 _bus_dmamap_sync_bp(dmat, map, op);
 1244         CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
 1245         bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED);
 1246         if (map->sync_count) {
 1247                 end = &map->slist[map->sync_count];
 1248                 for (sl = &map->slist[0]; sl != end; sl++)
 1249                         bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op,
 1250                             bufaligned);
 1251         }
 1252 
 1253 drain:
 1254 
 1255         cpu_drain_writebuf();
 1256 }
 1257 
 1258 static void
 1259 init_bounce_pages(void *dummy __unused)
 1260 {
 1261 
 1262         total_bpages = 0;
 1263         STAILQ_INIT(&bounce_zone_list);
 1264         STAILQ_INIT(&bounce_map_waitinglist);
 1265         STAILQ_INIT(&bounce_map_callbacklist);
 1266         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
 1267 }
 1268 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
 1269 
 1270 static struct sysctl_ctx_list *
 1271 busdma_sysctl_tree(struct bounce_zone *bz)
 1272 {
 1273         return (&bz->sysctl_tree);
 1274 }
 1275 
 1276 static struct sysctl_oid *
 1277 busdma_sysctl_tree_top(struct bounce_zone *bz)
 1278 {
 1279         return (bz->sysctl_tree_top);
 1280 }
 1281 
 1282 static int
 1283 alloc_bounce_zone(bus_dma_tag_t dmat)
 1284 {
 1285         struct bounce_zone *bz;
 1286 
 1287         /* Check to see if we already have a suitable zone */
 1288         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
 1289                 if ((dmat->alignment <= bz->alignment)
 1290                  && (dmat->lowaddr >= bz->lowaddr)) {
 1291                         dmat->bounce_zone = bz;
 1292                         return (0);
 1293                 }
 1294         }
 1295 
 1296         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
 1297             M_NOWAIT | M_ZERO)) == NULL)
 1298                 return (ENOMEM);
 1299 
 1300         STAILQ_INIT(&bz->bounce_page_list);
 1301         bz->free_bpages = 0;
 1302         bz->reserved_bpages = 0;
 1303         bz->active_bpages = 0;
 1304         bz->lowaddr = dmat->lowaddr;
 1305         bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
 1306         bz->map_count = 0;
 1307         snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
 1308         busdma_zonecount++;
 1309         snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
 1310         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
 1311         dmat->bounce_zone = bz;
 1312 
 1313         sysctl_ctx_init(&bz->sysctl_tree);
 1314         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
 1315             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
 1316             CTLFLAG_RD, 0, "");
 1317         if (bz->sysctl_tree_top == NULL) {
 1318                 sysctl_ctx_free(&bz->sysctl_tree);
 1319                 return (0);     /* XXX error code? */
 1320         }
 1321 
 1322         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1323             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1324             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
 1325             "Total bounce pages");
 1326         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1327             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1328             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
 1329             "Free bounce pages");
 1330         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1331             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1332             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
 1333             "Reserved bounce pages");
 1334         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1335             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1336             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
 1337             "Active bounce pages");
 1338         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1339             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1340             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
 1341             "Total bounce requests");
 1342         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1343             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1344             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
 1345             "Total bounce requests that were deferred");
 1346         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
 1347             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1348             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
 1349         SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz),
 1350             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1351             "alignment", CTLFLAG_RD, &bz->alignment, "");
 1352 
 1353         return (0);
 1354 }
 1355 
 1356 static int
 1357 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
 1358 {
 1359         struct bounce_zone *bz;
 1360         int count;
 1361 
 1362         bz = dmat->bounce_zone;
 1363         count = 0;
 1364         while (numpages > 0) {
 1365                 struct bounce_page *bpage;
 1366 
 1367                 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
 1368                                                      M_NOWAIT | M_ZERO);
 1369 
 1370                 if (bpage == NULL)
 1371                         break;
 1372                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
 1373                                                          M_NOWAIT, 0ul,
 1374                                                          bz->lowaddr,
 1375                                                          PAGE_SIZE,
 1376                                                          0);
 1377                 if (bpage->vaddr == 0) {
 1378                         free(bpage, M_DEVBUF);
 1379                         break;
 1380                 }
 1381                 bpage->busaddr = pmap_kextract(bpage->vaddr);
 1382                 mtx_lock(&bounce_lock);
 1383                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
 1384                 total_bpages++;
 1385                 bz->total_bpages++;
 1386                 bz->free_bpages++;
 1387                 mtx_unlock(&bounce_lock);
 1388                 count++;
 1389                 numpages--;
 1390         }
 1391         return (count);
 1392 }
 1393 
 1394 static int
 1395 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
 1396 {
 1397         struct bounce_zone *bz;
 1398         int pages;
 1399 
 1400         mtx_assert(&bounce_lock, MA_OWNED);
 1401         bz = dmat->bounce_zone;
 1402         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
 1403         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
 1404                 return (map->pagesneeded - (map->pagesreserved + pages));
 1405         bz->free_bpages -= pages;
 1406         bz->reserved_bpages += pages;
 1407         map->pagesreserved += pages;
 1408         pages = map->pagesneeded - map->pagesreserved;
 1409 
 1410         return (pages);
 1411 }
 1412 
 1413 static bus_addr_t
 1414 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 1415                 bus_addr_t addr, bus_size_t size)
 1416 {
 1417         struct bounce_zone *bz;
 1418         struct bounce_page *bpage;
 1419 
 1420         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
 1421         KASSERT(map != NULL, ("add_bounce_page: bad map %p", map));
 1422 
 1423         bz = dmat->bounce_zone;
 1424         if (map->pagesneeded == 0)
 1425                 panic("add_bounce_page: map doesn't need any pages");
 1426         map->pagesneeded--;
 1427 
 1428         if (map->pagesreserved == 0)
 1429                 panic("add_bounce_page: map doesn't need any pages");
 1430         map->pagesreserved--;
 1431 
 1432         mtx_lock(&bounce_lock);
 1433         bpage = STAILQ_FIRST(&bz->bounce_page_list);
 1434         if (bpage == NULL)
 1435                 panic("add_bounce_page: free page list is empty");
 1436 
 1437         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
 1438         bz->reserved_bpages--;
 1439         bz->active_bpages++;
 1440         mtx_unlock(&bounce_lock);
 1441 
 1442         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1443                 /* Page offset needs to be preserved. */
 1444                 bpage->vaddr |= addr & PAGE_MASK;
 1445                 bpage->busaddr |= addr & PAGE_MASK;
 1446         }
 1447         bpage->datavaddr = vaddr;
 1448         bpage->dataaddr = addr;
 1449         bpage->datacount = size;
 1450         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
 1451         return (bpage->busaddr);
 1452 }
 1453 
 1454 static void
 1455 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
 1456 {
 1457         struct bus_dmamap *map;
 1458         struct bounce_zone *bz;
 1459 
 1460         bz = dmat->bounce_zone;
 1461         bpage->datavaddr = 0;
 1462         bpage->datacount = 0;
 1463         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1464                 /*
 1465                  * Reset the bounce page to start at offset 0.  Other uses
 1466                  * of this bounce page may need to store a full page of
 1467                  * data and/or assume it starts on a page boundary.
 1468                  */
 1469                 bpage->vaddr &= ~PAGE_MASK;
 1470                 bpage->busaddr &= ~PAGE_MASK;
 1471         }
 1472 
 1473         mtx_lock(&bounce_lock);
 1474         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
 1475         bz->free_bpages++;
 1476         bz->active_bpages--;
 1477         if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
 1478                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
 1479                         STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
 1480                         STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
 1481                                            map, links);
 1482                         busdma_swi_pending = 1;
 1483                         bz->total_deferred++;
 1484                         swi_sched(vm_ih, 0);
 1485                 }
 1486         }
 1487         mtx_unlock(&bounce_lock);
 1488 }
 1489 
 1490 void
 1491 busdma_swi(void)
 1492 {
 1493         bus_dma_tag_t dmat;
 1494         struct bus_dmamap *map;
 1495 
 1496         mtx_lock(&bounce_lock);
 1497         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
 1498                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
 1499                 mtx_unlock(&bounce_lock);
 1500                 dmat = map->dmat;
 1501                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
 1502                 bus_dmamap_load_mem(map->dmat, map, &map->mem,
 1503                     map->callback, map->callback_arg, BUS_DMA_WAITOK);
 1504                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
 1505                 mtx_lock(&bounce_lock);
 1506         }
 1507         mtx_unlock(&bounce_lock);
 1508 }

Cache object: 5a78333819854e0f9e1524283a532e0b


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.