The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/arm/arm/busdma_machdep-v6.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2012-2014 Ian Lepore
    3  * Copyright (c) 2010 Mark Tinguely
    4  * Copyright (c) 2004 Olivier Houchard
    5  * Copyright (c) 2002 Peter Grehan
    6  * Copyright (c) 1997, 1998 Justin T. Gibbs.
    7  * All rights reserved.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions, and the following disclaimer,
   14  *    without modification, immediately at the beginning of the file.
   15  * 2. The name of the author may not be used to endorse or promote products
   16  *    derived from this software without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
   22  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  *  From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb
   31  */
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD: releng/10.2/sys/arm/arm/busdma_machdep-v6.c 282506 2015-05-05 19:47:17Z hselasky $");
   35 
   36 #define _ARM32_BUS_DMA_PRIVATE
   37 #include <sys/param.h>
   38 #include <sys/kdb.h>
   39 #include <ddb/ddb.h>
   40 #include <ddb/db_output.h>
   41 #include <sys/systm.h>
   42 #include <sys/malloc.h>
   43 #include <sys/bus.h>
   44 #include <sys/busdma_bufalloc.h>
   45 #include <sys/interrupt.h>
   46 #include <sys/kernel.h>
   47 #include <sys/ktr.h>
   48 #include <sys/lock.h>
   49 #include <sys/memdesc.h>
   50 #include <sys/proc.h>
   51 #include <sys/mutex.h>
   52 #include <sys/sysctl.h>
   53 #include <sys/uio.h>
   54 
   55 #include <vm/vm.h>
   56 #include <vm/vm_page.h>
   57 #include <vm/vm_map.h>
   58 #include <vm/vm_extern.h>
   59 #include <vm/vm_kern.h>
   60 
   61 #include <machine/atomic.h>
   62 #include <machine/bus.h>
   63 #include <machine/cpufunc.h>
   64 #include <machine/md_var.h>
   65 
   66 #define MAX_BPAGES 64
   67 #define MAX_DMA_SEGMENTS        4096
   68 #define BUS_DMA_EXCL_BOUNCE     BUS_DMA_BUS2
   69 #define BUS_DMA_ALIGN_BOUNCE    BUS_DMA_BUS3
   70 #define BUS_DMA_COULD_BOUNCE    (BUS_DMA_EXCL_BOUNCE | BUS_DMA_ALIGN_BOUNCE)
   71 #define BUS_DMA_MIN_ALLOC_COMP  BUS_DMA_BUS4
   72 
   73 struct bounce_zone;
   74 
   75 struct bus_dma_tag {
   76         bus_dma_tag_t     parent;
   77         bus_size_t        alignment;
   78         bus_size_t        boundary;
   79         bus_addr_t        lowaddr;
   80         bus_addr_t        highaddr;
   81         bus_dma_filter_t *filter;
   82         void             *filterarg;
   83         bus_size_t        maxsize;
   84         u_int             nsegments;
   85         bus_size_t        maxsegsz;
   86         int               flags;
   87         int               ref_count;
   88         int               map_count;
   89         bus_dma_lock_t   *lockfunc;
   90         void             *lockfuncarg;
   91         struct bounce_zone *bounce_zone;
   92         /*
   93          * DMA range for this tag.  If the page doesn't fall within
   94          * one of these ranges, an error is returned.  The caller
   95          * may then decide what to do with the transfer.  If the
   96          * range pointer is NULL, it is ignored.
   97          */
   98         struct arm32_dma_range  *ranges;
   99         int                     _nranges;
  100 };
  101 
  102 struct bounce_page {
  103         vm_offset_t     vaddr;          /* kva of bounce buffer */
  104         bus_addr_t      busaddr;        /* Physical address */
  105         vm_offset_t     datavaddr;      /* kva of client data */
  106         bus_addr_t      dataaddr;       /* client physical address */
  107         bus_size_t      datacount;      /* client data count */
  108         STAILQ_ENTRY(bounce_page) links;
  109 };
  110 
  111 struct sync_list {
  112         vm_offset_t     vaddr;          /* kva of bounce buffer */
  113         bus_addr_t      busaddr;        /* Physical address */
  114         bus_size_t      datacount;      /* client data count */
  115 };
  116 
  117 int busdma_swi_pending;
  118 
  119 struct bounce_zone {
  120         STAILQ_ENTRY(bounce_zone) links;
  121         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
  122         int             total_bpages;
  123         int             free_bpages;
  124         int             reserved_bpages;
  125         int             active_bpages;
  126         int             total_bounced;
  127         int             total_deferred;
  128         int             map_count;
  129         bus_size_t      alignment;
  130         bus_addr_t      lowaddr;
  131         char            zoneid[8];
  132         char            lowaddrid[20];
  133         struct sysctl_ctx_list sysctl_tree;
  134         struct sysctl_oid *sysctl_tree_top;
  135 };
  136 
  137 static struct mtx bounce_lock;
  138 static int total_bpages;
  139 static int busdma_zonecount;
  140 static uint32_t tags_total;
  141 static uint32_t maps_total;
  142 static uint32_t maps_dmamem;
  143 static uint32_t maps_coherent;
  144 static uint64_t maploads_total;
  145 static uint64_t maploads_bounced;
  146 static uint64_t maploads_coherent;
  147 static uint64_t maploads_dmamem;
  148 static uint64_t maploads_mbuf;
  149 static uint64_t maploads_physmem;
  150 
  151 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
  152 
  153 SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
  154 SYSCTL_UINT(_hw_busdma, OID_AUTO, tags_total, CTLFLAG_RD, &tags_total, 0,
  155            "Number of active tags");
  156 SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_total, CTLFLAG_RD, &maps_total, 0,
  157            "Number of active maps");
  158 SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_dmamem, CTLFLAG_RD, &maps_dmamem, 0,
  159            "Number of active maps for bus_dmamem_alloc buffers");
  160 SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_coherent, CTLFLAG_RD, &maps_coherent, 0,
  161            "Number of active maps with BUS_DMA_COHERENT flag set");
  162 SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_total, CTLFLAG_RD, &maploads_total, 0,
  163            "Number of load operations performed");
  164 SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_bounced, CTLFLAG_RD, &maploads_bounced, 0,
  165            "Number of load operations that used bounce buffers");
  166 SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_coherent, CTLFLAG_RD, &maploads_dmamem, 0,
  167            "Number of load operations on BUS_DMA_COHERENT memory");
  168 SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_dmamem, CTLFLAG_RD, &maploads_dmamem, 0,
  169            "Number of load operations on bus_dmamem_alloc buffers");
  170 SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_mbuf, CTLFLAG_RD, &maploads_mbuf, 0,
  171            "Number of load operations for mbufs");
  172 SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_physmem, CTLFLAG_RD, &maploads_physmem, 0,
  173            "Number of load operations on physical buffers");
  174 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
  175            "Total bounce pages");
  176 
  177 struct bus_dmamap {
  178         struct bp_list         bpages;
  179         int                    pagesneeded;
  180         int                    pagesreserved;
  181         bus_dma_tag_t          dmat;
  182         struct memdesc         mem;
  183         pmap_t                 pmap;
  184         bus_dmamap_callback_t *callback;
  185         void                  *callback_arg;
  186         int                   flags;
  187 #define DMAMAP_COHERENT         (1 << 0)
  188 #define DMAMAP_DMAMEM_ALLOC     (1 << 1)
  189 #define DMAMAP_MBUF             (1 << 2)
  190         STAILQ_ENTRY(bus_dmamap) links;
  191         bus_dma_segment_t       *segments;
  192         int                    sync_count;
  193         struct sync_list       slist[];
  194 };
  195 
  196 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
  197 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
  198 
  199 static void init_bounce_pages(void *dummy);
  200 static int alloc_bounce_zone(bus_dma_tag_t dmat);
  201 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
  202 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  203                                 int commit);
  204 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
  205                                   vm_offset_t vaddr, bus_addr_t addr,
  206                                   bus_size_t size);
  207 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
  208 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  209     void *buf, bus_size_t buflen, int flags);
  210 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
  211     vm_paddr_t buf, bus_size_t buflen, int flags);
  212 static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  213     int flags);
  214 
  215 static busdma_bufalloc_t coherent_allocator;    /* Cache of coherent buffers */
  216 static busdma_bufalloc_t standard_allocator;    /* Cache of standard buffers */
  217 static void
  218 busdma_init(void *dummy)
  219 {
  220         int uma_flags;
  221 
  222         uma_flags = 0;
  223 
  224         /* Create a cache of buffers in standard (cacheable) memory. */
  225         standard_allocator = busdma_bufalloc_create("buffer", 
  226             arm_dcache_align,   /* minimum_alignment */
  227             NULL,               /* uma_alloc func */ 
  228             NULL,               /* uma_free func */
  229             uma_flags);         /* uma_zcreate_flags */
  230 
  231 #ifdef INVARIANTS
  232         /* 
  233          * Force UMA zone to allocate service structures like
  234          * slabs using own allocator. uma_debug code performs
  235          * atomic ops on uma_slab_t fields and safety of this
  236          * operation is not guaranteed for write-back caches
  237          */
  238         uma_flags = UMA_ZONE_OFFPAGE;
  239 #endif
  240         /*
  241          * Create a cache of buffers in uncacheable memory, to implement the
  242          * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag.
  243          */
  244         coherent_allocator = busdma_bufalloc_create("coherent",
  245             arm_dcache_align,   /* minimum_alignment */
  246             busdma_bufalloc_alloc_uncacheable, 
  247             busdma_bufalloc_free_uncacheable, 
  248             uma_flags); /* uma_zcreate_flags */
  249 }
  250 
  251 /*
  252  * This init historically used SI_SUB_VM, but now the init code requires
  253  * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by
  254  * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using
  255  * SI_SUB_KMEM and SI_ORDER_THIRD.
  256  */
  257 SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL);
  258 
  259 static int
  260 exclusion_bounce_check(vm_offset_t lowaddr, vm_offset_t highaddr)
  261 {
  262         int i;
  263         for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) {
  264                 if ((lowaddr >= phys_avail[i] && lowaddr < phys_avail[i + 1]) ||
  265                     (lowaddr < phys_avail[i] && highaddr >= phys_avail[i]))
  266                         return (1);
  267         }
  268         return (0);
  269 }
  270 
  271 /*
  272  * Return true if the tag has an exclusion zone that could lead to bouncing.
  273  */
  274 static __inline int
  275 exclusion_bounce(bus_dma_tag_t dmat)
  276 {
  277 
  278         return (dmat->flags & BUS_DMA_EXCL_BOUNCE);
  279 }
  280 
  281 /*
  282  * Return true if the given address does not fall on the alignment boundary.
  283  */
  284 static __inline int
  285 alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr)
  286 {
  287 
  288         return (addr & (dmat->alignment - 1));
  289 }
  290 
  291 /*
  292  * Return true if the DMA should bounce because the start or end does not fall
  293  * on a cacheline boundary (which would require a partial cacheline flush).
  294  * COHERENT memory doesn't trigger cacheline flushes.  Memory allocated by
  295  * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a
  296  * strict rule that such memory cannot be accessed by the CPU while DMA is in
  297  * progress (or by multiple DMA engines at once), so that it's always safe to do
  298  * full cacheline flushes even if that affects memory outside the range of a
  299  * given DMA operation that doesn't involve the full allocated buffer.  If we're
  300  * mapping an mbuf, that follows the same rules as a buffer we allocated.
  301  */
  302 static __inline int
  303 cacheline_bounce(bus_dmamap_t map, bus_addr_t addr, bus_size_t size)
  304 {
  305 
  306         if (map->flags & (DMAMAP_DMAMEM_ALLOC | DMAMAP_COHERENT | DMAMAP_MBUF))
  307                 return (0);
  308         return ((addr | size) & arm_dcache_align_mask);
  309 }
  310 
  311 /*
  312  * Return true if we might need to bounce the DMA described by addr and size.
  313  *
  314  * This is used to quick-check whether we need to do the more expensive work of
  315  * checking the DMA page-by-page looking for alignment and exclusion bounces.
  316  *
  317  * Note that the addr argument might be either virtual or physical.  It doesn't
  318  * matter because we only look at the low-order bits, which are the same in both
  319  * address spaces.
  320  */
  321 static __inline int
  322 might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t addr, 
  323     bus_size_t size)
  324 {
  325 
  326         return ((dmat->flags & BUS_DMA_EXCL_BOUNCE) ||
  327             alignment_bounce(dmat, addr) ||
  328             cacheline_bounce(map, addr, size));
  329 }
  330 
  331 /*
  332  * Return true if we must bounce the DMA described by paddr and size.
  333  *
  334  * Bouncing can be triggered by DMA that doesn't begin and end on cacheline
  335  * boundaries, or doesn't begin on an alignment boundary, or falls within the
  336  * exclusion zone of any tag in the ancestry chain.
  337  *
  338  * For exclusions, walk the chain of tags comparing paddr to the exclusion zone
  339  * within each tag.  If the tag has a filter function, use it to decide whether
  340  * the DMA needs to bounce, otherwise any DMA within the zone bounces.
  341  */
  342 static int
  343 must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 
  344     bus_size_t size)
  345 {
  346 
  347         if (cacheline_bounce(map, paddr, size))
  348                 return (1);
  349 
  350         /*
  351          *  The tag already contains ancestors' alignment restrictions so this
  352          *  check doesn't need to be inside the loop.
  353          */
  354         if (alignment_bounce(dmat, paddr))
  355                 return (1);
  356 
  357         /*
  358          * Even though each tag has an exclusion zone that is a superset of its
  359          * own and all its ancestors' exclusions, the exclusion zone of each tag
  360          * up the chain must be checked within the loop, because the busdma
  361          * rules say the filter function is called only when the address lies
  362          * within the low-highaddr range of the tag that filterfunc belongs to.
  363          */
  364         while (dmat != NULL && exclusion_bounce(dmat)) {
  365                 if ((paddr >= dmat->lowaddr && paddr <= dmat->highaddr) &&
  366                     (dmat->filter == NULL || 
  367                     dmat->filter(dmat->filterarg, paddr) != 0))
  368                         return (1);
  369                 dmat = dmat->parent;
  370         } 
  371 
  372         return (0);
  373 }
  374 
  375 static __inline struct arm32_dma_range *
  376 _bus_dma_inrange(struct arm32_dma_range *ranges, int nranges,
  377     bus_addr_t curaddr)
  378 {
  379         struct arm32_dma_range *dr;
  380         int i;
  381 
  382         for (i = 0, dr = ranges; i < nranges; i++, dr++) {
  383                 if (curaddr >= dr->dr_sysbase &&
  384                     round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len))
  385                         return (dr);
  386         }
  387 
  388         return (NULL);
  389 }
  390 
  391 /*
  392  * Convenience function for manipulating driver locks from busdma (during
  393  * busdma_swi, for example).  Drivers that don't provide their own locks
  394  * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
  395  * non-mutex locking scheme don't have to use this at all.
  396  */
  397 void
  398 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
  399 {
  400         struct mtx *dmtx;
  401 
  402         dmtx = (struct mtx *)arg;
  403         switch (op) {
  404         case BUS_DMA_LOCK:
  405                 mtx_lock(dmtx);
  406                 break;
  407         case BUS_DMA_UNLOCK:
  408                 mtx_unlock(dmtx);
  409                 break;
  410         default:
  411                 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
  412         }
  413 }
  414 
  415 /*
  416  * dflt_lock should never get called.  It gets put into the dma tag when
  417  * lockfunc == NULL, which is only valid if the maps that are associated
  418  * with the tag are meant to never be defered.
  419  * XXX Should have a way to identify which driver is responsible here.
  420  */
  421 static void
  422 dflt_lock(void *arg, bus_dma_lock_op_t op)
  423 {
  424 
  425         panic("driver error: busdma dflt_lock called");
  426 }
  427 
  428 /*
  429  * Allocate a device specific dma_tag.
  430  */
  431 int
  432 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  433                    bus_size_t boundary, bus_addr_t lowaddr,
  434                    bus_addr_t highaddr, bus_dma_filter_t *filter,
  435                    void *filterarg, bus_size_t maxsize, int nsegments,
  436                    bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  437                    void *lockfuncarg, bus_dma_tag_t *dmat)
  438 {
  439         bus_dma_tag_t newtag;
  440         int error = 0;
  441 
  442 #if 0
  443         if (!parent)
  444                 parent = arm_root_dma_tag;
  445 #endif
  446 
  447         /* Basic sanity checking */
  448         if (boundary != 0 && boundary < maxsegsz)
  449                 maxsegsz = boundary;
  450 
  451         /* Return a NULL tag on failure */
  452         *dmat = NULL;
  453 
  454         if (maxsegsz == 0) {
  455                 return (EINVAL);
  456         }
  457 
  458         newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
  459             M_ZERO | M_NOWAIT);
  460         if (newtag == NULL) {
  461                 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  462                     __func__, newtag, 0, error);
  463                 return (ENOMEM);
  464         }
  465 
  466         newtag->parent = parent;
  467         newtag->alignment = alignment;
  468         newtag->boundary = boundary;
  469         newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
  470         newtag->highaddr = trunc_page((vm_paddr_t)highaddr) +
  471             (PAGE_SIZE - 1);
  472         newtag->filter = filter;
  473         newtag->filterarg = filterarg;
  474         newtag->maxsize = maxsize;
  475         newtag->nsegments = nsegments;
  476         newtag->maxsegsz = maxsegsz;
  477         newtag->flags = flags;
  478         newtag->ref_count = 1; /* Count ourself */
  479         newtag->map_count = 0;
  480         newtag->ranges = bus_dma_get_range();
  481         newtag->_nranges = bus_dma_get_range_nb();
  482         if (lockfunc != NULL) {
  483                 newtag->lockfunc = lockfunc;
  484                 newtag->lockfuncarg = lockfuncarg;
  485         } else {
  486                 newtag->lockfunc = dflt_lock;
  487                 newtag->lockfuncarg = NULL;
  488         }
  489 
  490         /* Take into account any restrictions imposed by our parent tag */
  491         if (parent != NULL) {
  492                 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
  493                 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
  494                 newtag->alignment = MAX(parent->alignment, newtag->alignment);
  495                 newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE;
  496                 if (newtag->boundary == 0)
  497                         newtag->boundary = parent->boundary;
  498                 else if (parent->boundary != 0)
  499                         newtag->boundary = MIN(parent->boundary,
  500                                                newtag->boundary);
  501                 if (newtag->filter == NULL) {
  502                         /*
  503                          * Short circuit to looking at our parent directly
  504                          * since we have encapsulated all of its information
  505                          */
  506                         newtag->filter = parent->filter;
  507                         newtag->filterarg = parent->filterarg;
  508                         newtag->parent = parent->parent;
  509                 }
  510                 if (newtag->parent != NULL)
  511                         atomic_add_int(&parent->ref_count, 1);
  512         }
  513 
  514         if (exclusion_bounce_check(newtag->lowaddr, newtag->highaddr))
  515                 newtag->flags |= BUS_DMA_EXCL_BOUNCE;
  516         if (alignment_bounce(newtag, 1))
  517                 newtag->flags |= BUS_DMA_ALIGN_BOUNCE;
  518 
  519         /*
  520          * Any request can auto-bounce due to cacheline alignment, in addition
  521          * to any alignment or boundary specifications in the tag, so if the
  522          * ALLOCNOW flag is set, there's always work to do.
  523          */
  524         if ((flags & BUS_DMA_ALLOCNOW) != 0) {
  525                 struct bounce_zone *bz;
  526                 /*
  527                  * Round size up to a full page, and add one more page because
  528                  * there can always be one more boundary crossing than the
  529                  * number of pages in a transfer.
  530                  */
  531                 maxsize = roundup2(maxsize, PAGE_SIZE) + PAGE_SIZE;
  532                 
  533                 if ((error = alloc_bounce_zone(newtag)) != 0) {
  534                         free(newtag, M_DEVBUF);
  535                         return (error);
  536                 }
  537                 bz = newtag->bounce_zone;
  538 
  539                 if (ptoa(bz->total_bpages) < maxsize) {
  540                         int pages;
  541 
  542                         pages = atop(maxsize) - bz->total_bpages;
  543 
  544                         /* Add pages to our bounce pool */
  545                         if (alloc_bounce_pages(newtag, pages) < pages)
  546                                 error = ENOMEM;
  547                 }
  548                 /* Performed initial allocation */
  549                 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
  550         } else
  551                 newtag->bounce_zone = NULL;
  552 
  553         if (error != 0) {
  554                 free(newtag, M_DEVBUF);
  555         } else {
  556                 atomic_add_32(&tags_total, 1);
  557                 *dmat = newtag;
  558         }
  559         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  560             __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
  561         return (error);
  562 }
  563 
  564 int
  565 bus_dma_tag_destroy(bus_dma_tag_t dmat)
  566 {
  567         bus_dma_tag_t dmat_copy;
  568         int error;
  569 
  570         error = 0;
  571         dmat_copy = dmat;
  572 
  573         if (dmat != NULL) {
  574 
  575                 if (dmat->map_count != 0) {
  576                         error = EBUSY;
  577                         goto out;
  578                 }
  579 
  580                 while (dmat != NULL) {
  581                         bus_dma_tag_t parent;
  582 
  583                         parent = dmat->parent;
  584                         atomic_subtract_int(&dmat->ref_count, 1);
  585                         if (dmat->ref_count == 0) {
  586                                 atomic_subtract_32(&tags_total, 1);
  587                                 free(dmat, M_DEVBUF);
  588                                 /*
  589                                  * Last reference count, so
  590                                  * release our reference
  591                                  * count on our parent.
  592                                  */
  593                                 dmat = parent;
  594                         } else
  595                                 dmat = NULL;
  596                 }
  597         }
  598 out:
  599         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
  600         return (error);
  601 }
  602 
  603 static int allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp)
  604 {
  605         struct bounce_zone *bz;
  606         int maxpages;
  607         int error;
  608                 
  609         if (dmat->bounce_zone == NULL)
  610                 if ((error = alloc_bounce_zone(dmat)) != 0)
  611                         return (error);
  612         bz = dmat->bounce_zone;
  613         /* Initialize the new map */
  614         STAILQ_INIT(&(mapp->bpages));
  615 
  616         /*
  617          * Attempt to add pages to our pool on a per-instance basis up to a sane
  618          * limit.  Even if the tag isn't flagged as COULD_BOUNCE due to
  619          * alignment and boundary constraints, it could still auto-bounce due to
  620          * cacheline alignment, which requires at most two bounce pages.
  621          */
  622         if (dmat->flags & BUS_DMA_COULD_BOUNCE)
  623                 maxpages = MAX_BPAGES;
  624         else
  625                 maxpages = 2 * bz->map_count;
  626         if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 ||
  627             (bz->map_count > 0 && bz->total_bpages < maxpages)) {
  628                 int pages;
  629                 
  630                 pages = atop(roundup2(dmat->maxsize, PAGE_SIZE)) + 1;
  631                 pages = MIN(maxpages - bz->total_bpages, pages);
  632                 pages = MAX(pages, 2);
  633                 if (alloc_bounce_pages(dmat, pages) < pages)
  634                         return (ENOMEM);
  635                 
  636                 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0)
  637                         dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
  638         }
  639         bz->map_count++;
  640         return (0);
  641 }
  642 
  643 static bus_dmamap_t
  644 allocate_map(bus_dma_tag_t dmat, int mflags)
  645 {
  646         int mapsize, segsize;
  647         bus_dmamap_t map;
  648 
  649         /*
  650          * Allocate the map.  The map structure ends with an embedded
  651          * variable-sized array of sync_list structures.  Following that
  652          * we allocate enough extra space to hold the array of bus_dma_segments.
  653          */
  654         KASSERT(dmat->nsegments <= MAX_DMA_SEGMENTS, 
  655            ("cannot allocate %u dma segments (max is %u)",
  656             dmat->nsegments, MAX_DMA_SEGMENTS));
  657         segsize = sizeof(struct bus_dma_segment) * dmat->nsegments;
  658         mapsize = sizeof(*map) + sizeof(struct sync_list) * dmat->nsegments;
  659         map = malloc(mapsize + segsize, M_DEVBUF, mflags | M_ZERO);
  660         if (map == NULL) {
  661                 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM);
  662                 return (NULL);
  663         }
  664         map->segments = (bus_dma_segment_t *)((uintptr_t)map + mapsize);
  665         return (map);
  666 }
  667 
  668 /*
  669  * Allocate a handle for mapping from kva/uva/physical
  670  * address space into bus device space.
  671  */
  672 int
  673 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  674 {
  675         bus_dmamap_t map;
  676         int error = 0;
  677 
  678         *mapp = map = allocate_map(dmat, M_NOWAIT);
  679         if (map == NULL) {
  680                 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM);
  681                 return (ENOMEM);
  682         }
  683 
  684         /*
  685          * Bouncing might be required if the driver asks for an exclusion
  686          * region, a data alignment that is stricter than 1, or DMA that begins
  687          * or ends with a partial cacheline.  Whether bouncing will actually
  688          * happen can't be known until mapping time, but we need to pre-allocate
  689          * resources now because we might not be allowed to at mapping time.
  690          */
  691         error = allocate_bz_and_pages(dmat, map);
  692         if (error != 0) {
  693                 free(map, M_DEVBUF);
  694                 *mapp = NULL;
  695                 return (error);
  696         }
  697         if (map->flags & DMAMAP_COHERENT)
  698                 atomic_add_32(&maps_coherent, 1);
  699         atomic_add_32(&maps_total, 1);
  700         dmat->map_count++;
  701 
  702         return (0);
  703 }
  704 
  705 /*
  706  * Destroy a handle for mapping from kva/uva/physical
  707  * address space into bus device space.
  708  */
  709 int
  710 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
  711 {
  712         if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
  713                 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  714                     __func__, dmat, EBUSY);
  715                 return (EBUSY);
  716         }
  717         if (dmat->bounce_zone)
  718                 dmat->bounce_zone->map_count--;
  719         if (map->flags & DMAMAP_COHERENT)
  720                 atomic_subtract_32(&maps_coherent, 1);
  721         atomic_subtract_32(&maps_total, 1);
  722         free(map, M_DEVBUF);
  723         dmat->map_count--;
  724         CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
  725         return (0);
  726 }
  727 
  728 
  729 /*
  730  * Allocate a piece of memory that can be efficiently mapped into
  731  * bus device space based on the constraints lited in the dma tag.
  732  * A dmamap to for use with dmamap_load is also allocated.
  733  */
  734 int
  735 bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
  736                  bus_dmamap_t *mapp)
  737 {
  738         busdma_bufalloc_t ba;
  739         struct busdma_bufzone *bufzone;
  740         bus_dmamap_t map;
  741         vm_memattr_t memattr;
  742         int mflags;
  743 
  744         if (flags & BUS_DMA_NOWAIT)
  745                 mflags = M_NOWAIT;
  746         else
  747                 mflags = M_WAITOK;
  748         if (flags & BUS_DMA_ZERO)
  749                 mflags |= M_ZERO;
  750 
  751         *mapp = map = allocate_map(dmat, mflags);
  752         if (map == NULL) {
  753                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  754                     __func__, dmat, dmat->flags, ENOMEM);
  755                 return (ENOMEM);
  756         }
  757         map->flags = DMAMAP_DMAMEM_ALLOC;
  758 
  759         /* Choose a busdma buffer allocator based on memory type flags. */
  760         if (flags & BUS_DMA_COHERENT) {
  761                 memattr = VM_MEMATTR_UNCACHEABLE;
  762                 ba = coherent_allocator;
  763                 map->flags |= DMAMAP_COHERENT;
  764         } else {
  765                 memattr = VM_MEMATTR_DEFAULT;
  766                 ba = standard_allocator;
  767         }
  768 
  769         /*
  770          * Try to find a bufzone in the allocator that holds a cache of buffers
  771          * of the right size for this request.  If the buffer is too big to be
  772          * held in the allocator cache, this returns NULL.
  773          */
  774         bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
  775 
  776         /*
  777          * Allocate the buffer from the uma(9) allocator if...
  778          *  - It's small enough to be in the allocator (bufzone not NULL).
  779          *  - The alignment constraint isn't larger than the allocation size
  780          *    (the allocator aligns buffers to their size boundaries).
  781          *  - There's no need to handle lowaddr/highaddr exclusion zones.
  782          * else allocate non-contiguous pages if...
  783          *  - The page count that could get allocated doesn't exceed nsegments.
  784          *  - The alignment constraint isn't larger than a page boundary.
  785          *  - There are no boundary-crossing constraints.
  786          * else allocate a block of contiguous pages because one or more of the
  787          * constraints is something that only the contig allocator can fulfill.
  788          */
  789         if (bufzone != NULL && dmat->alignment <= bufzone->size &&
  790             !exclusion_bounce(dmat)) {
  791                 *vaddr = uma_zalloc(bufzone->umazone, mflags);
  792         } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
  793             dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) {
  794                 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
  795                     mflags, 0, dmat->lowaddr, memattr);
  796         } else {
  797                 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
  798                     mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary,
  799                     memattr);
  800         }
  801 
  802 
  803         if (*vaddr == NULL) {
  804                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  805                     __func__, dmat, dmat->flags, ENOMEM);
  806                 free(map, M_DEVBUF);
  807                 *mapp = NULL;
  808                 return (ENOMEM);
  809         } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) {
  810                 printf("bus_dmamem_alloc failed to align memory properly.\n");
  811         }
  812         if (map->flags & DMAMAP_COHERENT)
  813                 atomic_add_32(&maps_coherent, 1);
  814         atomic_add_32(&maps_dmamem, 1);
  815         atomic_add_32(&maps_total, 1);
  816         dmat->map_count++;
  817 
  818         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  819             __func__, dmat, dmat->flags, 0);
  820         return (0);
  821 }
  822 
  823 /*
  824  * Free a piece of memory and it's allociated dmamap, that was allocated
  825  * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
  826  */
  827 void
  828 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
  829 {
  830         struct busdma_bufzone *bufzone;
  831         busdma_bufalloc_t ba;
  832 
  833         if (map->flags & DMAMAP_COHERENT)
  834                 ba = coherent_allocator;
  835         else
  836                 ba = standard_allocator;
  837 
  838         bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
  839 
  840         if (bufzone != NULL && dmat->alignment <= bufzone->size &&
  841             !exclusion_bounce(dmat))
  842                 uma_zfree(bufzone->umazone, vaddr);
  843         else
  844                 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
  845 
  846         dmat->map_count--;
  847         if (map->flags & DMAMAP_COHERENT)
  848                 atomic_subtract_32(&maps_coherent, 1);
  849         atomic_subtract_32(&maps_total, 1);
  850         atomic_subtract_32(&maps_dmamem, 1);
  851         free(map, M_DEVBUF);
  852         CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
  853 }
  854 
  855 static void
  856 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
  857     bus_size_t buflen, int flags)
  858 {
  859         bus_addr_t curaddr;
  860         bus_size_t sgsize;
  861 
  862         if (map->pagesneeded == 0) {
  863                 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d"
  864                     " map= %p, pagesneeded= %d",
  865                     dmat->lowaddr, dmat->boundary, dmat->alignment,
  866                     map, map->pagesneeded);
  867                 /*
  868                  * Count the number of bounce pages
  869                  * needed in order to complete this transfer
  870                  */
  871                 curaddr = buf;
  872                 while (buflen != 0) {
  873                         sgsize = MIN(buflen, dmat->maxsegsz);
  874                         if (must_bounce(dmat, map, curaddr, sgsize) != 0) {
  875                                 sgsize = MIN(sgsize, PAGE_SIZE);
  876                                 map->pagesneeded++;
  877                         }
  878                         curaddr += sgsize;
  879                         buflen -= sgsize;
  880                 }
  881                 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded);
  882         }
  883 }
  884 
  885 static void
  886 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  887     void *buf, bus_size_t buflen, int flags)
  888 {
  889         vm_offset_t vaddr;
  890         vm_offset_t vendaddr;
  891         bus_addr_t paddr;
  892 
  893         if (map->pagesneeded == 0) {
  894                 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d"
  895                     " map= %p, pagesneeded= %d",
  896                     dmat->lowaddr, dmat->boundary, dmat->alignment,
  897                     map, map->pagesneeded);
  898                 /*
  899                  * Count the number of bounce pages
  900                  * needed in order to complete this transfer
  901                  */
  902                 vaddr = (vm_offset_t)buf;
  903                 vendaddr = (vm_offset_t)buf + buflen;
  904 
  905                 while (vaddr < vendaddr) {
  906                         if (__predict_true(map->pmap == kernel_pmap))
  907                                 paddr = pmap_kextract(vaddr);
  908                         else
  909                                 paddr = pmap_extract(map->pmap, vaddr);
  910                         if (must_bounce(dmat, map, paddr,
  911                             min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & 
  912                             PAGE_MASK)))) != 0) {
  913                                 map->pagesneeded++;
  914                         }
  915                         vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
  916 
  917                 }
  918                 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded);
  919         }
  920 }
  921 
  922 static int
  923 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
  924 {
  925 
  926         /* Reserve Necessary Bounce Pages */
  927         mtx_lock(&bounce_lock);
  928         if (flags & BUS_DMA_NOWAIT) {
  929                 if (reserve_bounce_pages(dmat, map, 0) != 0) {
  930                         map->pagesneeded = 0;
  931                         mtx_unlock(&bounce_lock);
  932                         return (ENOMEM);
  933                 }
  934         } else {
  935                 if (reserve_bounce_pages(dmat, map, 1) != 0) {
  936                         /* Queue us for resources */
  937                         STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
  938                         mtx_unlock(&bounce_lock);
  939                         return (EINPROGRESS);
  940                 }
  941         }
  942         mtx_unlock(&bounce_lock);
  943 
  944         return (0);
  945 }
  946 
  947 /*
  948  * Add a single contiguous physical range to the segment list.
  949  */
  950 static int
  951 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
  952                    bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
  953 {
  954         bus_addr_t baddr, bmask;
  955         int seg;
  956 
  957         /*
  958          * Make sure we don't cross any boundaries.
  959          */
  960         bmask = ~(dmat->boundary - 1);
  961         if (dmat->boundary > 0) {
  962                 baddr = (curaddr + dmat->boundary) & bmask;
  963                 if (sgsize > (baddr - curaddr))
  964                         sgsize = (baddr - curaddr);
  965         }
  966 
  967         if (dmat->ranges) {
  968                 struct arm32_dma_range *dr;
  969 
  970                 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
  971                     curaddr);
  972                 if (dr == NULL) {
  973                         _bus_dmamap_unload(dmat, map);
  974                         return (0);
  975                 }
  976                 /*
  977                  * In a valid DMA range.  Translate the physical
  978                  * memory address to an address in the DMA window.
  979                  */
  980                 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
  981         }
  982 
  983         /*
  984          * Insert chunk into a segment, coalescing with
  985          * previous segment if possible.
  986          */
  987         seg = *segp;
  988         if (seg == -1) {
  989                 seg = 0;
  990                 segs[seg].ds_addr = curaddr;
  991                 segs[seg].ds_len = sgsize;
  992         } else {
  993                 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
  994                     (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
  995                     (dmat->boundary == 0 ||
  996                      (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
  997                         segs[seg].ds_len += sgsize;
  998                 else {
  999                         if (++seg >= dmat->nsegments)
 1000                                 return (0);
 1001                         segs[seg].ds_addr = curaddr;
 1002                         segs[seg].ds_len = sgsize;
 1003                 }
 1004         }
 1005         *segp = seg;
 1006         return (sgsize);
 1007 }
 1008 
 1009 /*
 1010  * Utility function to load a physical buffer.  segp contains
 1011  * the starting segment on entrace, and the ending segment on exit.
 1012  */
 1013 int
 1014 _bus_dmamap_load_phys(bus_dma_tag_t dmat,
 1015                       bus_dmamap_t map,
 1016                       vm_paddr_t buf, bus_size_t buflen,
 1017                       int flags,
 1018                       bus_dma_segment_t *segs,
 1019                       int *segp)
 1020 {
 1021         bus_addr_t curaddr;
 1022         bus_size_t sgsize;
 1023         int error;
 1024 
 1025         if (segs == NULL)
 1026                 segs = map->segments;
 1027 
 1028         maploads_total++;
 1029         maploads_physmem++;
 1030 
 1031         if (might_bounce(dmat, map, buflen, buflen)) {
 1032                 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
 1033                 if (map->pagesneeded != 0) {
 1034                         maploads_bounced++;
 1035                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
 1036                         if (error)
 1037                                 return (error);
 1038                 }
 1039         }
 1040 
 1041         while (buflen > 0) {
 1042                 curaddr = buf;
 1043                 sgsize = MIN(buflen, dmat->maxsegsz);
 1044                 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
 1045                     sgsize)) {
 1046                         sgsize = MIN(sgsize, PAGE_SIZE);
 1047                         curaddr = add_bounce_page(dmat, map, 0, curaddr,
 1048                                                   sgsize);
 1049                 }
 1050                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 1051                     segp);
 1052                 if (sgsize == 0)
 1053                         break;
 1054                 buf += sgsize;
 1055                 buflen -= sgsize;
 1056         }
 1057 
 1058         /*
 1059          * Did we fit?
 1060          */
 1061         if (buflen != 0) {
 1062                 _bus_dmamap_unload(dmat, map);
 1063                 return (EFBIG); /* XXX better return value here? */
 1064         }
 1065         return (0);
 1066 }
 1067 
 1068 int
 1069 _bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
 1070     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
 1071     bus_dma_segment_t *segs, int *segp)
 1072 {
 1073 
 1074         return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
 1075             segs, segp));
 1076 }
 1077 
 1078 /*
 1079  * Utility function to load a linear buffer.  segp contains
 1080  * the starting segment on entrace, and the ending segment on exit.
 1081  */
 1082 int
 1083 _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
 1084                         bus_dmamap_t map,
 1085                         void *buf, bus_size_t buflen,
 1086                         pmap_t pmap,
 1087                         int flags,
 1088                         bus_dma_segment_t *segs,
 1089                         int *segp)
 1090 {
 1091         bus_size_t sgsize;
 1092         bus_addr_t curaddr;
 1093         vm_offset_t vaddr;
 1094         struct sync_list *sl;
 1095         int error;
 1096 
 1097         maploads_total++;
 1098         if (map->flags & DMAMAP_COHERENT)
 1099                 maploads_coherent++;
 1100         if (map->flags & DMAMAP_DMAMEM_ALLOC)
 1101                 maploads_dmamem++;
 1102 
 1103         if (segs == NULL)
 1104                 segs = map->segments;
 1105 
 1106         if (flags & BUS_DMA_LOAD_MBUF) {
 1107                 maploads_mbuf++;
 1108                 map->flags |= DMAMAP_MBUF;
 1109         }
 1110 
 1111         map->pmap = pmap;
 1112 
 1113         if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
 1114                 _bus_dmamap_count_pages(dmat, map, buf, buflen, flags);
 1115                 if (map->pagesneeded != 0) {
 1116                         maploads_bounced++;
 1117                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
 1118                         if (error)
 1119                                 return (error);
 1120                 }
 1121         }
 1122 
 1123         sl = NULL;
 1124         vaddr = (vm_offset_t)buf;
 1125 
 1126         while (buflen > 0) {
 1127                 /*
 1128                  * Get the physical address for this segment.
 1129                  */
 1130                 if (__predict_true(map->pmap == kernel_pmap))
 1131                         curaddr = pmap_kextract(vaddr);
 1132                 else
 1133                         curaddr = pmap_extract(map->pmap, vaddr);
 1134 
 1135                 /*
 1136                  * Compute the segment size, and adjust counts.
 1137                  */
 1138                 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
 1139                 if (sgsize > dmat->maxsegsz)
 1140                         sgsize = dmat->maxsegsz;
 1141                 if (buflen < sgsize)
 1142                         sgsize = buflen;
 1143 
 1144                 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
 1145                     sgsize)) {
 1146                         curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
 1147                                                   sgsize);
 1148                 } else {
 1149                         sl = &map->slist[map->sync_count - 1];
 1150                         if (map->sync_count == 0 ||
 1151 #ifdef ARM_L2_PIPT
 1152                             curaddr != sl->busaddr + sl->datacount ||
 1153 #endif
 1154                             vaddr != sl->vaddr + sl->datacount) {
 1155                                 if (++map->sync_count > dmat->nsegments)
 1156                                         goto cleanup;
 1157                                 sl++;
 1158                                 sl->vaddr = vaddr;
 1159                                 sl->datacount = sgsize;
 1160                                 sl->busaddr = curaddr;
 1161                         } else
 1162                                 sl->datacount += sgsize;
 1163                 }
 1164                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 1165                                             segp);
 1166                 if (sgsize == 0)
 1167                         break;
 1168                 vaddr += sgsize;
 1169                 buflen -= sgsize;
 1170         }
 1171 
 1172 cleanup:
 1173         /*
 1174          * Did we fit?
 1175          */
 1176         if (buflen != 0) {
 1177                 _bus_dmamap_unload(dmat, map);
 1178                 return (EFBIG); /* XXX better return value here? */
 1179         }
 1180         return (0);
 1181 }
 1182 
 1183 
 1184 void
 1185 __bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
 1186                     struct memdesc *mem, bus_dmamap_callback_t *callback,
 1187                     void *callback_arg)
 1188 {
 1189 
 1190         map->mem = *mem;
 1191         map->dmat = dmat;
 1192         map->callback = callback;
 1193         map->callback_arg = callback_arg;
 1194 }
 1195 
 1196 bus_dma_segment_t *
 1197 _bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
 1198                      bus_dma_segment_t *segs, int nsegs, int error)
 1199 {
 1200 
 1201         if (segs == NULL)
 1202                 segs = map->segments;
 1203         return (segs);
 1204 }
 1205 
 1206 /*
 1207  * Release the mapping held by map.
 1208  */
 1209 void
 1210 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
 1211 {
 1212         struct bounce_page *bpage;
 1213         struct bounce_zone *bz;
 1214 
 1215         if ((bz = dmat->bounce_zone) != NULL) {
 1216                 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
 1217                         STAILQ_REMOVE_HEAD(&map->bpages, links);
 1218                         free_bounce_page(dmat, bpage);
 1219                 }
 1220 
 1221                 bz = dmat->bounce_zone;
 1222                 bz->free_bpages += map->pagesreserved;
 1223                 bz->reserved_bpages -= map->pagesreserved;
 1224                 map->pagesreserved = 0;
 1225                 map->pagesneeded = 0;
 1226         }
 1227         map->sync_count = 0;
 1228         map->flags &= ~DMAMAP_MBUF;
 1229 }
 1230 
 1231 #ifdef notyetbounceuser
 1232 /* If busdma uses user pages, then the interrupt handler could
 1233  * be use the kernel vm mapping. Both bounce pages and sync list
 1234  * do not cross page boundaries.
 1235  * Below is a rough sequence that a person would do to fix the
 1236  * user page reference in the kernel vmspace. This would be
 1237  * done in the dma post routine.
 1238  */
 1239 void
 1240 _bus_dmamap_fix_user(vm_offset_t buf, bus_size_t len,
 1241                         pmap_t pmap, int op)
 1242 {
 1243         bus_size_t sgsize;
 1244         bus_addr_t curaddr;
 1245         vm_offset_t va;
 1246 
 1247         /* 
 1248          * each synclist entry is contained within a single page.
 1249          * this would be needed if BUS_DMASYNC_POSTxxxx was implemented
 1250          */
 1251         curaddr = pmap_extract(pmap, buf);
 1252         va = pmap_dma_map(curaddr);
 1253         switch (op) {
 1254         case SYNC_USER_INV:
 1255                 cpu_dcache_wb_range(va, sgsize);
 1256                 break;
 1257 
 1258         case SYNC_USER_COPYTO:
 1259                 bcopy((void *)va, (void *)bounce, sgsize);
 1260                 break;
 1261 
 1262         case SYNC_USER_COPYFROM:
 1263                 bcopy((void *) bounce, (void *)va, sgsize);
 1264                 break;
 1265 
 1266         default:
 1267                 break;
 1268         }
 1269 
 1270         pmap_dma_unmap(va);
 1271 }
 1272 #endif
 1273 
 1274 #ifdef ARM_L2_PIPT
 1275 #define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(pa, size)
 1276 #define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(pa, size)
 1277 #define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(pa, size)
 1278 #else
 1279 #define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(va, size)
 1280 #define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size)
 1281 #define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(va, size)
 1282 #endif
 1283 
 1284 void
 1285 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 1286 {
 1287         struct bounce_page *bpage;
 1288         struct sync_list *sl, *end;
 1289         /*
 1290          * If the buffer was from user space, it is possible that this is not
 1291          * the same vm map, especially on a POST operation.  It's not clear that
 1292          * dma on userland buffers can work at all right now.  To be safe, until
 1293          * we're able to test direct userland dma, panic on a map mismatch.
 1294          */
 1295         if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
 1296                 if (!pmap_dmap_iscurrent(map->pmap))
 1297                         panic("_bus_dmamap_sync: wrong user map for bounce sync.");
 1298 
 1299                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
 1300                     "performing bounce", __func__, dmat, dmat->flags, op);
 1301 
 1302                 /*
 1303                  * For PREWRITE do a writeback.  Clean the caches from the
 1304                  * innermost to the outermost levels.
 1305                  */
 1306                 if (op & BUS_DMASYNC_PREWRITE) {
 1307                         while (bpage != NULL) {
 1308                                 if (bpage->datavaddr != 0)
 1309                                         bcopy((void *)bpage->datavaddr,
 1310                                             (void *)bpage->vaddr,
 1311                                             bpage->datacount);
 1312                                 else
 1313                                         physcopyout(bpage->dataaddr,
 1314                                             (void *)bpage->vaddr,
 1315                                             bpage->datacount);
 1316                                 cpu_dcache_wb_range((vm_offset_t)bpage->vaddr,
 1317                                     bpage->datacount);
 1318                                 l2cache_wb_range((vm_offset_t)bpage->vaddr,
 1319                                     (vm_offset_t)bpage->busaddr, 
 1320                                     bpage->datacount);
 1321                                 bpage = STAILQ_NEXT(bpage, links);
 1322                         }
 1323                         dmat->bounce_zone->total_bounced++;
 1324                 }
 1325 
 1326                 /*
 1327                  * Do an invalidate for PREREAD unless a writeback was already
 1328                  * done above due to PREWRITE also being set.  The reason for a
 1329                  * PREREAD invalidate is to prevent dirty lines currently in the
 1330                  * cache from being evicted during the DMA.  If a writeback was
 1331                  * done due to PREWRITE also being set there will be no dirty
 1332                  * lines and the POSTREAD invalidate handles the rest. The
 1333                  * invalidate is done from the innermost to outermost level. If
 1334                  * L2 were done first, a dirty cacheline could be automatically
 1335                  * evicted from L1 before we invalidated it, re-dirtying the L2.
 1336                  */
 1337                 if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) {
 1338                         bpage = STAILQ_FIRST(&map->bpages);
 1339                         while (bpage != NULL) {
 1340                                 cpu_dcache_inv_range((vm_offset_t)bpage->vaddr,
 1341                                     bpage->datacount);
 1342                                 l2cache_inv_range((vm_offset_t)bpage->vaddr,
 1343                                     (vm_offset_t)bpage->busaddr,
 1344                                     bpage->datacount);
 1345                                 bpage = STAILQ_NEXT(bpage, links);
 1346                         }
 1347                 }
 1348 
 1349                 /*
 1350                  * Re-invalidate the caches on a POSTREAD, even though they were
 1351                  * already invalidated at PREREAD time.  Aggressive prefetching
 1352                  * due to accesses to other data near the dma buffer could have
 1353                  * brought buffer data into the caches which is now stale.  The
 1354                  * caches are invalidated from the outermost to innermost; the
 1355                  * prefetches could be happening right now, and if L1 were
 1356                  * invalidated first, stale L2 data could be prefetched into L1.
 1357                  */
 1358                 if (op & BUS_DMASYNC_POSTREAD) {
 1359                         while (bpage != NULL) {
 1360                                 vm_offset_t startv;
 1361                                 vm_paddr_t startp;
 1362                                 int len;
 1363 
 1364                                 startv = bpage->vaddr &~ arm_dcache_align_mask;
 1365                                 startp = bpage->busaddr &~ arm_dcache_align_mask;
 1366                                 len = bpage->datacount;
 1367                                 
 1368                                 if (startv != bpage->vaddr)
 1369                                         len += bpage->vaddr & arm_dcache_align_mask;
 1370                                 if (len & arm_dcache_align_mask) 
 1371                                         len = (len -
 1372                                             (len & arm_dcache_align_mask)) +
 1373                                             arm_dcache_align;
 1374                                 l2cache_inv_range(startv, startp, len);
 1375                                 cpu_dcache_inv_range(startv, len);
 1376                                 if (bpage->datavaddr != 0)
 1377                                         bcopy((void *)bpage->vaddr,
 1378                                             (void *)bpage->datavaddr,
 1379                                             bpage->datacount);
 1380                                 else
 1381                                         physcopyin((void *)bpage->vaddr,
 1382                                             bpage->dataaddr,
 1383                                             bpage->datacount);
 1384                                 bpage = STAILQ_NEXT(bpage, links);
 1385                         }
 1386                         dmat->bounce_zone->total_bounced++;
 1387                 }
 1388         }
 1389 
 1390         /*
 1391          * For COHERENT memory no cache maintenance is necessary, but ensure all
 1392          * writes have reached memory for the PREWRITE case.  No action is
 1393          * needed for a PREREAD without PREWRITE also set, because that would
 1394          * imply that the cpu had written to the COHERENT buffer and expected
 1395          * the dma device to see that change, and by definition a PREWRITE sync
 1396          * is required to make that happen.
 1397          */
 1398         if (map->flags & DMAMAP_COHERENT) {
 1399                 if (op & BUS_DMASYNC_PREWRITE) {
 1400                         dsb();
 1401                         cpu_l2cache_drain_writebuf();
 1402                 }
 1403                 return;
 1404         }
 1405 
 1406         /*
 1407          * Cache maintenance for normal (non-COHERENT non-bounce) buffers.  All
 1408          * the comments about the sequences for flushing cache levels in the
 1409          * bounce buffer code above apply here as well.  In particular, the fact
 1410          * that the sequence is inner-to-outer for PREREAD invalidation and
 1411          * outer-to-inner for POSTREAD invalidation is not a mistake.
 1412          */
 1413         if (map->sync_count != 0) {
 1414                 if (!pmap_dmap_iscurrent(map->pmap))
 1415                         panic("_bus_dmamap_sync: wrong user map for sync.");
 1416 
 1417                 sl = &map->slist[0];
 1418                 end = &map->slist[map->sync_count];
 1419                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
 1420                     "performing sync", __func__, dmat, dmat->flags, op);
 1421 
 1422                 switch (op) {
 1423                 case BUS_DMASYNC_PREWRITE:
 1424                 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
 1425                         while (sl != end) {
 1426                                 cpu_dcache_wb_range(sl->vaddr, sl->datacount);
 1427                                 l2cache_wb_range(sl->vaddr, sl->busaddr,
 1428                                     sl->datacount);
 1429                                 sl++;
 1430                         }
 1431                         break;
 1432 
 1433                 case BUS_DMASYNC_PREREAD:
 1434                         /*
 1435                          * An mbuf may start in the middle of a cacheline. There
 1436                          * will be no cpu writes to the beginning of that line
 1437                          * (which contains the mbuf header) while dma is in
 1438                          * progress.  Handle that case by doing a writeback of
 1439                          * just the first cacheline before invalidating the
 1440                          * overall buffer.  Any mbuf in a chain may have this
 1441                          * misalignment.  Buffers which are not mbufs bounce if
 1442                          * they are not aligned to a cacheline.
 1443                          */
 1444                         while (sl != end) {
 1445                                 if (sl->vaddr & arm_dcache_align_mask) {
 1446                                         KASSERT(map->flags & DMAMAP_MBUF,
 1447                                             ("unaligned buffer is not an mbuf"));
 1448                                         cpu_dcache_wb_range(sl->vaddr, 1);
 1449                                         l2cache_wb_range(sl->vaddr,
 1450                                             sl->busaddr, 1);
 1451                                 }
 1452                                 cpu_dcache_inv_range(sl->vaddr, sl->datacount);
 1453                                 l2cache_inv_range(sl->vaddr, sl->busaddr, 
 1454                                     sl->datacount);
 1455                                 sl++;
 1456                         }
 1457                         break;
 1458 
 1459                 case BUS_DMASYNC_POSTWRITE:
 1460                         break;
 1461 
 1462                 case BUS_DMASYNC_POSTREAD:
 1463                 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
 1464                         while (sl != end) {
 1465                                 l2cache_inv_range(sl->vaddr, sl->busaddr, 
 1466                                     sl->datacount);
 1467                                 cpu_dcache_inv_range(sl->vaddr, sl->datacount);
 1468                                 sl++;
 1469                         }
 1470                         break;
 1471 
 1472                 default:
 1473                         panic("unsupported combination of sync operations: 0x%08x\n", op);
 1474                         break;
 1475                 }
 1476         }
 1477 }
 1478 
 1479 static void
 1480 init_bounce_pages(void *dummy __unused)
 1481 {
 1482 
 1483         total_bpages = 0;
 1484         STAILQ_INIT(&bounce_zone_list);
 1485         STAILQ_INIT(&bounce_map_waitinglist);
 1486         STAILQ_INIT(&bounce_map_callbacklist);
 1487         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
 1488 }
 1489 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
 1490 
 1491 static struct sysctl_ctx_list *
 1492 busdma_sysctl_tree(struct bounce_zone *bz)
 1493 {
 1494 
 1495         return (&bz->sysctl_tree);
 1496 }
 1497 
 1498 static struct sysctl_oid *
 1499 busdma_sysctl_tree_top(struct bounce_zone *bz)
 1500 {
 1501 
 1502         return (bz->sysctl_tree_top);
 1503 }
 1504 
 1505 static int
 1506 alloc_bounce_zone(bus_dma_tag_t dmat)
 1507 {
 1508         struct bounce_zone *bz;
 1509 
 1510         /* Check to see if we already have a suitable zone */
 1511         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
 1512                 if ((dmat->alignment <= bz->alignment) &&
 1513                     (dmat->lowaddr >= bz->lowaddr)) {
 1514                         dmat->bounce_zone = bz;
 1515                         return (0);
 1516                 }
 1517         }
 1518 
 1519         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
 1520             M_NOWAIT | M_ZERO)) == NULL)
 1521                 return (ENOMEM);
 1522 
 1523         STAILQ_INIT(&bz->bounce_page_list);
 1524         bz->free_bpages = 0;
 1525         bz->reserved_bpages = 0;
 1526         bz->active_bpages = 0;
 1527         bz->lowaddr = dmat->lowaddr;
 1528         bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
 1529         bz->map_count = 0;
 1530         snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
 1531         busdma_zonecount++;
 1532         snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
 1533         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
 1534         dmat->bounce_zone = bz;
 1535 
 1536         sysctl_ctx_init(&bz->sysctl_tree);
 1537         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
 1538             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
 1539             CTLFLAG_RD, 0, "");
 1540         if (bz->sysctl_tree_top == NULL) {
 1541                 sysctl_ctx_free(&bz->sysctl_tree);
 1542                 return (0);     /* XXX error code? */
 1543         }
 1544 
 1545         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1546             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1547             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
 1548             "Total bounce pages");
 1549         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1550             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1551             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
 1552             "Free bounce pages");
 1553         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1554             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1555             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
 1556             "Reserved bounce pages");
 1557         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1558             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1559             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
 1560             "Active bounce pages");
 1561         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1562             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1563             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
 1564             "Total bounce requests (pages bounced)");
 1565         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1566             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1567             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
 1568             "Total bounce requests that were deferred");
 1569         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
 1570             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1571             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
 1572         SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz),
 1573             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1574             "alignment", CTLFLAG_RD, &bz->alignment, "");
 1575 
 1576         return (0);
 1577 }
 1578 
 1579 static int
 1580 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
 1581 {
 1582         struct bounce_zone *bz;
 1583         int count;
 1584 
 1585         bz = dmat->bounce_zone;
 1586         count = 0;
 1587         while (numpages > 0) {
 1588                 struct bounce_page *bpage;
 1589 
 1590                 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
 1591                     M_NOWAIT | M_ZERO);
 1592 
 1593                 if (bpage == NULL)
 1594                         break;
 1595                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
 1596                     M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0);
 1597                 if (bpage->vaddr == 0) {
 1598                         free(bpage, M_DEVBUF);
 1599                         break;
 1600                 }
 1601                 bpage->busaddr = pmap_kextract(bpage->vaddr);
 1602                 mtx_lock(&bounce_lock);
 1603                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
 1604                 total_bpages++;
 1605                 bz->total_bpages++;
 1606                 bz->free_bpages++;
 1607                 mtx_unlock(&bounce_lock);
 1608                 count++;
 1609                 numpages--;
 1610         }
 1611         return (count);
 1612 }
 1613 
 1614 static int
 1615 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
 1616 {
 1617         struct bounce_zone *bz;
 1618         int pages;
 1619 
 1620         mtx_assert(&bounce_lock, MA_OWNED);
 1621         bz = dmat->bounce_zone;
 1622         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
 1623         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
 1624                 return (map->pagesneeded - (map->pagesreserved + pages));
 1625         bz->free_bpages -= pages;
 1626         bz->reserved_bpages += pages;
 1627         map->pagesreserved += pages;
 1628         pages = map->pagesneeded - map->pagesreserved;
 1629 
 1630         return (pages);
 1631 }
 1632 
 1633 static bus_addr_t
 1634 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 1635                 bus_addr_t addr, bus_size_t size)
 1636 {
 1637         struct bounce_zone *bz;
 1638         struct bounce_page *bpage;
 1639 
 1640         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
 1641         KASSERT(map != NULL,
 1642             ("add_bounce_page: bad map %p", map));
 1643 
 1644         bz = dmat->bounce_zone;
 1645         if (map->pagesneeded == 0)
 1646                 panic("add_bounce_page: map doesn't need any pages");
 1647         map->pagesneeded--;
 1648 
 1649         if (map->pagesreserved == 0)
 1650                 panic("add_bounce_page: map doesn't need any pages");
 1651         map->pagesreserved--;
 1652 
 1653         mtx_lock(&bounce_lock);
 1654         bpage = STAILQ_FIRST(&bz->bounce_page_list);
 1655         if (bpage == NULL)
 1656                 panic("add_bounce_page: free page list is empty");
 1657 
 1658         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
 1659         bz->reserved_bpages--;
 1660         bz->active_bpages++;
 1661         mtx_unlock(&bounce_lock);
 1662 
 1663         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1664                 /* Page offset needs to be preserved. */
 1665                 bpage->vaddr |= addr & PAGE_MASK;
 1666                 bpage->busaddr |= addr & PAGE_MASK;
 1667         }
 1668         bpage->datavaddr = vaddr;
 1669         bpage->dataaddr = addr;
 1670         bpage->datacount = size;
 1671         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
 1672         return (bpage->busaddr);
 1673 }
 1674 
 1675 static void
 1676 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
 1677 {
 1678         struct bus_dmamap *map;
 1679         struct bounce_zone *bz;
 1680 
 1681         bz = dmat->bounce_zone;
 1682         bpage->datavaddr = 0;
 1683         bpage->datacount = 0;
 1684         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1685                 /*
 1686                  * Reset the bounce page to start at offset 0.  Other uses
 1687                  * of this bounce page may need to store a full page of
 1688                  * data and/or assume it starts on a page boundary.
 1689                  */
 1690                 bpage->vaddr &= ~PAGE_MASK;
 1691                 bpage->busaddr &= ~PAGE_MASK;
 1692         }
 1693 
 1694         mtx_lock(&bounce_lock);
 1695         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
 1696         bz->free_bpages++;
 1697         bz->active_bpages--;
 1698         if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
 1699                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
 1700                         STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
 1701                         STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
 1702                             map, links);
 1703                         busdma_swi_pending = 1;
 1704                         bz->total_deferred++;
 1705                         swi_sched(vm_ih, 0);
 1706                 }
 1707         }
 1708         mtx_unlock(&bounce_lock);
 1709 }
 1710 
 1711 void
 1712 busdma_swi(void)
 1713 {
 1714         bus_dma_tag_t dmat;
 1715         struct bus_dmamap *map;
 1716 
 1717         mtx_lock(&bounce_lock);
 1718         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
 1719                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
 1720                 mtx_unlock(&bounce_lock);
 1721                 dmat = map->dmat;
 1722                 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_LOCK);
 1723                 bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback,
 1724                     map->callback_arg, BUS_DMA_WAITOK);
 1725                 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_UNLOCK);
 1726                 mtx_lock(&bounce_lock);
 1727         }
 1728         mtx_unlock(&bounce_lock);
 1729 }

Cache object: 6f8b1b0d20f50af71e736cd4f36fe044


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.