The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/powerpc/powerpc/busdma_machdep.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997, 1998 Justin T. Gibbs.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions, and the following disclaimer,
   10  *    without modification, immediately at the beginning of the file.
   11  * 2. The name of the author may not be used to endorse or promote products
   12  *    derived from this software without specific prior written permission.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
   18  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 /*
   28  * From amd64/busdma_machdep.c, r204214
   29  */
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD$");
   33 
   34 #include <sys/param.h>
   35 #include <sys/systm.h>
   36 #include <sys/malloc.h>
   37 #include <sys/bus.h>
   38 #include <sys/interrupt.h>
   39 #include <sys/kernel.h>
   40 #include <sys/ktr.h>
   41 #include <sys/lock.h>
   42 #include <sys/proc.h>
   43 #include <sys/mutex.h>
   44 #include <sys/mbuf.h>
   45 #include <sys/uio.h>
   46 #include <sys/sysctl.h>
   47 
   48 #include <vm/vm.h>
   49 #include <vm/vm_page.h>
   50 #include <vm/vm_map.h>
   51 
   52 #include <machine/atomic.h>
   53 #include <machine/bus.h>
   54 #include <machine/md_var.h>
   55 
   56 #define MAX_BPAGES 8192
   57 
   58 struct bounce_zone;
   59 
   60 struct bus_dma_tag {
   61         bus_dma_tag_t     parent;
   62         bus_size_t        alignment;
   63         bus_size_t        boundary;
   64         bus_addr_t        lowaddr;
   65         bus_addr_t        highaddr;
   66         bus_dma_filter_t *filter;
   67         void             *filterarg;
   68         bus_size_t        maxsize;
   69         u_int             nsegments;
   70         bus_size_t        maxsegsz;
   71         int               flags;
   72         int               ref_count;
   73         int               map_count;
   74         bus_dma_lock_t   *lockfunc;
   75         void             *lockfuncarg;
   76         bus_dma_segment_t *segments;
   77         struct bounce_zone *bounce_zone;
   78 };
   79 
   80 struct bounce_page {
   81         vm_offset_t     vaddr;          /* kva of bounce buffer */
   82         bus_addr_t      busaddr;        /* Physical address */
   83         vm_offset_t     datavaddr;      /* kva of client data */
   84         bus_size_t      datacount;      /* client data count */
   85         STAILQ_ENTRY(bounce_page) links;
   86 };
   87 
   88 int busdma_swi_pending;
   89 
   90 struct bounce_zone {
   91         STAILQ_ENTRY(bounce_zone) links;
   92         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
   93         int             total_bpages;
   94         int             free_bpages;
   95         int             reserved_bpages;
   96         int             active_bpages;
   97         int             total_bounced;
   98         int             total_deferred;
   99         int             map_count;
  100         bus_size_t      alignment;
  101         bus_addr_t      lowaddr;
  102         char            zoneid[8];
  103         char            lowaddrid[20];
  104         struct sysctl_ctx_list sysctl_tree;
  105         struct sysctl_oid *sysctl_tree_top;
  106 };
  107 
  108 static struct mtx bounce_lock;
  109 static int total_bpages;
  110 static int busdma_zonecount;
  111 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
  112 
  113 SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
  114 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
  115            "Total bounce pages");
  116 
  117 struct bus_dmamap {
  118         struct bp_list         bpages;
  119         int                    pagesneeded;
  120         int                    pagesreserved;
  121         bus_dma_tag_t          dmat;
  122         void                  *buf;             /* unmapped buffer pointer */
  123         bus_size_t             buflen;          /* unmapped buffer length */
  124         bus_dmamap_callback_t *callback;
  125         void                  *callback_arg;
  126         STAILQ_ENTRY(bus_dmamap) links;
  127 };
  128 
  129 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
  130 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
  131 static struct bus_dmamap nobounce_dmamap;
  132 
  133 static void init_bounce_pages(void *dummy);
  134 static int alloc_bounce_zone(bus_dma_tag_t dmat);
  135 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
  136 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  137                                 int commit);
  138 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
  139                                    vm_offset_t vaddr, bus_size_t size);
  140 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
  141 static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
  142 
  143 /*
  144  * Return true if a match is made.
  145  *
  146  * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
  147  *
  148  * If paddr is within the bounds of the dma tag then call the filter callback
  149  * to check for a match, if there is no filter callback then assume a match.
  150  */
  151 static __inline int
  152 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
  153 {
  154         int retval;
  155 
  156         retval = 0;
  157 
  158         do {
  159                 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
  160                  || ((paddr & (dmat->alignment - 1)) != 0))
  161                  && (dmat->filter == NULL
  162                   || (*dmat->filter)(dmat->filterarg, paddr) != 0))
  163                         retval = 1;
  164 
  165                 dmat = dmat->parent;            
  166         } while (retval == 0 && dmat != NULL);
  167         return (retval);
  168 }
  169 
  170 /*
  171  * Convenience function for manipulating driver locks from busdma (during
  172  * busdma_swi, for example).  Drivers that don't provide their own locks
  173  * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
  174  * non-mutex locking scheme don't have to use this at all.
  175  */
  176 void
  177 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
  178 {
  179         struct mtx *dmtx;
  180 
  181         dmtx = (struct mtx *)arg;
  182         switch (op) {
  183         case BUS_DMA_LOCK:
  184                 mtx_lock(dmtx);
  185                 break;
  186         case BUS_DMA_UNLOCK:
  187                 mtx_unlock(dmtx);
  188                 break;
  189         default:
  190                 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
  191         }
  192 }
  193 
  194 /*
  195  * dflt_lock should never get called.  It gets put into the dma tag when
  196  * lockfunc == NULL, which is only valid if the maps that are associated
  197  * with the tag are meant to never be defered.
  198  * XXX Should have a way to identify which driver is responsible here.
  199  */
  200 static void
  201 dflt_lock(void *arg, bus_dma_lock_op_t op)
  202 {
  203         panic("driver error: busdma dflt_lock called");
  204 }
  205 
  206 #define BUS_DMA_COULD_BOUNCE    BUS_DMA_BUS3
  207 #define BUS_DMA_MIN_ALLOC_COMP  BUS_DMA_BUS4
  208 /*
  209  * Allocate a device specific dma_tag.
  210  */
  211 int
  212 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  213                    bus_size_t boundary, bus_addr_t lowaddr,
  214                    bus_addr_t highaddr, bus_dma_filter_t *filter,
  215                    void *filterarg, bus_size_t maxsize, int nsegments,
  216                    bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  217                    void *lockfuncarg, bus_dma_tag_t *dmat)
  218 {
  219         bus_dma_tag_t newtag;
  220         int error = 0;
  221 
  222         /* Basic sanity checking */
  223         if (boundary != 0 && boundary < maxsegsz)
  224                 maxsegsz = boundary;
  225 
  226         if (maxsegsz == 0) {
  227                 return (EINVAL);
  228         }
  229 
  230         /* Return a NULL tag on failure */
  231         *dmat = NULL;
  232 
  233         newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
  234             M_ZERO | M_NOWAIT);
  235         if (newtag == NULL) {
  236                 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  237                     __func__, newtag, 0, error);
  238                 return (ENOMEM);
  239         }
  240 
  241         newtag->parent = parent;
  242         newtag->alignment = alignment;
  243         newtag->boundary = boundary;
  244         newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
  245         newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
  246         newtag->filter = filter;
  247         newtag->filterarg = filterarg;
  248         newtag->maxsize = maxsize;
  249         newtag->nsegments = nsegments;
  250         newtag->maxsegsz = maxsegsz;
  251         newtag->flags = flags;
  252         newtag->ref_count = 1; /* Count ourself */
  253         newtag->map_count = 0;
  254         if (lockfunc != NULL) {
  255                 newtag->lockfunc = lockfunc;
  256                 newtag->lockfuncarg = lockfuncarg;
  257         } else {
  258                 newtag->lockfunc = dflt_lock;
  259                 newtag->lockfuncarg = NULL;
  260         }
  261         newtag->segments = NULL;
  262 
  263         /* Take into account any restrictions imposed by our parent tag */
  264         if (parent != NULL) {
  265                 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
  266                 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
  267                 if (newtag->boundary == 0)
  268                         newtag->boundary = parent->boundary;
  269                 else if (parent->boundary != 0)
  270                         newtag->boundary = MIN(parent->boundary,
  271                                                newtag->boundary);
  272                 if (newtag->filter == NULL) {
  273                         /*
  274                          * Short circuit looking at our parent directly
  275                          * since we have encapsulated all of its information
  276                          */
  277                         newtag->filter = parent->filter;
  278                         newtag->filterarg = parent->filterarg;
  279                         newtag->parent = parent->parent;
  280                 }
  281                 if (newtag->parent != NULL)
  282                         atomic_add_int(&parent->ref_count, 1);
  283         }
  284 
  285         if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
  286          || newtag->alignment > 1)
  287                 newtag->flags |= BUS_DMA_COULD_BOUNCE;
  288 
  289         if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
  290             (flags & BUS_DMA_ALLOCNOW) != 0) {
  291                 struct bounce_zone *bz;
  292 
  293                 /* Must bounce */
  294 
  295                 if ((error = alloc_bounce_zone(newtag)) != 0) {
  296                         free(newtag, M_DEVBUF);
  297                         return (error);
  298                 }
  299                 bz = newtag->bounce_zone;
  300 
  301                 if (ptoa(bz->total_bpages) < maxsize) {
  302                         int pages;
  303 
  304                         pages = atop(maxsize) - bz->total_bpages;
  305 
  306                         /* Add pages to our bounce pool */
  307                         if (alloc_bounce_pages(newtag, pages) < pages)
  308                                 error = ENOMEM;
  309                 }
  310                 /* Performed initial allocation */
  311                 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
  312         }
  313         
  314         if (error != 0) {
  315                 free(newtag, M_DEVBUF);
  316         } else {
  317                 *dmat = newtag;
  318         }
  319         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  320             __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
  321         return (error);
  322 }
  323 
  324 int
  325 bus_dma_tag_destroy(bus_dma_tag_t dmat)
  326 {
  327         bus_dma_tag_t dmat_copy;
  328         int error;
  329 
  330         error = 0;
  331         dmat_copy = dmat;
  332 
  333         if (dmat != NULL) {
  334 
  335                 if (dmat->map_count != 0) {
  336                         error = EBUSY;
  337                         goto out;
  338                 }
  339 
  340                 while (dmat != NULL) {
  341                         bus_dma_tag_t parent;
  342 
  343                         parent = dmat->parent;
  344                         atomic_subtract_int(&dmat->ref_count, 1);
  345                         if (dmat->ref_count == 0) {
  346                                 if (dmat->segments != NULL)
  347                                         free(dmat->segments, M_DEVBUF);
  348                                 free(dmat, M_DEVBUF);
  349                                 /*
  350                                  * Last reference count, so
  351                                  * release our reference
  352                                  * count on our parent.
  353                                  */
  354                                 dmat = parent;
  355                         } else
  356                                 dmat = NULL;
  357                 }
  358         }
  359 out:
  360         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
  361         return (error);
  362 }
  363 
  364 /*
  365  * Allocate a handle for mapping from kva/uva/physical
  366  * address space into bus device space.
  367  */
  368 int
  369 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  370 {
  371         int error;
  372 
  373         error = 0;
  374 
  375         if (dmat->segments == NULL) {
  376                 dmat->segments = (bus_dma_segment_t *)malloc(
  377                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
  378                     M_NOWAIT);
  379                 if (dmat->segments == NULL) {
  380                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  381                             __func__, dmat, ENOMEM);
  382                         return (ENOMEM);
  383                 }
  384         }
  385 
  386         /*
  387          * Bouncing might be required if the driver asks for an active
  388          * exclusion region, a data alignment that is stricter than 1, and/or
  389          * an active address boundary.
  390          */
  391         if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
  392 
  393                 /* Must bounce */
  394                 struct bounce_zone *bz;
  395                 int maxpages;
  396 
  397                 if (dmat->bounce_zone == NULL) {
  398                         if ((error = alloc_bounce_zone(dmat)) != 0)
  399                                 return (error);
  400                 }
  401                 bz = dmat->bounce_zone;
  402 
  403                 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
  404                                              M_NOWAIT | M_ZERO);
  405                 if (*mapp == NULL) {
  406                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  407                             __func__, dmat, ENOMEM);
  408                         return (ENOMEM);
  409                 }
  410 
  411                 /* Initialize the new map */
  412                 STAILQ_INIT(&((*mapp)->bpages));
  413 
  414                 /*
  415                  * Attempt to add pages to our pool on a per-instance
  416                  * basis up to a sane limit.
  417                  */
  418                 if (dmat->alignment > 1)
  419                         maxpages = MAX_BPAGES;
  420                 else
  421                         maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
  422                 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
  423                  || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
  424                         int pages;
  425 
  426                         pages = MAX(atop(dmat->maxsize), 1);
  427                         pages = MIN(maxpages - bz->total_bpages, pages);
  428                         pages = MAX(pages, 1);
  429                         if (alloc_bounce_pages(dmat, pages) < pages)
  430                                 error = ENOMEM;
  431 
  432                         if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
  433                                 if (error == 0)
  434                                         dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
  435                         } else {
  436                                 error = 0;
  437                         }
  438                 }
  439                 bz->map_count++;
  440         } else {
  441                 *mapp = NULL;
  442         }
  443         if (error == 0)
  444                 dmat->map_count++;
  445         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  446             __func__, dmat, dmat->flags, error);
  447         return (error);
  448 }
  449 
  450 /*
  451  * Destroy a handle for mapping from kva/uva/physical
  452  * address space into bus device space.
  453  */
  454 int
  455 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
  456 {
  457         if (map != NULL && map != &nobounce_dmamap) {
  458                 if (STAILQ_FIRST(&map->bpages) != NULL) {
  459                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  460                             __func__, dmat, EBUSY);
  461                         return (EBUSY);
  462                 }
  463                 if (dmat->bounce_zone)
  464                         dmat->bounce_zone->map_count--;
  465                 free(map, M_DEVBUF);
  466         }
  467         dmat->map_count--;
  468         CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
  469         return (0);
  470 }
  471 
  472 
  473 /*
  474  * Allocate a piece of memory that can be efficiently mapped into
  475  * bus device space based on the constraints lited in the dma tag.
  476  * A dmamap to for use with dmamap_load is also allocated.
  477  */
  478 int
  479 bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
  480                  bus_dmamap_t *mapp)
  481 {
  482         int mflags;
  483 
  484         if (flags & BUS_DMA_NOWAIT)
  485                 mflags = M_NOWAIT;
  486         else
  487                 mflags = M_WAITOK;
  488 
  489         /* If we succeed, no mapping/bouncing will be required */
  490         *mapp = NULL;
  491 
  492         if (dmat->segments == NULL) {
  493                 dmat->segments = (bus_dma_segment_t *)malloc(
  494                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
  495                     mflags);
  496                 if (dmat->segments == NULL) {
  497                         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  498                             __func__, dmat, dmat->flags, ENOMEM);
  499                         return (ENOMEM);
  500                 }
  501         }
  502         if (flags & BUS_DMA_ZERO)
  503                 mflags |= M_ZERO;
  504 
  505         /* 
  506          * XXX:
  507          * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
  508          * alignment guarantees of malloc need to be nailed down, and the
  509          * code below should be rewritten to take that into account.
  510          *
  511          * In the meantime, we'll warn the user if malloc gets it wrong.
  512          */
  513         if ((dmat->maxsize <= PAGE_SIZE) &&
  514            (dmat->alignment < dmat->maxsize) &&
  515             dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) {
  516                 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
  517         } else {
  518                 /*
  519                  * XXX Use Contigmalloc until it is merged into this facility
  520                  *     and handles multi-seg allocations.  Nobody is doing
  521                  *     multi-seg allocations yet though.
  522                  * XXX Certain AGP hardware does.
  523                  */
  524                 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags,
  525                     0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul,
  526                     dmat->boundary);
  527         }
  528         if (*vaddr == NULL) {
  529                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  530                     __func__, dmat, dmat->flags, ENOMEM);
  531                 return (ENOMEM);
  532         } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) {
  533                 printf("bus_dmamem_alloc failed to align memory properly.\n");
  534         }
  535 #ifdef NOTYET
  536         if (flags & BUS_DMA_NOCACHE)
  537                 pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize,
  538                     PAT_UNCACHEABLE);
  539 #endif
  540         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  541             __func__, dmat, dmat->flags, 0);
  542         return (0);
  543 }
  544 
  545 /*
  546  * Free a piece of memory and it's allociated dmamap, that was allocated
  547  * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
  548  */
  549 void
  550 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
  551 {
  552         /*
  553          * dmamem does not need to be bounced, so the map should be
  554          * NULL
  555          */
  556         if (map != NULL)
  557                 panic("bus_dmamem_free: Invalid map freed\n");
  558 #ifdef NOTYET
  559         pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, PAT_WRITE_BACK);
  560 #endif
  561         if ((dmat->maxsize <= PAGE_SIZE) &&
  562            (dmat->alignment < dmat->maxsize) &&
  563             dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem))
  564                 free(vaddr, M_DEVBUF);
  565         else {
  566                 contigfree(vaddr, dmat->maxsize, M_DEVBUF);
  567         }
  568         CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
  569 }
  570 
  571 /*
  572  * Utility function to load a linear buffer.  lastaddrp holds state
  573  * between invocations (for multiple-buffer loads).  segp contains
  574  * the starting segment on entrance, and the ending segment on exit.
  575  * first indicates if this is the first invocation of this function.
  576  */
  577 static __inline int
  578 _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
  579                         bus_dmamap_t map,
  580                         void *buf, bus_size_t buflen,
  581                         pmap_t pmap,
  582                         int flags,
  583                         bus_addr_t *lastaddrp,
  584                         bus_dma_segment_t *segs,
  585                         int *segp,
  586                         int first)
  587 {
  588         bus_size_t sgsize;
  589         bus_addr_t curaddr, lastaddr, baddr, bmask;
  590         vm_offset_t vaddr;
  591         bus_addr_t paddr;
  592         int seg;
  593 
  594         if (map == NULL)
  595                 map = &nobounce_dmamap;
  596 
  597         if ((map != &nobounce_dmamap && map->pagesneeded == 0) 
  598          && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) {
  599                 vm_offset_t     vendaddr;
  600 
  601                 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
  602                     "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
  603                     dmat->boundary, dmat->alignment);
  604                 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
  605                     map, &nobounce_dmamap, map->pagesneeded);
  606                 /*
  607                  * Count the number of bounce pages
  608                  * needed in order to complete this transfer
  609                  */
  610                 vaddr = (vm_offset_t)buf;
  611                 vendaddr = (vm_offset_t)buf + buflen;
  612 
  613                 while (vaddr < vendaddr) {
  614                         bus_size_t sg_len;
  615 
  616                         sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
  617                         if (pmap)
  618                                 paddr = pmap_extract(pmap, vaddr);
  619                         else
  620                                 paddr = pmap_kextract(vaddr);
  621                         if (run_filter(dmat, paddr) != 0) {
  622                                 sg_len = roundup2(sg_len, dmat->alignment);
  623                                 map->pagesneeded++;
  624                         }
  625                         vaddr += sg_len;
  626                 }
  627                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
  628         }
  629 
  630         /* Reserve Necessary Bounce Pages */
  631         if (map->pagesneeded != 0) {
  632                 mtx_lock(&bounce_lock);
  633                 if (flags & BUS_DMA_NOWAIT) {
  634                         if (reserve_bounce_pages(dmat, map, 0) != 0) {
  635                                 mtx_unlock(&bounce_lock);
  636                                 return (ENOMEM);
  637                         }
  638                 } else {
  639                         if (reserve_bounce_pages(dmat, map, 1) != 0) {
  640                                 /* Queue us for resources */
  641                                 map->dmat = dmat;
  642                                 map->buf = buf;
  643                                 map->buflen = buflen;
  644                                 STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
  645                                     map, links);
  646                                 mtx_unlock(&bounce_lock);
  647                                 return (EINPROGRESS);
  648                         }
  649                 }
  650                 mtx_unlock(&bounce_lock);
  651         }
  652 
  653         vaddr = (vm_offset_t)buf;
  654         lastaddr = *lastaddrp;
  655         bmask = ~(dmat->boundary - 1);
  656 
  657         for (seg = *segp; buflen > 0 ; ) {
  658                 bus_size_t max_sgsize;
  659 
  660                 /*
  661                  * Get the physical address for this segment.
  662                  */
  663                 if (pmap)
  664                         curaddr = pmap_extract(pmap, vaddr);
  665                 else
  666                         curaddr = pmap_kextract(vaddr);
  667 
  668                 /*
  669                  * Compute the segment size, and adjust counts.
  670                  */
  671                 max_sgsize = MIN(buflen, dmat->maxsegsz);
  672                 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
  673                 if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
  674                         sgsize = roundup2(sgsize, dmat->alignment);
  675                         sgsize = MIN(sgsize, max_sgsize);
  676                         curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
  677                 } else {
  678                         sgsize = MIN(sgsize, max_sgsize);
  679                 }
  680 
  681                 /*
  682                  * Make sure we don't cross any boundaries.
  683                  */
  684                 if (dmat->boundary > 0) {
  685                         baddr = (curaddr + dmat->boundary) & bmask;
  686                         if (sgsize > (baddr - curaddr))
  687                                 sgsize = (baddr - curaddr);
  688                 }
  689 
  690                 /*
  691                  * Insert chunk into a segment, coalescing with
  692                  * previous segment if possible.
  693                  */
  694                 if (first) {
  695                         segs[seg].ds_addr = curaddr;
  696                         segs[seg].ds_len = sgsize;
  697                         first = 0;
  698                 } else {
  699                         if (curaddr == lastaddr &&
  700                             (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
  701                             (dmat->boundary == 0 ||
  702                              (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
  703                                 segs[seg].ds_len += sgsize;
  704                         else {
  705                                 if (++seg >= dmat->nsegments)
  706                                         break;
  707                                 segs[seg].ds_addr = curaddr;
  708                                 segs[seg].ds_len = sgsize;
  709                         }
  710                 }
  711 
  712                 lastaddr = curaddr + sgsize;
  713                 vaddr += sgsize;
  714                 buflen -= sgsize;
  715         }
  716 
  717         *segp = seg;
  718         *lastaddrp = lastaddr;
  719 
  720         /*
  721          * Did we fit?
  722          */
  723         return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
  724 }
  725 
  726 /*
  727  * Map the buffer buf into bus space using the dmamap map.
  728  */
  729 int
  730 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
  731                 bus_size_t buflen, bus_dmamap_callback_t *callback,
  732                 void *callback_arg, int flags)
  733 {
  734         bus_addr_t              lastaddr = 0;
  735         int                     error, nsegs = 0;
  736 
  737         if (map != NULL) {
  738                 flags |= BUS_DMA_WAITOK;
  739                 map->callback = callback;
  740                 map->callback_arg = callback_arg;
  741         }
  742 
  743         error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags,
  744              &lastaddr, dmat->segments, &nsegs, 1);
  745 
  746         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  747             __func__, dmat, dmat->flags, error, nsegs + 1);
  748 
  749         if (error == EINPROGRESS) {
  750                 return (error);
  751         }
  752 
  753         if (error)
  754                 (*callback)(callback_arg, dmat->segments, 0, error);
  755         else
  756                 (*callback)(callback_arg, dmat->segments, nsegs + 1, 0);
  757 
  758         /*
  759          * Return ENOMEM to the caller so that it can pass it up the stack.
  760          * This error only happens when NOWAIT is set, so deferal is disabled.
  761          */
  762         if (error == ENOMEM)
  763                 return (error);
  764 
  765         return (0);
  766 }
  767 
  768 
  769 /*
  770  * Like _bus_dmamap_load(), but for mbufs.
  771  */
  772 int
  773 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
  774                      struct mbuf *m0,
  775                      bus_dmamap_callback2_t *callback, void *callback_arg,
  776                      int flags)
  777 {
  778         int nsegs, error;
  779 
  780         M_ASSERTPKTHDR(m0);
  781 
  782         flags |= BUS_DMA_NOWAIT;
  783         nsegs = 0;
  784         error = 0;
  785         if (m0->m_pkthdr.len <= dmat->maxsize) {
  786                 int first = 1;
  787                 bus_addr_t lastaddr = 0;
  788                 struct mbuf *m;
  789 
  790                 for (m = m0; m != NULL && error == 0; m = m->m_next) {
  791                         if (m->m_len > 0) {
  792                                 error = _bus_dmamap_load_buffer(dmat, map,
  793                                                 m->m_data, m->m_len,
  794                                                 NULL, flags, &lastaddr,
  795                                                 dmat->segments, &nsegs, first);
  796                                 first = 0;
  797                         }
  798                 }
  799         } else {
  800                 error = EINVAL;
  801         }
  802 
  803         if (error) {
  804                 /* force "no valid mappings" in callback */
  805                 (*callback)(callback_arg, dmat->segments, 0, 0, error);
  806         } else {
  807                 (*callback)(callback_arg, dmat->segments,
  808                             nsegs+1, m0->m_pkthdr.len, error);
  809         }
  810         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  811             __func__, dmat, dmat->flags, error, nsegs + 1);
  812         return (error);
  813 }
  814 
  815 int
  816 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
  817                         struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
  818                         int flags)
  819 {
  820         int error;
  821 
  822         M_ASSERTPKTHDR(m0);
  823 
  824         flags |= BUS_DMA_NOWAIT;
  825         *nsegs = 0;
  826         error = 0;
  827         if (m0->m_pkthdr.len <= dmat->maxsize) {
  828                 int first = 1;
  829                 bus_addr_t lastaddr = 0;
  830                 struct mbuf *m;
  831 
  832                 for (m = m0; m != NULL && error == 0; m = m->m_next) {
  833                         if (m->m_len > 0) {
  834                                 error = _bus_dmamap_load_buffer(dmat, map,
  835                                                 m->m_data, m->m_len,
  836                                                 NULL, flags, &lastaddr,
  837                                                 segs, nsegs, first);
  838                                 first = 0;
  839                         }
  840                 }
  841         } else {
  842                 error = EINVAL;
  843         }
  844 
  845         /* XXX FIXME: Having to increment nsegs is really annoying */
  846         ++*nsegs;
  847         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  848             __func__, dmat, dmat->flags, error, *nsegs);
  849         return (error);
  850 }
  851 
  852 /*
  853  * Like _bus_dmamap_load(), but for uios.
  854  */
  855 int
  856 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
  857                     struct uio *uio,
  858                     bus_dmamap_callback2_t *callback, void *callback_arg,
  859                     int flags)
  860 {
  861         bus_addr_t lastaddr = 0;
  862         int nsegs, error, first, i;
  863         bus_size_t resid;
  864         struct iovec *iov;
  865         pmap_t pmap;
  866 
  867         flags |= BUS_DMA_NOWAIT;
  868         resid = uio->uio_resid;
  869         iov = uio->uio_iov;
  870 
  871         if (uio->uio_segflg == UIO_USERSPACE) {
  872                 KASSERT(uio->uio_td != NULL,
  873                         ("bus_dmamap_load_uio: USERSPACE but no proc"));
  874                 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
  875         } else
  876                 pmap = NULL;
  877 
  878         nsegs = 0;
  879         error = 0;
  880         first = 1;
  881         for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
  882                 /*
  883                  * Now at the first iovec to load.  Load each iovec
  884                  * until we have exhausted the residual count.
  885                  */
  886                 bus_size_t minlen =
  887                         resid < iov[i].iov_len ? resid : iov[i].iov_len;
  888                 caddr_t addr = (caddr_t) iov[i].iov_base;
  889 
  890                 if (minlen > 0) {
  891                         error = _bus_dmamap_load_buffer(dmat, map,
  892                                         addr, minlen, pmap, flags, &lastaddr,
  893                                         dmat->segments, &nsegs, first);
  894                         first = 0;
  895 
  896                         resid -= minlen;
  897                 }
  898         }
  899 
  900         if (error) {
  901                 /* force "no valid mappings" in callback */
  902                 (*callback)(callback_arg, dmat->segments, 0, 0, error);
  903         } else {
  904                 (*callback)(callback_arg, dmat->segments,
  905                             nsegs+1, uio->uio_resid, error);
  906         }
  907         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  908             __func__, dmat, dmat->flags, error, nsegs + 1);
  909         return (error);
  910 }
  911 
  912 /*
  913  * Release the mapping held by map.
  914  */
  915 void
  916 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
  917 {
  918         struct bounce_page *bpage;
  919 
  920         while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
  921                 STAILQ_REMOVE_HEAD(&map->bpages, links);
  922                 free_bounce_page(dmat, bpage);
  923         }
  924 }
  925 
  926 void
  927 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
  928 {
  929         struct bounce_page *bpage;
  930 
  931         if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
  932                 /*
  933                  * Handle data bouncing.  We might also
  934                  * want to add support for invalidating
  935                  * the caches on broken hardware
  936                  */
  937                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
  938                     "performing bounce", __func__, op, dmat, dmat->flags);
  939 
  940                 if (op & BUS_DMASYNC_PREWRITE) {
  941                         while (bpage != NULL) {
  942                                 bcopy((void *)bpage->datavaddr,
  943                                       (void *)bpage->vaddr,
  944                                       bpage->datacount);
  945                                 bpage = STAILQ_NEXT(bpage, links);
  946                         }
  947                         dmat->bounce_zone->total_bounced++;
  948                 }
  949 
  950                 if (op & BUS_DMASYNC_POSTREAD) {
  951                         while (bpage != NULL) {
  952                                 bcopy((void *)bpage->vaddr,
  953                                       (void *)bpage->datavaddr,
  954                                       bpage->datacount);
  955                                 bpage = STAILQ_NEXT(bpage, links);
  956                         }
  957                         dmat->bounce_zone->total_bounced++;
  958                 }
  959         }
  960 }
  961 
  962 static void
  963 init_bounce_pages(void *dummy __unused)
  964 {
  965 
  966         total_bpages = 0;
  967         STAILQ_INIT(&bounce_zone_list);
  968         STAILQ_INIT(&bounce_map_waitinglist);
  969         STAILQ_INIT(&bounce_map_callbacklist);
  970         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
  971 }
  972 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
  973 
  974 static struct sysctl_ctx_list *
  975 busdma_sysctl_tree(struct bounce_zone *bz)
  976 {
  977         return (&bz->sysctl_tree);
  978 }
  979 
  980 static struct sysctl_oid *
  981 busdma_sysctl_tree_top(struct bounce_zone *bz)
  982 {
  983         return (bz->sysctl_tree_top);
  984 }
  985 
  986 static int
  987 alloc_bounce_zone(bus_dma_tag_t dmat)
  988 {
  989         struct bounce_zone *bz;
  990 
  991         /* Check to see if we already have a suitable zone */
  992         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
  993                 if ((dmat->alignment <= bz->alignment)
  994                  && (dmat->lowaddr >= bz->lowaddr)) {
  995                         dmat->bounce_zone = bz;
  996                         return (0);
  997                 }
  998         }
  999 
 1000         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
 1001             M_NOWAIT | M_ZERO)) == NULL)
 1002                 return (ENOMEM);
 1003 
 1004         STAILQ_INIT(&bz->bounce_page_list);
 1005         bz->free_bpages = 0;
 1006         bz->reserved_bpages = 0;
 1007         bz->active_bpages = 0;
 1008         bz->lowaddr = dmat->lowaddr;
 1009         bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
 1010         bz->map_count = 0;
 1011         snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
 1012         busdma_zonecount++;
 1013         snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
 1014         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
 1015         dmat->bounce_zone = bz;
 1016 
 1017         sysctl_ctx_init(&bz->sysctl_tree);
 1018         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
 1019             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
 1020             CTLFLAG_RD, 0, "");
 1021         if (bz->sysctl_tree_top == NULL) {
 1022                 sysctl_ctx_free(&bz->sysctl_tree);
 1023                 return (0);     /* XXX error code? */
 1024         }
 1025 
 1026         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1027             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1028             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
 1029             "Total bounce pages");
 1030         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1031             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1032             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
 1033             "Free bounce pages");
 1034         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1035             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1036             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
 1037             "Reserved bounce pages");
 1038         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1039             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1040             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
 1041             "Active bounce pages");
 1042         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1043             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1044             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
 1045             "Total bounce requests");
 1046         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1047             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1048             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
 1049             "Total bounce requests that were deferred");
 1050         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
 1051             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1052             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
 1053         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1054             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1055             "alignment", CTLFLAG_RD, &bz->alignment, 0, "");
 1056 
 1057         return (0);
 1058 }
 1059 
 1060 static int
 1061 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
 1062 {
 1063         struct bounce_zone *bz;
 1064         int count;
 1065 
 1066         bz = dmat->bounce_zone;
 1067         count = 0;
 1068         while (numpages > 0) {
 1069                 struct bounce_page *bpage;
 1070 
 1071                 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
 1072                                                      M_NOWAIT | M_ZERO);
 1073 
 1074                 if (bpage == NULL)
 1075                         break;
 1076                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
 1077                                                          M_NOWAIT, 0ul,
 1078                                                          bz->lowaddr,
 1079                                                          PAGE_SIZE,
 1080                                                          0);
 1081                 if (bpage->vaddr == 0) {
 1082                         free(bpage, M_DEVBUF);
 1083                         break;
 1084                 }
 1085                 bpage->busaddr = pmap_kextract(bpage->vaddr);
 1086                 mtx_lock(&bounce_lock);
 1087                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
 1088                 total_bpages++;
 1089                 bz->total_bpages++;
 1090                 bz->free_bpages++;
 1091                 mtx_unlock(&bounce_lock);
 1092                 count++;
 1093                 numpages--;
 1094         }
 1095         return (count);
 1096 }
 1097 
 1098 static int
 1099 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
 1100 {
 1101         struct bounce_zone *bz;
 1102         int pages;
 1103 
 1104         mtx_assert(&bounce_lock, MA_OWNED);
 1105         bz = dmat->bounce_zone;
 1106         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
 1107         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
 1108                 return (map->pagesneeded - (map->pagesreserved + pages));
 1109         bz->free_bpages -= pages;
 1110         bz->reserved_bpages += pages;
 1111         map->pagesreserved += pages;
 1112         pages = map->pagesneeded - map->pagesreserved;
 1113 
 1114         return (pages);
 1115 }
 1116 
 1117 static bus_addr_t
 1118 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 1119                 bus_size_t size)
 1120 {
 1121         struct bounce_zone *bz;
 1122         struct bounce_page *bpage;
 1123 
 1124         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
 1125         KASSERT(map != NULL && map != &nobounce_dmamap,
 1126             ("add_bounce_page: bad map %p", map));
 1127 
 1128         bz = dmat->bounce_zone;
 1129         if (map->pagesneeded == 0)
 1130                 panic("add_bounce_page: map doesn't need any pages");
 1131         map->pagesneeded--;
 1132 
 1133         if (map->pagesreserved == 0)
 1134                 panic("add_bounce_page: map doesn't need any pages");
 1135         map->pagesreserved--;
 1136 
 1137         mtx_lock(&bounce_lock);
 1138         bpage = STAILQ_FIRST(&bz->bounce_page_list);
 1139         if (bpage == NULL)
 1140                 panic("add_bounce_page: free page list is empty");
 1141 
 1142         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
 1143         bz->reserved_bpages--;
 1144         bz->active_bpages++;
 1145         mtx_unlock(&bounce_lock);
 1146 
 1147         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1148                 /* Page offset needs to be preserved. */
 1149                 bpage->vaddr |= vaddr & PAGE_MASK;
 1150                 bpage->busaddr |= vaddr & PAGE_MASK;
 1151         }
 1152         bpage->datavaddr = vaddr;
 1153         bpage->datacount = size;
 1154         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
 1155         return (bpage->busaddr);
 1156 }
 1157 
 1158 static void
 1159 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
 1160 {
 1161         struct bus_dmamap *map;
 1162         struct bounce_zone *bz;
 1163 
 1164         bz = dmat->bounce_zone;
 1165         bpage->datavaddr = 0;
 1166         bpage->datacount = 0;
 1167         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1168                 /*
 1169                  * Reset the bounce page to start at offset 0.  Other uses
 1170                  * of this bounce page may need to store a full page of
 1171                  * data and/or assume it starts on a page boundary.
 1172                  */
 1173                 bpage->vaddr &= ~PAGE_MASK;
 1174                 bpage->busaddr &= ~PAGE_MASK;
 1175         }
 1176 
 1177         mtx_lock(&bounce_lock);
 1178         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
 1179         bz->free_bpages++;
 1180         bz->active_bpages--;
 1181         if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
 1182                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
 1183                         STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
 1184                         STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
 1185                                            map, links);
 1186                         busdma_swi_pending = 1;
 1187                         bz->total_deferred++;
 1188                         swi_sched(vm_ih, 0);
 1189                 }
 1190         }
 1191         mtx_unlock(&bounce_lock);
 1192 }
 1193 
 1194 void
 1195 busdma_swi(void)
 1196 {
 1197         bus_dma_tag_t dmat;
 1198         struct bus_dmamap *map;
 1199 
 1200         mtx_lock(&bounce_lock);
 1201         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
 1202                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
 1203                 mtx_unlock(&bounce_lock);
 1204                 dmat = map->dmat;
 1205                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
 1206                 bus_dmamap_load(map->dmat, map, map->buf, map->buflen,
 1207                                 map->callback, map->callback_arg, /*flags*/0);
 1208                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
 1209                 mtx_lock(&bounce_lock);
 1210         }
 1211         mtx_unlock(&bounce_lock);
 1212 }

Cache object: a9ae68272f2acea3ec188b8ad1ac3caf


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.