The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/busdma_machdep.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997, 1998 Justin T. Gibbs.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions, and the following disclaimer,
   10  *    without modification, immediately at the beginning of the file.
   11  * 2. The name of the author may not be used to endorse or promote products
   12  *    derived from this software without specific prior written permission.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
   18  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD: releng/6.4/sys/i386/i386/busdma_machdep.c 177306 2008-03-17 17:33:32Z jhb $");
   29 
   30 #include <sys/param.h>
   31 #include <sys/kdb.h>
   32 #include <ddb/ddb.h>
   33 #include <ddb/db_output.h>
   34 #include <sys/systm.h>
   35 #include <sys/malloc.h>
   36 #include <sys/bus.h>
   37 #include <sys/interrupt.h>
   38 #include <sys/kernel.h>
   39 #include <sys/ktr.h>
   40 #include <sys/lock.h>
   41 #include <sys/proc.h>
   42 #include <sys/mutex.h>
   43 #include <sys/mbuf.h>
   44 #include <sys/uio.h>
   45 #include <sys/sysctl.h>
   46 
   47 #include <vm/vm.h>
   48 #include <vm/vm_page.h>
   49 #include <vm/vm_map.h>
   50 
   51 #include <machine/atomic.h>
   52 #include <machine/bus.h>
   53 #include <machine/md_var.h>
   54 #include <machine/specialreg.h>
   55 
   56 #define MAX_BPAGES 512
   57 
   58 struct bounce_zone;
   59 
   60 struct bus_dma_tag {
   61         bus_dma_tag_t     parent;
   62         bus_size_t        alignment;
   63         bus_size_t        boundary;
   64         bus_addr_t        lowaddr;
   65         bus_addr_t        highaddr;
   66         bus_dma_filter_t *filter;
   67         void             *filterarg;
   68         bus_size_t        maxsize;
   69         u_int             nsegments;
   70         bus_size_t        maxsegsz;
   71         int               flags;
   72         int               ref_count;
   73         int               map_count;
   74         bus_dma_lock_t   *lockfunc;
   75         void             *lockfuncarg;
   76         bus_dma_segment_t *segments;
   77         struct bounce_zone *bounce_zone;
   78 };
   79 
   80 struct bounce_page {
   81         vm_offset_t     vaddr;          /* kva of bounce buffer */
   82         bus_addr_t      busaddr;        /* Physical address */
   83         vm_offset_t     datavaddr;      /* kva of client data */
   84         bus_size_t      datacount;      /* client data count */
   85         STAILQ_ENTRY(bounce_page) links;
   86 };
   87 
   88 int busdma_swi_pending;
   89 
   90 struct bounce_zone {
   91         STAILQ_ENTRY(bounce_zone) links;
   92         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
   93         int             total_bpages;
   94         int             free_bpages;
   95         int             reserved_bpages;
   96         int             active_bpages;
   97         int             total_bounced;
   98         int             total_deferred;
   99         bus_size_t      alignment;
  100         bus_size_t      boundary;
  101         bus_addr_t      lowaddr;
  102         char            zoneid[8];
  103         char            lowaddrid[20];
  104         struct sysctl_ctx_list sysctl_tree;
  105         struct sysctl_oid *sysctl_tree_top;
  106 };
  107 
  108 static struct mtx bounce_lock;
  109 static int total_bpages;
  110 static int busdma_zonecount;
  111 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
  112 
  113 SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
  114 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
  115            "Total bounce pages");
  116 
  117 struct bus_dmamap {
  118         struct bp_list         bpages;
  119         int                    pagesneeded;
  120         int                    pagesreserved;
  121         bus_dma_tag_t          dmat;
  122         void                  *buf;             /* unmapped buffer pointer */
  123         bus_size_t             buflen;          /* unmapped buffer length */
  124         bus_dmamap_callback_t *callback;
  125         void                  *callback_arg;
  126         STAILQ_ENTRY(bus_dmamap) links;
  127 };
  128 
  129 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
  130 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
  131 static struct bus_dmamap nobounce_dmamap;
  132 
  133 static void init_bounce_pages(void *dummy);
  134 static int alloc_bounce_zone(bus_dma_tag_t dmat);
  135 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
  136 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  137                                 int commit);
  138 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
  139                                    vm_offset_t vaddr, bus_size_t size);
  140 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
  141 static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
  142 
  143 /*
  144  * Return true if a match is made.
  145  *
  146  * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
  147  *
  148  * If paddr is within the bounds of the dma tag then call the filter callback
  149  * to check for a match, if there is no filter callback then assume a match.
  150  */
  151 static __inline int
  152 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
  153 {
  154         int retval;
  155 
  156         retval = 0;
  157 
  158         do {
  159                 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
  160                  || ((paddr & (dmat->alignment - 1)) != 0))
  161                  && (dmat->filter == NULL
  162                   || (*dmat->filter)(dmat->filterarg, paddr) != 0))
  163                         retval = 1;
  164 
  165                 dmat = dmat->parent;            
  166         } while (retval == 0 && dmat != NULL);
  167         return (retval);
  168 }
  169 
  170 /*
  171  * Convenience function for manipulating driver locks from busdma (during
  172  * busdma_swi, for example).  Drivers that don't provide their own locks
  173  * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
  174  * non-mutex locking scheme don't have to use this at all.
  175  */
  176 void
  177 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
  178 {
  179         struct mtx *dmtx;
  180 
  181         dmtx = (struct mtx *)arg;
  182         switch (op) {
  183         case BUS_DMA_LOCK:
  184                 mtx_lock(dmtx);
  185                 break;
  186         case BUS_DMA_UNLOCK:
  187                 mtx_unlock(dmtx);
  188                 break;
  189         default:
  190                 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
  191         }
  192 }
  193 
  194 /*
  195  * dflt_lock should never get called.  It gets put into the dma tag when
  196  * lockfunc == NULL, which is only valid if the maps that are associated
  197  * with the tag are meant to never be defered.
  198  * XXX Should have a way to identify which driver is responsible here.
  199  */
  200 static void
  201 dflt_lock(void *arg, bus_dma_lock_op_t op)
  202 {
  203         panic("driver error: busdma dflt_lock called");
  204 }
  205 
  206 #define BUS_DMA_COULD_BOUNCE    BUS_DMA_BUS3
  207 #define BUS_DMA_MIN_ALLOC_COMP  BUS_DMA_BUS4
  208 /*
  209  * Allocate a device specific dma_tag.
  210  */
  211 int
  212 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  213                    bus_size_t boundary, bus_addr_t lowaddr,
  214                    bus_addr_t highaddr, bus_dma_filter_t *filter,
  215                    void *filterarg, bus_size_t maxsize, int nsegments,
  216                    bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  217                    void *lockfuncarg, bus_dma_tag_t *dmat)
  218 {
  219         bus_dma_tag_t newtag;
  220         int error = 0;
  221 
  222         /* Basic sanity checking */
  223         if (boundary != 0 && boundary < maxsegsz)
  224                 maxsegsz = boundary;
  225 
  226         /* Return a NULL tag on failure */
  227         *dmat = NULL;
  228 
  229         newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
  230             M_ZERO | M_NOWAIT);
  231         if (newtag == NULL) {
  232                 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  233                     __func__, newtag, 0, error);
  234                 return (ENOMEM);
  235         }
  236 
  237         newtag->parent = parent;
  238         newtag->alignment = alignment;
  239         newtag->boundary = boundary;
  240         newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
  241         newtag->highaddr = trunc_page((vm_paddr_t)highaddr) +
  242             (PAGE_SIZE - 1);
  243         newtag->filter = filter;
  244         newtag->filterarg = filterarg;
  245         newtag->maxsize = maxsize;
  246         newtag->nsegments = nsegments;
  247         newtag->maxsegsz = maxsegsz;
  248         newtag->flags = flags;
  249         newtag->ref_count = 1; /* Count ourself */
  250         newtag->map_count = 0;
  251         if (lockfunc != NULL) {
  252                 newtag->lockfunc = lockfunc;
  253                 newtag->lockfuncarg = lockfuncarg;
  254         } else {
  255                 newtag->lockfunc = dflt_lock;
  256                 newtag->lockfuncarg = NULL;
  257         }
  258         newtag->segments = NULL;
  259 
  260         /* Take into account any restrictions imposed by our parent tag */
  261         if (parent != NULL) {
  262                 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
  263                 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
  264                 if (newtag->boundary == 0)
  265                         newtag->boundary = parent->boundary;
  266                 else if (parent->boundary != 0)
  267                         newtag->boundary = MIN(parent->boundary,
  268                                                newtag->boundary);
  269                 if (newtag->filter == NULL) {
  270                         /*
  271                          * Short circuit looking at our parent directly
  272                          * since we have encapsulated all of its information
  273                          */
  274                         newtag->filter = parent->filter;
  275                         newtag->filterarg = parent->filterarg;
  276                         newtag->parent = parent->parent;
  277                 }
  278                 if (newtag->parent != NULL)
  279                         atomic_add_int(&parent->ref_count, 1);
  280         }
  281 
  282         if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
  283          || newtag->alignment > 1)
  284                 newtag->flags |= BUS_DMA_COULD_BOUNCE;
  285 
  286         if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
  287             (flags & BUS_DMA_ALLOCNOW) != 0) {
  288                 struct bounce_zone *bz;
  289 
  290                 /* Must bounce */
  291 
  292                 if ((error = alloc_bounce_zone(newtag)) != 0) {
  293                         free(newtag, M_DEVBUF);
  294                         return (error);
  295                 }
  296                 bz = newtag->bounce_zone;
  297 
  298                 if (ptoa(bz->total_bpages) < maxsize) {
  299                         int pages;
  300 
  301                         pages = atop(maxsize) - bz->total_bpages;
  302 
  303                         /* Add pages to our bounce pool */
  304                         if (alloc_bounce_pages(newtag, pages) < pages)
  305                                 error = ENOMEM;
  306                 }
  307                 /* Performed initial allocation */
  308                 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
  309         }
  310         
  311         if (error != 0) {
  312                 free(newtag, M_DEVBUF);
  313         } else {
  314                 *dmat = newtag;
  315         }
  316         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  317             __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
  318         return (error);
  319 }
  320 
  321 int
  322 bus_dma_tag_destroy(bus_dma_tag_t dmat)
  323 {
  324         bus_dma_tag_t dmat_copy;
  325         int error;
  326 
  327         error = 0;
  328         dmat_copy = dmat;
  329 
  330         if (dmat != NULL) {
  331 
  332                 if (dmat->map_count != 0) {
  333                         error = EBUSY;
  334                         goto out;
  335                 }
  336 
  337                 while (dmat != NULL) {
  338                         bus_dma_tag_t parent;
  339 
  340                         parent = dmat->parent;
  341                         atomic_subtract_int(&dmat->ref_count, 1);
  342                         if (dmat->ref_count == 0) {
  343                                 if (dmat->segments != NULL)
  344                                         free(dmat->segments, M_DEVBUF);
  345                                 free(dmat, M_DEVBUF);
  346                                 /*
  347                                  * Last reference count, so
  348                                  * release our reference
  349                                  * count on our parent.
  350                                  */
  351                                 dmat = parent;
  352                         } else
  353                                 dmat = NULL;
  354                 }
  355         }
  356 out:
  357         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
  358         return (error);
  359 }
  360 
  361 /*
  362  * Allocate a handle for mapping from kva/uva/physical
  363  * address space into bus device space.
  364  */
  365 int
  366 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  367 {
  368         int error;
  369 
  370         error = 0;
  371 
  372         if (dmat->segments == NULL) {
  373                 dmat->segments = (bus_dma_segment_t *)malloc(
  374                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
  375                     M_NOWAIT);
  376                 if (dmat->segments == NULL) {
  377                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  378                             __func__, dmat, ENOMEM);
  379                         return (ENOMEM);
  380                 }
  381         }
  382 
  383         /*
  384          * Bouncing might be required if the driver asks for an active
  385          * exclusion region, a data alignment that is stricter than 1, and/or
  386          * an active address boundary.
  387          */
  388         if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
  389 
  390                 /* Must bounce */
  391                 struct bounce_zone *bz;
  392                 int maxpages;
  393 
  394                 if (dmat->bounce_zone == NULL) {
  395                         if ((error = alloc_bounce_zone(dmat)) != 0)
  396                                 return (error);
  397                 }
  398                 bz = dmat->bounce_zone;
  399 
  400                 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
  401                                              M_NOWAIT | M_ZERO);
  402                 if (*mapp == NULL) {
  403                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  404                             __func__, dmat, ENOMEM);
  405                         return (ENOMEM);
  406                 }
  407 
  408                 /* Initialize the new map */
  409                 STAILQ_INIT(&((*mapp)->bpages));
  410 
  411                 /*
  412                  * Attempt to add pages to our pool on a per-instance
  413                  * basis up to a sane limit.
  414                  */
  415                 if (dmat->alignment > 1)
  416                         maxpages = MAX_BPAGES;
  417                 else
  418                         maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
  419                 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
  420                  || (dmat->map_count > 0 && bz->total_bpages < maxpages)) {
  421                         int pages;
  422 
  423                         pages = MAX(atop(dmat->maxsize), 1);
  424                         pages = MIN(maxpages - bz->total_bpages, pages);
  425                         pages = MAX(pages, 1);
  426                         if (alloc_bounce_pages(dmat, pages) < pages)
  427                                 error = ENOMEM;
  428 
  429                         if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
  430                                 if (error == 0)
  431                                         dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
  432                         } else {
  433                                 error = 0;
  434                         }
  435                 }
  436         } else {
  437                 *mapp = NULL;
  438         }
  439         if (error == 0)
  440                 dmat->map_count++;
  441         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  442             __func__, dmat, dmat->flags, error);
  443         return (error);
  444 }
  445 
  446 /*
  447  * Destroy a handle for mapping from kva/uva/physical
  448  * address space into bus device space.
  449  */
  450 int
  451 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
  452 {
  453         if (map != NULL && map != &nobounce_dmamap) {
  454                 if (STAILQ_FIRST(&map->bpages) != NULL) {
  455                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  456                             __func__, dmat, EBUSY);
  457                         return (EBUSY);
  458                 }
  459                 free(map, M_DEVBUF);
  460         }
  461         dmat->map_count--;
  462         CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
  463         return (0);
  464 }
  465 
  466 
  467 /*
  468  * Allocate a piece of memory that can be efficiently mapped into
  469  * bus device space based on the constraints lited in the dma tag.
  470  * A dmamap to for use with dmamap_load is also allocated.
  471  */
  472 int
  473 bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
  474                  bus_dmamap_t *mapp)
  475 {
  476         int mflags;
  477 
  478         if (flags & BUS_DMA_NOWAIT)
  479                 mflags = M_NOWAIT;
  480         else
  481                 mflags = M_WAITOK;
  482         if (flags & BUS_DMA_ZERO)
  483                 mflags |= M_ZERO;
  484 
  485         /* If we succeed, no mapping/bouncing will be required */
  486         *mapp = NULL;
  487 
  488         if (dmat->segments == NULL) {
  489                 dmat->segments = (bus_dma_segment_t *)malloc(
  490                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
  491                     M_NOWAIT);
  492                 if (dmat->segments == NULL) {
  493                         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  494                             __func__, dmat, dmat->flags, ENOMEM);
  495                         return (ENOMEM);
  496                 }
  497         }
  498 
  499         /* 
  500          * XXX:
  501          * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
  502          * alignment guarantees of malloc need to be nailed down, and the
  503          * code below should be rewritten to take that into account.
  504          *
  505          * In the meantime, we'll warn the user if malloc gets it wrong.
  506          */
  507         if ((dmat->maxsize <= PAGE_SIZE) &&
  508            (dmat->alignment < dmat->maxsize) &&
  509             dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) {
  510                 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
  511         } else {
  512                 /*
  513                  * XXX Use Contigmalloc until it is merged into this facility
  514                  *     and handles multi-seg allocations.  Nobody is doing
  515                  *     multi-seg allocations yet though.
  516                  * XXX Certain AGP hardware does.
  517                  */
  518                 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags,
  519                     0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul,
  520                     dmat->boundary);
  521         }
  522         if (*vaddr == NULL) {
  523                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  524                     __func__, dmat, dmat->flags, ENOMEM);
  525                 return (ENOMEM);
  526         } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) {
  527                 printf("bus_dmamem_alloc failed to align memory properly.\n");
  528         }
  529         if (flags & BUS_DMA_NOCACHE)
  530                 pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize,
  531                     PAT_UNCACHEABLE);
  532         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  533             __func__, dmat, dmat->flags, ENOMEM);
  534         return (0);
  535 }
  536 
  537 /*
  538  * Free a piece of memory and it's allociated dmamap, that was allocated
  539  * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
  540  */
  541 void
  542 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
  543 {
  544         /*
  545          * dmamem does not need to be bounced, so the map should be
  546          * NULL
  547          */
  548         if (map != NULL)
  549                 panic("bus_dmamem_free: Invalid map freed\n");
  550         pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, PAT_WRITE_BACK);
  551         if ((dmat->maxsize <= PAGE_SIZE) &&
  552            (dmat->alignment < dmat->maxsize) &&
  553             dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem))
  554                 free(vaddr, M_DEVBUF);
  555         else {
  556                 contigfree(vaddr, dmat->maxsize, M_DEVBUF);
  557         }
  558         CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
  559 }
  560 
  561 /*
  562  * Utility function to load a linear buffer.  lastaddrp holds state
  563  * between invocations (for multiple-buffer loads).  segp contains
  564  * the starting segment on entrace, and the ending segment on exit.
  565  * first indicates if this is the first invocation of this function.
  566  */
  567 static __inline int
  568 _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
  569                         bus_dmamap_t map,
  570                         void *buf, bus_size_t buflen,
  571                         pmap_t pmap,
  572                         int flags,
  573                         bus_addr_t *lastaddrp,
  574                         bus_dma_segment_t *segs,
  575                         int *segp,
  576                         int first)
  577 {
  578         bus_size_t sgsize;
  579         bus_addr_t curaddr, lastaddr, baddr, bmask;
  580         vm_offset_t vaddr;
  581         bus_addr_t paddr;
  582         int seg;
  583 
  584         if (map == NULL)
  585                 map = &nobounce_dmamap;
  586 
  587         if ((map != &nobounce_dmamap && map->pagesneeded == 0) 
  588          && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) {
  589                 vm_offset_t     vendaddr;
  590 
  591                 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
  592                     "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
  593                     dmat->boundary, dmat->alignment);
  594                 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
  595                     map, &nobounce_dmamap, map->pagesneeded);
  596                 /*
  597                  * Count the number of bounce pages
  598                  * needed in order to complete this transfer
  599                  */
  600                 vaddr = trunc_page((vm_offset_t)buf);
  601                 vendaddr = (vm_offset_t)buf + buflen;
  602 
  603                 while (vaddr < vendaddr) {
  604                         paddr = pmap_kextract(vaddr);
  605                         if (run_filter(dmat, paddr) != 0)
  606                                 map->pagesneeded++;
  607                         vaddr += PAGE_SIZE;
  608                 }
  609                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
  610         }
  611 
  612         /* Reserve Necessary Bounce Pages */
  613         if (map->pagesneeded != 0) {
  614                 mtx_lock(&bounce_lock);
  615                 if (flags & BUS_DMA_NOWAIT) {
  616                         if (reserve_bounce_pages(dmat, map, 0) != 0) {
  617                                 mtx_unlock(&bounce_lock);
  618                                 return (ENOMEM);
  619                         }
  620                 } else {
  621                         if (reserve_bounce_pages(dmat, map, 1) != 0) {
  622                                 /* Queue us for resources */
  623                                 map->dmat = dmat;
  624                                 map->buf = buf;
  625                                 map->buflen = buflen;
  626                                 STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
  627                                     map, links);
  628                                 mtx_unlock(&bounce_lock);
  629                                 return (EINPROGRESS);
  630                         }
  631                 }
  632                 mtx_unlock(&bounce_lock);
  633         }
  634 
  635         vaddr = (vm_offset_t)buf;
  636         lastaddr = *lastaddrp;
  637         bmask = ~(dmat->boundary - 1);
  638 
  639         for (seg = *segp; buflen > 0 ; ) {
  640                 /*
  641                  * Get the physical address for this segment.
  642                  */
  643                 if (pmap)
  644                         curaddr = pmap_extract(pmap, vaddr);
  645                 else
  646                         curaddr = pmap_kextract(vaddr);
  647 
  648                 /*
  649                  * Compute the segment size, and adjust counts.
  650                  */
  651                 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
  652                 if (buflen < sgsize)
  653                         sgsize = buflen;
  654 
  655                 /*
  656                  * Make sure we don't cross any boundaries.
  657                  */
  658                 if (dmat->boundary > 0) {
  659                         baddr = (curaddr + dmat->boundary) & bmask;
  660                         if (sgsize > (baddr - curaddr))
  661                                 sgsize = (baddr - curaddr);
  662                 }
  663 
  664                 if (map->pagesneeded != 0 && run_filter(dmat, curaddr))
  665                         curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
  666 
  667                 /*
  668                  * Insert chunk into a segment, coalescing with
  669                  * previous segment if possible.
  670                  */
  671                 if (first) {
  672                         segs[seg].ds_addr = curaddr;
  673                         segs[seg].ds_len = sgsize;
  674                         first = 0;
  675                 } else {
  676                         if (curaddr == lastaddr &&
  677                             (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
  678                             (dmat->boundary == 0 ||
  679                              (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
  680                                 segs[seg].ds_len += sgsize;
  681                         else {
  682                                 if (++seg >= dmat->nsegments)
  683                                         break;
  684                                 segs[seg].ds_addr = curaddr;
  685                                 segs[seg].ds_len = sgsize;
  686                         }
  687                 }
  688 
  689                 lastaddr = curaddr + sgsize;
  690                 vaddr += sgsize;
  691                 buflen -= sgsize;
  692         }
  693 
  694         *segp = seg;
  695         *lastaddrp = lastaddr;
  696 
  697         /*
  698          * Did we fit?
  699          */
  700         return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
  701 }
  702 
  703 /*
  704  * Map the buffer buf into bus space using the dmamap map.
  705  */
  706 int
  707 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
  708                 bus_size_t buflen, bus_dmamap_callback_t *callback,
  709                 void *callback_arg, int flags)
  710 {
  711         bus_addr_t              lastaddr = 0;
  712         int                     error, nsegs = 0;
  713 
  714         if (map != NULL) {
  715                 flags |= BUS_DMA_WAITOK;
  716                 map->callback = callback;
  717                 map->callback_arg = callback_arg;
  718         }
  719 
  720         error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags,
  721              &lastaddr, dmat->segments, &nsegs, 1);
  722 
  723         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  724             __func__, dmat, dmat->flags, error, nsegs + 1);
  725 
  726         if (error == EINPROGRESS) {
  727                 return (error);
  728         }
  729 
  730         if (error)
  731                 (*callback)(callback_arg, dmat->segments, 0, error);
  732         else
  733                 (*callback)(callback_arg, dmat->segments, nsegs + 1, 0);
  734 
  735         /*
  736          * Return ENOMEM to the caller so that it can pass it up the stack.
  737          * This error only happens when NOWAIT is set, so deferal is disabled.
  738          */
  739         if (error == ENOMEM)
  740                 return (error);
  741 
  742         return (0);
  743 }
  744 
  745 
  746 /*
  747  * Like _bus_dmamap_load(), but for mbufs.
  748  */
  749 int
  750 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
  751                      struct mbuf *m0,
  752                      bus_dmamap_callback2_t *callback, void *callback_arg,
  753                      int flags)
  754 {
  755         int nsegs, error;
  756 
  757         M_ASSERTPKTHDR(m0);
  758 
  759         flags |= BUS_DMA_NOWAIT;
  760         nsegs = 0;
  761         error = 0;
  762         if (m0->m_pkthdr.len <= dmat->maxsize) {
  763                 int first = 1;
  764                 bus_addr_t lastaddr = 0;
  765                 struct mbuf *m;
  766 
  767                 for (m = m0; m != NULL && error == 0; m = m->m_next) {
  768                         if (m->m_len > 0) {
  769                                 error = _bus_dmamap_load_buffer(dmat, map,
  770                                                 m->m_data, m->m_len,
  771                                                 NULL, flags, &lastaddr,
  772                                                 dmat->segments, &nsegs, first);
  773                                 first = 0;
  774                         }
  775                 }
  776         } else {
  777                 error = EINVAL;
  778         }
  779 
  780         if (error) {
  781                 /* force "no valid mappings" in callback */
  782                 (*callback)(callback_arg, dmat->segments, 0, 0, error);
  783         } else {
  784                 (*callback)(callback_arg, dmat->segments,
  785                             nsegs+1, m0->m_pkthdr.len, error);
  786         }
  787         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  788             __func__, dmat, dmat->flags, error, nsegs + 1);
  789         return (error);
  790 }
  791 
  792 int
  793 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
  794                         struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
  795                         int flags)
  796 {
  797         int error;
  798 
  799         M_ASSERTPKTHDR(m0);
  800 
  801         flags |= BUS_DMA_NOWAIT;
  802         *nsegs = 0;
  803         error = 0;
  804         if (m0->m_pkthdr.len <= dmat->maxsize) {
  805                 int first = 1;
  806                 bus_addr_t lastaddr = 0;
  807                 struct mbuf *m;
  808 
  809                 for (m = m0; m != NULL && error == 0; m = m->m_next) {
  810                         if (m->m_len > 0) {
  811                                 error = _bus_dmamap_load_buffer(dmat, map,
  812                                                 m->m_data, m->m_len,
  813                                                 NULL, flags, &lastaddr,
  814                                                 segs, nsegs, first);
  815                                 first = 0;
  816                         }
  817                 }
  818         } else {
  819                 error = EINVAL;
  820         }
  821 
  822         /* XXX FIXME: Having to increment nsegs is really annoying */
  823         ++*nsegs;
  824         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  825             __func__, dmat, dmat->flags, error, *nsegs);
  826         return (error);
  827 }
  828 
  829 /*
  830  * Like _bus_dmamap_load(), but for uios.
  831  */
  832 int
  833 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
  834                     struct uio *uio,
  835                     bus_dmamap_callback2_t *callback, void *callback_arg,
  836                     int flags)
  837 {
  838         bus_addr_t lastaddr;
  839         int nsegs, error, first, i;
  840         bus_size_t resid;
  841         struct iovec *iov;
  842         pmap_t pmap;
  843 
  844         flags |= BUS_DMA_NOWAIT;
  845         resid = uio->uio_resid;
  846         iov = uio->uio_iov;
  847 
  848         if (uio->uio_segflg == UIO_USERSPACE) {
  849                 KASSERT(uio->uio_td != NULL,
  850                         ("bus_dmamap_load_uio: USERSPACE but no proc"));
  851                 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
  852         } else
  853                 pmap = NULL;
  854 
  855         nsegs = 0;
  856         error = 0;
  857         first = 1;
  858         lastaddr = (bus_addr_t) 0;
  859         for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
  860                 /*
  861                  * Now at the first iovec to load.  Load each iovec
  862                  * until we have exhausted the residual count.
  863                  */
  864                 bus_size_t minlen =
  865                         resid < iov[i].iov_len ? resid : iov[i].iov_len;
  866                 caddr_t addr = (caddr_t) iov[i].iov_base;
  867 
  868                 if (minlen > 0) {
  869                         error = _bus_dmamap_load_buffer(dmat, map,
  870                                         addr, minlen, pmap, flags, &lastaddr,
  871                                         dmat->segments, &nsegs, first);
  872                         first = 0;
  873 
  874                         resid -= minlen;
  875                 }
  876         }
  877 
  878         if (error) {
  879                 /* force "no valid mappings" in callback */
  880                 (*callback)(callback_arg, dmat->segments, 0, 0, error);
  881         } else {
  882                 (*callback)(callback_arg, dmat->segments,
  883                             nsegs+1, uio->uio_resid, error);
  884         }
  885         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  886             __func__, dmat, dmat->flags, error, nsegs + 1);
  887         return (error);
  888 }
  889 
  890 /*
  891  * Release the mapping held by map.
  892  */
  893 void
  894 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
  895 {
  896         struct bounce_page *bpage;
  897 
  898         while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
  899                 STAILQ_REMOVE_HEAD(&map->bpages, links);
  900                 free_bounce_page(dmat, bpage);
  901         }
  902 }
  903 
  904 void
  905 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
  906 {
  907         struct bounce_page *bpage;
  908 
  909         if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
  910                 /*
  911                  * Handle data bouncing.  We might also
  912                  * want to add support for invalidating
  913                  * the caches on broken hardware
  914                  */
  915                 dmat->bounce_zone->total_bounced++;
  916                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
  917                     "performing bounce", __func__, op, dmat, dmat->flags);
  918 
  919                 if (op & BUS_DMASYNC_PREWRITE) {
  920                         while (bpage != NULL) {
  921                                 bcopy((void *)bpage->datavaddr,
  922                                       (void *)bpage->vaddr,
  923                                       bpage->datacount);
  924                                 bpage = STAILQ_NEXT(bpage, links);
  925                         }
  926                 }
  927 
  928                 if (op & BUS_DMASYNC_POSTREAD) {
  929                         while (bpage != NULL) {
  930                                 bcopy((void *)bpage->vaddr,
  931                                       (void *)bpage->datavaddr,
  932                                       bpage->datacount);
  933                                 bpage = STAILQ_NEXT(bpage, links);
  934                         }
  935                 }
  936         }
  937 }
  938 
  939 static void
  940 init_bounce_pages(void *dummy __unused)
  941 {
  942 
  943         total_bpages = 0;
  944         STAILQ_INIT(&bounce_zone_list);
  945         STAILQ_INIT(&bounce_map_waitinglist);
  946         STAILQ_INIT(&bounce_map_callbacklist);
  947         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
  948 }
  949 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
  950 
  951 static struct sysctl_ctx_list *
  952 busdma_sysctl_tree(struct bounce_zone *bz)
  953 {
  954         return (&bz->sysctl_tree);
  955 }
  956 
  957 static struct sysctl_oid *
  958 busdma_sysctl_tree_top(struct bounce_zone *bz)
  959 {
  960         return (bz->sysctl_tree_top);
  961 }
  962 
  963 static int
  964 alloc_bounce_zone(bus_dma_tag_t dmat)
  965 {
  966         struct bounce_zone *bz;
  967 
  968         /* Check to see if we already have a suitable zone */
  969         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
  970                 if ((dmat->alignment <= bz->alignment)
  971                  && (dmat->boundary <= bz->boundary)
  972                  && (dmat->lowaddr >= bz->lowaddr)) {
  973                         dmat->bounce_zone = bz;
  974                         return (0);
  975                 }
  976         }
  977 
  978         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
  979             M_NOWAIT | M_ZERO)) == NULL)
  980                 return (ENOMEM);
  981 
  982         STAILQ_INIT(&bz->bounce_page_list);
  983         bz->free_bpages = 0;
  984         bz->reserved_bpages = 0;
  985         bz->active_bpages = 0;
  986         bz->lowaddr = dmat->lowaddr;
  987         bz->alignment = dmat->alignment;
  988         bz->boundary = dmat->boundary;
  989         snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
  990         busdma_zonecount++;
  991         snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
  992         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
  993         dmat->bounce_zone = bz;
  994 
  995         sysctl_ctx_init(&bz->sysctl_tree);
  996         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
  997             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
  998             CTLFLAG_RD, 0, "");
  999         if (bz->sysctl_tree_top == NULL) {
 1000                 sysctl_ctx_free(&bz->sysctl_tree);
 1001                 return (0);     /* XXX error code? */
 1002         }
 1003 
 1004         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1005             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1006             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
 1007             "Total bounce pages");
 1008         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1009             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1010             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
 1011             "Free bounce pages");
 1012         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1013             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1014             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
 1015             "Reserved bounce pages");
 1016         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1017             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1018             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
 1019             "Active bounce pages");
 1020         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1021             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1022             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
 1023             "Total bounce requests");
 1024         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1025             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1026             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
 1027             "Total bounce requests that were deferred");
 1028         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
 1029             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1030             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
 1031         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1032             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1033             "alignment", CTLFLAG_RD, &bz->alignment, 0, "");
 1034         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1035             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1036             "boundary", CTLFLAG_RD, &bz->boundary, 0, "");
 1037 
 1038         return (0);
 1039 }
 1040 
 1041 static int
 1042 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
 1043 {
 1044         struct bounce_zone *bz;
 1045         int count;
 1046 
 1047         bz = dmat->bounce_zone;
 1048         count = 0;
 1049         while (numpages > 0) {
 1050                 struct bounce_page *bpage;
 1051 
 1052                 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
 1053                                                      M_NOWAIT | M_ZERO);
 1054 
 1055                 if (bpage == NULL)
 1056                         break;
 1057                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
 1058                                                          M_NOWAIT, 0ul,
 1059                                                          bz->lowaddr,
 1060                                                          PAGE_SIZE,
 1061                                                          bz->boundary);
 1062                 if (bpage->vaddr == 0) {
 1063                         free(bpage, M_DEVBUF);
 1064                         break;
 1065                 }
 1066                 bpage->busaddr = pmap_kextract(bpage->vaddr);
 1067                 mtx_lock(&bounce_lock);
 1068                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
 1069                 total_bpages++;
 1070                 bz->total_bpages++;
 1071                 bz->free_bpages++;
 1072                 mtx_unlock(&bounce_lock);
 1073                 count++;
 1074                 numpages--;
 1075         }
 1076         return (count);
 1077 }
 1078 
 1079 static int
 1080 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
 1081 {
 1082         struct bounce_zone *bz;
 1083         int pages;
 1084 
 1085         mtx_assert(&bounce_lock, MA_OWNED);
 1086         bz = dmat->bounce_zone;
 1087         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
 1088         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
 1089                 return (map->pagesneeded - (map->pagesreserved + pages));
 1090         bz->free_bpages -= pages;
 1091         bz->reserved_bpages += pages;
 1092         map->pagesreserved += pages;
 1093         pages = map->pagesneeded - map->pagesreserved;
 1094 
 1095         return (pages);
 1096 }
 1097 
 1098 static bus_addr_t
 1099 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 1100                 bus_size_t size)
 1101 {
 1102         struct bounce_zone *bz;
 1103         struct bounce_page *bpage;
 1104 
 1105         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
 1106         KASSERT(map != NULL && map != &nobounce_dmamap,
 1107             ("add_bounce_page: bad map %p", map));
 1108 
 1109         bz = dmat->bounce_zone;
 1110         if (map->pagesneeded == 0)
 1111                 panic("add_bounce_page: map doesn't need any pages");
 1112         map->pagesneeded--;
 1113 
 1114         if (map->pagesreserved == 0)
 1115                 panic("add_bounce_page: map doesn't need any pages");
 1116         map->pagesreserved--;
 1117 
 1118         mtx_lock(&bounce_lock);
 1119         bpage = STAILQ_FIRST(&bz->bounce_page_list);
 1120         if (bpage == NULL)
 1121                 panic("add_bounce_page: free page list is empty");
 1122 
 1123         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
 1124         bz->reserved_bpages--;
 1125         bz->active_bpages++;
 1126         mtx_unlock(&bounce_lock);
 1127 
 1128         bpage->datavaddr = vaddr;
 1129         bpage->datacount = size;
 1130         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
 1131         return (bpage->busaddr);
 1132 }
 1133 
 1134 static void
 1135 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
 1136 {
 1137         struct bus_dmamap *map;
 1138         struct bounce_zone *bz;
 1139 
 1140         bz = dmat->bounce_zone;
 1141         bpage->datavaddr = 0;
 1142         bpage->datacount = 0;
 1143 
 1144         mtx_lock(&bounce_lock);
 1145         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
 1146         bz->free_bpages++;
 1147         bz->active_bpages--;
 1148         if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
 1149                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
 1150                         STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
 1151                         STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
 1152                                            map, links);
 1153                         busdma_swi_pending = 1;
 1154                         bz->total_deferred++;
 1155                         swi_sched(vm_ih, 0);
 1156                 }
 1157         }
 1158         mtx_unlock(&bounce_lock);
 1159 }
 1160 
 1161 void
 1162 busdma_swi(void)
 1163 {
 1164         bus_dma_tag_t dmat;
 1165         struct bus_dmamap *map;
 1166 
 1167         mtx_lock(&bounce_lock);
 1168         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
 1169                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
 1170                 mtx_unlock(&bounce_lock);
 1171                 dmat = map->dmat;
 1172                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
 1173                 bus_dmamap_load(map->dmat, map, map->buf, map->buflen,
 1174                                 map->callback, map->callback_arg, /*flags*/0);
 1175                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
 1176                 mtx_lock(&bounce_lock);
 1177         }
 1178         mtx_unlock(&bounce_lock);
 1179 }

Cache object: 9ca775c052858e9fa3e191fd3b81118d


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.