The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/x86/busdma_machdep.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997, 1998 Justin T. Gibbs.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions, and the following disclaimer,
   10  *    without modification, immediately at the beginning of the file.
   11  * 2. The name of the author may not be used to endorse or promote products
   12  *    derived from this software without specific prior written permission.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
   18  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD: releng/9.1/sys/x86/x86/busdma_machdep.c 232372 2012-03-02 00:21:07Z emaste $");
   29 
   30 #include <sys/param.h>
   31 #include <sys/systm.h>
   32 #include <sys/malloc.h>
   33 #include <sys/bus.h>
   34 #include <sys/interrupt.h>
   35 #include <sys/kernel.h>
   36 #include <sys/ktr.h>
   37 #include <sys/lock.h>
   38 #include <sys/proc.h>
   39 #include <sys/mutex.h>
   40 #include <sys/mbuf.h>
   41 #include <sys/uio.h>
   42 #include <sys/sysctl.h>
   43 
   44 #include <vm/vm.h>
   45 #include <vm/vm_page.h>
   46 #include <vm/vm_map.h>
   47 
   48 #include <machine/atomic.h>
   49 #include <machine/bus.h>
   50 #include <machine/md_var.h>
   51 #include <machine/specialreg.h>
   52 
   53 #ifdef __i386__
   54 #define MAX_BPAGES 512
   55 #else
   56 #define MAX_BPAGES 8192
   57 #endif
   58 #define BUS_DMA_COULD_BOUNCE    BUS_DMA_BUS3
   59 #define BUS_DMA_MIN_ALLOC_COMP  BUS_DMA_BUS4
   60 
   61 struct bounce_zone;
   62 
   63 struct bus_dma_tag {
   64         bus_dma_tag_t     parent;
   65         bus_size_t        alignment;
   66         bus_size_t        boundary;
   67         bus_addr_t        lowaddr;
   68         bus_addr_t        highaddr;
   69         bus_dma_filter_t *filter;
   70         void             *filterarg;
   71         bus_size_t        maxsize;
   72         u_int             nsegments;
   73         bus_size_t        maxsegsz;
   74         int               flags;
   75         int               ref_count;
   76         int               map_count;
   77         bus_dma_lock_t   *lockfunc;
   78         void             *lockfuncarg;
   79         bus_dma_segment_t *segments;
   80         struct bounce_zone *bounce_zone;
   81 };
   82 
   83 struct bounce_page {
   84         vm_offset_t     vaddr;          /* kva of bounce buffer */
   85         bus_addr_t      busaddr;        /* Physical address */
   86         vm_offset_t     datavaddr;      /* kva of client data */
   87         bus_size_t      datacount;      /* client data count */
   88         STAILQ_ENTRY(bounce_page) links;
   89 };
   90 
   91 int busdma_swi_pending;
   92 
   93 struct bounce_zone {
   94         STAILQ_ENTRY(bounce_zone) links;
   95         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
   96         int             total_bpages;
   97         int             free_bpages;
   98         int             reserved_bpages;
   99         int             active_bpages;
  100         int             total_bounced;
  101         int             total_deferred;
  102         int             map_count;
  103         bus_size_t      alignment;
  104         bus_addr_t      lowaddr;
  105         char            zoneid[8];
  106         char            lowaddrid[20];
  107         struct sysctl_ctx_list sysctl_tree;
  108         struct sysctl_oid *sysctl_tree_top;
  109 };
  110 
  111 static struct mtx bounce_lock;
  112 static int total_bpages;
  113 static int busdma_zonecount;
  114 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
  115 
  116 SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
  117 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
  118            "Total bounce pages");
  119 
  120 struct bus_dmamap {
  121         struct bp_list         bpages;
  122         int                    pagesneeded;
  123         int                    pagesreserved;
  124         bus_dma_tag_t          dmat;
  125         void                  *buf;             /* unmapped buffer pointer */
  126         bus_size_t             buflen;          /* unmapped buffer length */
  127         bus_dmamap_callback_t *callback;
  128         void                  *callback_arg;
  129         STAILQ_ENTRY(bus_dmamap) links;
  130 };
  131 
  132 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
  133 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
  134 static struct bus_dmamap nobounce_dmamap;
  135 
  136 static void init_bounce_pages(void *dummy);
  137 static int alloc_bounce_zone(bus_dma_tag_t dmat);
  138 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
  139 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
  140                                 int commit);
  141 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
  142                                    vm_offset_t vaddr, bus_size_t size);
  143 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
  144 int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
  145 int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
  146     void *buf, bus_size_t buflen, int flags);
  147 
  148 #ifdef XEN
  149 #undef pmap_kextract
  150 #define pmap_kextract pmap_kextract_ma
  151 #endif
  152 
  153 /*
  154  * Return true if a match is made.
  155  *
  156  * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
  157  *
  158  * If paddr is within the bounds of the dma tag then call the filter callback
  159  * to check for a match, if there is no filter callback then assume a match.
  160  */
  161 int
  162 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
  163 {
  164         int retval;
  165 
  166         retval = 0;
  167 
  168         do {
  169                 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
  170                  || ((paddr & (dmat->alignment - 1)) != 0))
  171                  && (dmat->filter == NULL
  172                   || (*dmat->filter)(dmat->filterarg, paddr) != 0))
  173                         retval = 1;
  174 
  175                 dmat = dmat->parent;            
  176         } while (retval == 0 && dmat != NULL);
  177         return (retval);
  178 }
  179 
  180 /*
  181  * Convenience function for manipulating driver locks from busdma (during
  182  * busdma_swi, for example).  Drivers that don't provide their own locks
  183  * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
  184  * non-mutex locking scheme don't have to use this at all.
  185  */
  186 void
  187 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
  188 {
  189         struct mtx *dmtx;
  190 
  191         dmtx = (struct mtx *)arg;
  192         switch (op) {
  193         case BUS_DMA_LOCK:
  194                 mtx_lock(dmtx);
  195                 break;
  196         case BUS_DMA_UNLOCK:
  197                 mtx_unlock(dmtx);
  198                 break;
  199         default:
  200                 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
  201         }
  202 }
  203 
  204 /*
  205  * dflt_lock should never get called.  It gets put into the dma tag when
  206  * lockfunc == NULL, which is only valid if the maps that are associated
  207  * with the tag are meant to never be defered.
  208  * XXX Should have a way to identify which driver is responsible here.
  209  */
  210 static void
  211 dflt_lock(void *arg, bus_dma_lock_op_t op)
  212 {
  213         panic("driver error: busdma dflt_lock called");
  214 }
  215 
  216 /*
  217  * Allocate a device specific dma_tag.
  218  */
  219 int
  220 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  221                    bus_size_t boundary, bus_addr_t lowaddr,
  222                    bus_addr_t highaddr, bus_dma_filter_t *filter,
  223                    void *filterarg, bus_size_t maxsize, int nsegments,
  224                    bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  225                    void *lockfuncarg, bus_dma_tag_t *dmat)
  226 {
  227         bus_dma_tag_t newtag;
  228         int error = 0;
  229 
  230         /* Always enforce at least a 4GB (2GB for PAE) boundary. */
  231 #if defined(__amd64__)
  232         if (boundary == 0 || boundary > ((bus_addr_t)1 << 32))
  233                 boundary = (bus_size_t)1 << 32;
  234 #elif defined(PAE)
  235         if (boundary == 0 || boundary > ((bus_addr_t)1 << 31))
  236                 boundary = (bus_size_t)1 << 31;
  237 #endif
  238         /* Basic sanity checking */
  239         if (boundary != 0 && boundary < maxsegsz)
  240                 maxsegsz = boundary;
  241 
  242         if (maxsegsz == 0) {
  243                 return (EINVAL);
  244         }
  245 
  246         /* Return a NULL tag on failure */
  247         *dmat = NULL;
  248 
  249         newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
  250             M_ZERO | M_NOWAIT);
  251         if (newtag == NULL) {
  252                 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  253                     __func__, newtag, 0, error);
  254                 return (ENOMEM);
  255         }
  256 
  257         newtag->parent = parent;
  258         newtag->alignment = alignment;
  259         newtag->boundary = boundary;
  260         newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
  261         newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
  262         newtag->filter = filter;
  263         newtag->filterarg = filterarg;
  264         newtag->maxsize = maxsize;
  265         newtag->nsegments = nsegments;
  266         newtag->maxsegsz = maxsegsz;
  267         newtag->flags = flags;
  268         newtag->ref_count = 1; /* Count ourself */
  269         newtag->map_count = 0;
  270         if (lockfunc != NULL) {
  271                 newtag->lockfunc = lockfunc;
  272                 newtag->lockfuncarg = lockfuncarg;
  273         } else {
  274                 newtag->lockfunc = dflt_lock;
  275                 newtag->lockfuncarg = NULL;
  276         }
  277         newtag->segments = NULL;
  278 
  279         /* Take into account any restrictions imposed by our parent tag */
  280         if (parent != NULL) {
  281                 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
  282                 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
  283                 if (newtag->boundary == 0)
  284                         newtag->boundary = parent->boundary;
  285                 else if (parent->boundary != 0)
  286                         newtag->boundary = MIN(parent->boundary,
  287                                                newtag->boundary);
  288                 if ((newtag->filter != NULL) ||
  289                     ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
  290                         newtag->flags |= BUS_DMA_COULD_BOUNCE;
  291                 if (newtag->filter == NULL) {
  292                         /*
  293                          * Short circuit looking at our parent directly
  294                          * since we have encapsulated all of its information
  295                          */
  296                         newtag->filter = parent->filter;
  297                         newtag->filterarg = parent->filterarg;
  298                         newtag->parent = parent->parent;
  299                 }
  300                 if (newtag->parent != NULL)
  301                         atomic_add_int(&parent->ref_count, 1);
  302         }
  303 
  304         if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
  305          || newtag->alignment > 1)
  306                 newtag->flags |= BUS_DMA_COULD_BOUNCE;
  307 
  308         if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
  309             (flags & BUS_DMA_ALLOCNOW) != 0) {
  310                 struct bounce_zone *bz;
  311 
  312                 /* Must bounce */
  313 
  314                 if ((error = alloc_bounce_zone(newtag)) != 0) {
  315                         free(newtag, M_DEVBUF);
  316                         return (error);
  317                 }
  318                 bz = newtag->bounce_zone;
  319 
  320                 if (ptoa(bz->total_bpages) < maxsize) {
  321                         int pages;
  322 
  323                         pages = atop(maxsize) - bz->total_bpages;
  324 
  325                         /* Add pages to our bounce pool */
  326                         if (alloc_bounce_pages(newtag, pages) < pages)
  327                                 error = ENOMEM;
  328                 }
  329                 /* Performed initial allocation */
  330                 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
  331         }
  332         
  333         if (error != 0) {
  334                 free(newtag, M_DEVBUF);
  335         } else {
  336                 *dmat = newtag;
  337         }
  338         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  339             __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
  340         return (error);
  341 }
  342 
  343 int
  344 bus_dma_tag_destroy(bus_dma_tag_t dmat)
  345 {
  346         bus_dma_tag_t dmat_copy;
  347         int error;
  348 
  349         error = 0;
  350         dmat_copy = dmat;
  351 
  352         if (dmat != NULL) {
  353 
  354                 if (dmat->map_count != 0) {
  355                         error = EBUSY;
  356                         goto out;
  357                 }
  358 
  359                 while (dmat != NULL) {
  360                         bus_dma_tag_t parent;
  361 
  362                         parent = dmat->parent;
  363                         atomic_subtract_int(&dmat->ref_count, 1);
  364                         if (dmat->ref_count == 0) {
  365                                 if (dmat->segments != NULL)
  366                                         free(dmat->segments, M_DEVBUF);
  367                                 free(dmat, M_DEVBUF);
  368                                 /*
  369                                  * Last reference count, so
  370                                  * release our reference
  371                                  * count on our parent.
  372                                  */
  373                                 dmat = parent;
  374                         } else
  375                                 dmat = NULL;
  376                 }
  377         }
  378 out:
  379         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
  380         return (error);
  381 }
  382 
  383 /*
  384  * Allocate a handle for mapping from kva/uva/physical
  385  * address space into bus device space.
  386  */
  387 int
  388 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  389 {
  390         int error;
  391 
  392         error = 0;
  393 
  394         if (dmat->segments == NULL) {
  395                 dmat->segments = (bus_dma_segment_t *)malloc(
  396                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
  397                     M_NOWAIT);
  398                 if (dmat->segments == NULL) {
  399                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  400                             __func__, dmat, ENOMEM);
  401                         return (ENOMEM);
  402                 }
  403         }
  404 
  405         /*
  406          * Bouncing might be required if the driver asks for an active
  407          * exclusion region, a data alignment that is stricter than 1, and/or
  408          * an active address boundary.
  409          */
  410         if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
  411 
  412                 /* Must bounce */
  413                 struct bounce_zone *bz;
  414                 int maxpages;
  415 
  416                 if (dmat->bounce_zone == NULL) {
  417                         if ((error = alloc_bounce_zone(dmat)) != 0)
  418                                 return (error);
  419                 }
  420                 bz = dmat->bounce_zone;
  421 
  422                 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
  423                                              M_NOWAIT | M_ZERO);
  424                 if (*mapp == NULL) {
  425                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  426                             __func__, dmat, ENOMEM);
  427                         return (ENOMEM);
  428                 }
  429 
  430                 /* Initialize the new map */
  431                 STAILQ_INIT(&((*mapp)->bpages));
  432 
  433                 /*
  434                  * Attempt to add pages to our pool on a per-instance
  435                  * basis up to a sane limit.
  436                  */
  437                 if (dmat->alignment > 1)
  438                         maxpages = MAX_BPAGES;
  439                 else
  440                         maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
  441                 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
  442                  || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
  443                         int pages;
  444 
  445                         pages = MAX(atop(dmat->maxsize), 1);
  446                         pages = MIN(maxpages - bz->total_bpages, pages);
  447                         pages = MAX(pages, 1);
  448                         if (alloc_bounce_pages(dmat, pages) < pages)
  449                                 error = ENOMEM;
  450 
  451                         if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
  452                                 if (error == 0)
  453                                         dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
  454                         } else {
  455                                 error = 0;
  456                         }
  457                 }
  458                 bz->map_count++;
  459         } else {
  460                 *mapp = NULL;
  461         }
  462         if (error == 0)
  463                 dmat->map_count++;
  464         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  465             __func__, dmat, dmat->flags, error);
  466         return (error);
  467 }
  468 
  469 /*
  470  * Destroy a handle for mapping from kva/uva/physical
  471  * address space into bus device space.
  472  */
  473 int
  474 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
  475 {
  476         if (map != NULL && map != &nobounce_dmamap) {
  477                 if (STAILQ_FIRST(&map->bpages) != NULL) {
  478                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
  479                             __func__, dmat, EBUSY);
  480                         return (EBUSY);
  481                 }
  482                 if (dmat->bounce_zone)
  483                         dmat->bounce_zone->map_count--;
  484                 free(map, M_DEVBUF);
  485         }
  486         dmat->map_count--;
  487         CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
  488         return (0);
  489 }
  490 
  491 
  492 /*
  493  * Allocate a piece of memory that can be efficiently mapped into
  494  * bus device space based on the constraints lited in the dma tag.
  495  * A dmamap to for use with dmamap_load is also allocated.
  496  */
  497 int
  498 bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
  499                  bus_dmamap_t *mapp)
  500 {
  501         int mflags;
  502 
  503         if (flags & BUS_DMA_NOWAIT)
  504                 mflags = M_NOWAIT;
  505         else
  506                 mflags = M_WAITOK;
  507 
  508         /* If we succeed, no mapping/bouncing will be required */
  509         *mapp = NULL;
  510 
  511         if (dmat->segments == NULL) {
  512                 dmat->segments = (bus_dma_segment_t *)malloc(
  513                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
  514                     mflags);
  515                 if (dmat->segments == NULL) {
  516                         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  517                             __func__, dmat, dmat->flags, ENOMEM);
  518                         return (ENOMEM);
  519                 }
  520         }
  521         if (flags & BUS_DMA_ZERO)
  522                 mflags |= M_ZERO;
  523 
  524         /* 
  525          * XXX:
  526          * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
  527          * alignment guarantees of malloc need to be nailed down, and the
  528          * code below should be rewritten to take that into account.
  529          *
  530          * In the meantime, we'll warn the user if malloc gets it wrong.
  531          */
  532         if ((dmat->maxsize <= PAGE_SIZE) &&
  533            (dmat->alignment < dmat->maxsize) &&
  534             dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) {
  535                 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
  536         } else {
  537                 /*
  538                  * XXX Use Contigmalloc until it is merged into this facility
  539                  *     and handles multi-seg allocations.  Nobody is doing
  540                  *     multi-seg allocations yet though.
  541                  * XXX Certain AGP hardware does.
  542                  */
  543                 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags,
  544                     0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul,
  545                     dmat->boundary);
  546         }
  547         if (*vaddr == NULL) {
  548                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  549                     __func__, dmat, dmat->flags, ENOMEM);
  550                 return (ENOMEM);
  551         } else if (vtophys(*vaddr) & (dmat->alignment - 1)) {
  552                 printf("bus_dmamem_alloc failed to align memory properly.\n");
  553         }
  554         if (flags & BUS_DMA_NOCACHE)
  555                 pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize,
  556                     PAT_UNCACHEABLE);
  557         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
  558             __func__, dmat, dmat->flags, 0);
  559         return (0);
  560 }
  561 
  562 /*
  563  * Free a piece of memory and it's allociated dmamap, that was allocated
  564  * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
  565  */
  566 void
  567 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
  568 {
  569         /*
  570          * dmamem does not need to be bounced, so the map should be
  571          * NULL
  572          */
  573         if (map != NULL)
  574                 panic("bus_dmamem_free: Invalid map freed\n");
  575         pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, PAT_WRITE_BACK);
  576         if ((dmat->maxsize <= PAGE_SIZE) &&
  577            (dmat->alignment < dmat->maxsize) &&
  578             dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem))
  579                 free(vaddr, M_DEVBUF);
  580         else {
  581                 contigfree(vaddr, dmat->maxsize, M_DEVBUF);
  582         }
  583         CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
  584 }
  585 
  586 int
  587 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
  588     void *buf, bus_size_t buflen, int flags)
  589 {
  590         vm_offset_t vaddr;
  591         vm_offset_t vendaddr;
  592         bus_addr_t paddr;
  593 
  594         if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
  595                 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
  596                     "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
  597                     dmat->boundary, dmat->alignment);
  598                 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
  599                     map, &nobounce_dmamap, map->pagesneeded);
  600                 /*
  601                  * Count the number of bounce pages
  602                  * needed in order to complete this transfer
  603                  */
  604                 vaddr = (vm_offset_t)buf;
  605                 vendaddr = (vm_offset_t)buf + buflen;
  606 
  607                 while (vaddr < vendaddr) {
  608                         bus_size_t sg_len;
  609 
  610                         sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
  611                         if (pmap)
  612                                 paddr = pmap_extract(pmap, vaddr);
  613                         else
  614                                 paddr = pmap_kextract(vaddr);
  615                         if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
  616                             run_filter(dmat, paddr) != 0) {
  617                                 sg_len = roundup2(sg_len, dmat->alignment);
  618                                 map->pagesneeded++;
  619                         }
  620                         vaddr += sg_len;
  621                 }
  622                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
  623         }
  624 
  625         /* Reserve Necessary Bounce Pages */
  626         if (map->pagesneeded != 0) {
  627                 mtx_lock(&bounce_lock);
  628                 if (flags & BUS_DMA_NOWAIT) {
  629                         if (reserve_bounce_pages(dmat, map, 0) != 0) {
  630                                 mtx_unlock(&bounce_lock);
  631                                 return (ENOMEM);
  632                         }
  633                 } else {
  634                         if (reserve_bounce_pages(dmat, map, 1) != 0) {
  635                                 /* Queue us for resources */
  636                                 map->dmat = dmat;
  637                                 map->buf = buf;
  638                                 map->buflen = buflen;
  639                                 STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
  640                                     map, links);
  641                                 mtx_unlock(&bounce_lock);
  642                                 return (EINPROGRESS);
  643                         }
  644                 }
  645                 mtx_unlock(&bounce_lock);
  646         }
  647 
  648         return (0);
  649 }
  650 
  651 /*
  652  * Utility function to load a linear buffer.  lastaddrp holds state
  653  * between invocations (for multiple-buffer loads).  segp contains
  654  * the starting segment on entrace, and the ending segment on exit.
  655  * first indicates if this is the first invocation of this function.
  656  */
  657 static __inline int
  658 _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
  659                         bus_dmamap_t map,
  660                         void *buf, bus_size_t buflen,
  661                         pmap_t pmap,
  662                         int flags,
  663                         bus_addr_t *lastaddrp,
  664                         bus_dma_segment_t *segs,
  665                         int *segp,
  666                         int first)
  667 {
  668         bus_size_t sgsize;
  669         bus_addr_t curaddr, lastaddr, baddr, bmask;
  670         vm_offset_t vaddr;
  671         int seg, error;
  672 
  673         if (map == NULL)
  674                 map = &nobounce_dmamap;
  675 
  676         if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
  677                 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
  678                 if (error)
  679                         return (error);
  680         }
  681 
  682         vaddr = (vm_offset_t)buf;
  683         lastaddr = *lastaddrp;
  684         bmask = ~(dmat->boundary - 1);
  685 
  686         for (seg = *segp; buflen > 0 ; ) {
  687                 bus_size_t max_sgsize;
  688 
  689                 /*
  690                  * Get the physical address for this segment.
  691                  */
  692                 if (pmap)
  693                         curaddr = pmap_extract(pmap, vaddr);
  694                 else
  695                         curaddr = pmap_kextract(vaddr);
  696 
  697                 /*
  698                  * Compute the segment size, and adjust counts.
  699                  */
  700                 max_sgsize = MIN(buflen, dmat->maxsegsz);
  701                 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
  702                 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
  703                     map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
  704                         sgsize = roundup2(sgsize, dmat->alignment);
  705                         sgsize = MIN(sgsize, max_sgsize);
  706                         curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
  707                 } else {
  708                         sgsize = MIN(sgsize, max_sgsize);
  709                 }
  710 
  711                 /*
  712                  * Make sure we don't cross any boundaries.
  713                  */
  714                 if (dmat->boundary > 0) {
  715                         baddr = (curaddr + dmat->boundary) & bmask;
  716                         if (sgsize > (baddr - curaddr))
  717                                 sgsize = (baddr - curaddr);
  718                 }
  719 
  720                 /*
  721                  * Insert chunk into a segment, coalescing with
  722                  * previous segment if possible.
  723                  */
  724                 if (first) {
  725                         segs[seg].ds_addr = curaddr;
  726                         segs[seg].ds_len = sgsize;
  727                         first = 0;
  728                 } else {
  729                         if (curaddr == lastaddr &&
  730                             (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
  731                             (dmat->boundary == 0 ||
  732                              (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
  733                                 segs[seg].ds_len += sgsize;
  734                         else {
  735                                 if (++seg >= dmat->nsegments)
  736                                         break;
  737                                 segs[seg].ds_addr = curaddr;
  738                                 segs[seg].ds_len = sgsize;
  739                         }
  740                 }
  741 
  742                 lastaddr = curaddr + sgsize;
  743                 vaddr += sgsize;
  744                 buflen -= sgsize;
  745         }
  746 
  747         *segp = seg;
  748         *lastaddrp = lastaddr;
  749 
  750         /*
  751          * Did we fit?
  752          */
  753         return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
  754 }
  755 
  756 /*
  757  * Map the buffer buf into bus space using the dmamap map.
  758  */
  759 int
  760 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
  761                 bus_size_t buflen, bus_dmamap_callback_t *callback,
  762                 void *callback_arg, int flags)
  763 {
  764         bus_addr_t              lastaddr = 0;
  765         int                     error, nsegs = 0;
  766 
  767         if (map != NULL) {
  768                 flags |= BUS_DMA_WAITOK;
  769                 map->callback = callback;
  770                 map->callback_arg = callback_arg;
  771         }
  772 
  773         error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags,
  774              &lastaddr, dmat->segments, &nsegs, 1);
  775 
  776         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  777             __func__, dmat, dmat->flags, error, nsegs + 1);
  778 
  779         if (error == EINPROGRESS) {
  780                 return (error);
  781         }
  782 
  783         if (error)
  784                 (*callback)(callback_arg, dmat->segments, 0, error);
  785         else
  786                 (*callback)(callback_arg, dmat->segments, nsegs + 1, 0);
  787 
  788         /*
  789          * Return ENOMEM to the caller so that it can pass it up the stack.
  790          * This error only happens when NOWAIT is set, so deferal is disabled.
  791          */
  792         if (error == ENOMEM)
  793                 return (error);
  794 
  795         return (0);
  796 }
  797 
  798 
  799 /*
  800  * Like _bus_dmamap_load(), but for mbufs.
  801  */
  802 static __inline int
  803 _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
  804                         struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
  805                         int flags)
  806 {
  807         int error;
  808 
  809         M_ASSERTPKTHDR(m0);
  810 
  811         flags |= BUS_DMA_NOWAIT;
  812         *nsegs = 0;
  813         error = 0;
  814         if (m0->m_pkthdr.len <= dmat->maxsize) {
  815                 int first = 1;
  816                 bus_addr_t lastaddr = 0;
  817                 struct mbuf *m;
  818 
  819                 for (m = m0; m != NULL && error == 0; m = m->m_next) {
  820                         if (m->m_len > 0) {
  821                                 error = _bus_dmamap_load_buffer(dmat, map,
  822                                                 m->m_data, m->m_len,
  823                                                 NULL, flags, &lastaddr,
  824                                                 segs, nsegs, first);
  825                                 first = 0;
  826                         }
  827                 }
  828         } else {
  829                 error = EINVAL;
  830         }
  831 
  832         /* XXX FIXME: Having to increment nsegs is really annoying */
  833         ++*nsegs;
  834         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  835             __func__, dmat, dmat->flags, error, *nsegs);
  836         return (error);
  837 }
  838 
  839 int
  840 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
  841                      struct mbuf *m0,
  842                      bus_dmamap_callback2_t *callback, void *callback_arg,
  843                      int flags)
  844 {
  845         int nsegs, error;
  846 
  847         error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs,
  848             flags);
  849 
  850         if (error) {
  851                 /* force "no valid mappings" in callback */
  852                 (*callback)(callback_arg, dmat->segments, 0, 0, error);
  853         } else {
  854                 (*callback)(callback_arg, dmat->segments,
  855                             nsegs, m0->m_pkthdr.len, error);
  856         }
  857         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  858             __func__, dmat, dmat->flags, error, nsegs);
  859         return (error);
  860 }
  861 
  862 int
  863 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
  864                         struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
  865                         int flags)
  866 {
  867         return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags));
  868 }
  869 
  870 /*
  871  * Like _bus_dmamap_load(), but for uios.
  872  */
  873 int
  874 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
  875                     struct uio *uio,
  876                     bus_dmamap_callback2_t *callback, void *callback_arg,
  877                     int flags)
  878 {
  879         bus_addr_t lastaddr = 0;
  880         int nsegs, error, first, i;
  881         bus_size_t resid;
  882         struct iovec *iov;
  883         pmap_t pmap;
  884 
  885         flags |= BUS_DMA_NOWAIT;
  886         resid = uio->uio_resid;
  887         iov = uio->uio_iov;
  888 
  889         if (uio->uio_segflg == UIO_USERSPACE) {
  890                 KASSERT(uio->uio_td != NULL,
  891                         ("bus_dmamap_load_uio: USERSPACE but no proc"));
  892                 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
  893         } else
  894                 pmap = NULL;
  895 
  896         nsegs = 0;
  897         error = 0;
  898         first = 1;
  899         for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
  900                 /*
  901                  * Now at the first iovec to load.  Load each iovec
  902                  * until we have exhausted the residual count.
  903                  */
  904                 bus_size_t minlen =
  905                         resid < iov[i].iov_len ? resid : iov[i].iov_len;
  906                 caddr_t addr = (caddr_t) iov[i].iov_base;
  907 
  908                 if (minlen > 0) {
  909                         error = _bus_dmamap_load_buffer(dmat, map,
  910                                         addr, minlen, pmap, flags, &lastaddr,
  911                                         dmat->segments, &nsegs, first);
  912                         first = 0;
  913 
  914                         resid -= minlen;
  915                 }
  916         }
  917 
  918         if (error) {
  919                 /* force "no valid mappings" in callback */
  920                 (*callback)(callback_arg, dmat->segments, 0, 0, error);
  921         } else {
  922                 (*callback)(callback_arg, dmat->segments,
  923                             nsegs+1, uio->uio_resid, error);
  924         }
  925         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  926             __func__, dmat, dmat->flags, error, nsegs + 1);
  927         return (error);
  928 }
  929 
  930 /*
  931  * Release the mapping held by map.
  932  */
  933 void
  934 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
  935 {
  936         struct bounce_page *bpage;
  937 
  938         while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
  939                 STAILQ_REMOVE_HEAD(&map->bpages, links);
  940                 free_bounce_page(dmat, bpage);
  941         }
  942 }
  943 
  944 void
  945 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
  946 {
  947         struct bounce_page *bpage;
  948 
  949         if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
  950                 /*
  951                  * Handle data bouncing.  We might also
  952                  * want to add support for invalidating
  953                  * the caches on broken hardware
  954                  */
  955                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
  956                     "performing bounce", __func__, op, dmat, dmat->flags);
  957 
  958                 if (op & BUS_DMASYNC_PREWRITE) {
  959                         while (bpage != NULL) {
  960                                 bcopy((void *)bpage->datavaddr,
  961                                       (void *)bpage->vaddr,
  962                                       bpage->datacount);
  963                                 bpage = STAILQ_NEXT(bpage, links);
  964                         }
  965                         dmat->bounce_zone->total_bounced++;
  966                 }
  967 
  968                 if (op & BUS_DMASYNC_POSTREAD) {
  969                         while (bpage != NULL) {
  970                                 bcopy((void *)bpage->vaddr,
  971                                       (void *)bpage->datavaddr,
  972                                       bpage->datacount);
  973                                 bpage = STAILQ_NEXT(bpage, links);
  974                         }
  975                         dmat->bounce_zone->total_bounced++;
  976                 }
  977         }
  978 }
  979 
  980 static void
  981 init_bounce_pages(void *dummy __unused)
  982 {
  983 
  984         total_bpages = 0;
  985         STAILQ_INIT(&bounce_zone_list);
  986         STAILQ_INIT(&bounce_map_waitinglist);
  987         STAILQ_INIT(&bounce_map_callbacklist);
  988         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
  989 }
  990 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
  991 
  992 static struct sysctl_ctx_list *
  993 busdma_sysctl_tree(struct bounce_zone *bz)
  994 {
  995         return (&bz->sysctl_tree);
  996 }
  997 
  998 static struct sysctl_oid *
  999 busdma_sysctl_tree_top(struct bounce_zone *bz)
 1000 {
 1001         return (bz->sysctl_tree_top);
 1002 }
 1003 
 1004 #if defined(__amd64__) || defined(PAE)
 1005 #define SYSCTL_ADD_BUS_SIZE_T   SYSCTL_ADD_UQUAD
 1006 #else
 1007 #define SYSCTL_ADD_BUS_SIZE_T(ctx, parent, nbr, name, flag, ptr, desc)  \
 1008         SYSCTL_ADD_UINT(ctx, parent, nbr, name, flag, ptr, 0, desc)
 1009 #endif
 1010 
 1011 static int
 1012 alloc_bounce_zone(bus_dma_tag_t dmat)
 1013 {
 1014         struct bounce_zone *bz;
 1015 
 1016         /* Check to see if we already have a suitable zone */
 1017         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
 1018                 if ((dmat->alignment <= bz->alignment)
 1019                  && (dmat->lowaddr >= bz->lowaddr)) {
 1020                         dmat->bounce_zone = bz;
 1021                         return (0);
 1022                 }
 1023         }
 1024 
 1025         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
 1026             M_NOWAIT | M_ZERO)) == NULL)
 1027                 return (ENOMEM);
 1028 
 1029         STAILQ_INIT(&bz->bounce_page_list);
 1030         bz->free_bpages = 0;
 1031         bz->reserved_bpages = 0;
 1032         bz->active_bpages = 0;
 1033         bz->lowaddr = dmat->lowaddr;
 1034         bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
 1035         bz->map_count = 0;
 1036         snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
 1037         busdma_zonecount++;
 1038         snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
 1039         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
 1040         dmat->bounce_zone = bz;
 1041 
 1042         sysctl_ctx_init(&bz->sysctl_tree);
 1043         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
 1044             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
 1045             CTLFLAG_RD, 0, "");
 1046         if (bz->sysctl_tree_top == NULL) {
 1047                 sysctl_ctx_free(&bz->sysctl_tree);
 1048                 return (0);     /* XXX error code? */
 1049         }
 1050 
 1051         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1052             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1053             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
 1054             "Total bounce pages");
 1055         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1056             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1057             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
 1058             "Free bounce pages");
 1059         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1060             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1061             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
 1062             "Reserved bounce pages");
 1063         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1064             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1065             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
 1066             "Active bounce pages");
 1067         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1068             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1069             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
 1070             "Total bounce requests");
 1071         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 1072             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1073             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
 1074             "Total bounce requests that were deferred");
 1075         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
 1076             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1077             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
 1078         SYSCTL_ADD_BUS_SIZE_T(busdma_sysctl_tree(bz),
 1079             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 1080             "alignment", CTLFLAG_RD, &bz->alignment, "");
 1081 
 1082         return (0);
 1083 }
 1084 
 1085 static int
 1086 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
 1087 {
 1088         struct bounce_zone *bz;
 1089         int count;
 1090 
 1091         bz = dmat->bounce_zone;
 1092         count = 0;
 1093         while (numpages > 0) {
 1094                 struct bounce_page *bpage;
 1095 
 1096                 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
 1097                                                      M_NOWAIT | M_ZERO);
 1098 
 1099                 if (bpage == NULL)
 1100                         break;
 1101                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
 1102                                                          M_NOWAIT, 0ul,
 1103                                                          bz->lowaddr,
 1104                                                          PAGE_SIZE,
 1105                                                          0);
 1106                 if (bpage->vaddr == 0) {
 1107                         free(bpage, M_DEVBUF);
 1108                         break;
 1109                 }
 1110                 bpage->busaddr = pmap_kextract(bpage->vaddr);
 1111                 mtx_lock(&bounce_lock);
 1112                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
 1113                 total_bpages++;
 1114                 bz->total_bpages++;
 1115                 bz->free_bpages++;
 1116                 mtx_unlock(&bounce_lock);
 1117                 count++;
 1118                 numpages--;
 1119         }
 1120         return (count);
 1121 }
 1122 
 1123 static int
 1124 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
 1125 {
 1126         struct bounce_zone *bz;
 1127         int pages;
 1128 
 1129         mtx_assert(&bounce_lock, MA_OWNED);
 1130         bz = dmat->bounce_zone;
 1131         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
 1132         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
 1133                 return (map->pagesneeded - (map->pagesreserved + pages));
 1134         bz->free_bpages -= pages;
 1135         bz->reserved_bpages += pages;
 1136         map->pagesreserved += pages;
 1137         pages = map->pagesneeded - map->pagesreserved;
 1138 
 1139         return (pages);
 1140 }
 1141 
 1142 static bus_addr_t
 1143 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 1144                 bus_size_t size)
 1145 {
 1146         struct bounce_zone *bz;
 1147         struct bounce_page *bpage;
 1148 
 1149         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
 1150         KASSERT(map != NULL && map != &nobounce_dmamap,
 1151             ("add_bounce_page: bad map %p", map));
 1152 
 1153         bz = dmat->bounce_zone;
 1154         if (map->pagesneeded == 0)
 1155                 panic("add_bounce_page: map doesn't need any pages");
 1156         map->pagesneeded--;
 1157 
 1158         if (map->pagesreserved == 0)
 1159                 panic("add_bounce_page: map doesn't need any pages");
 1160         map->pagesreserved--;
 1161 
 1162         mtx_lock(&bounce_lock);
 1163         bpage = STAILQ_FIRST(&bz->bounce_page_list);
 1164         if (bpage == NULL)
 1165                 panic("add_bounce_page: free page list is empty");
 1166 
 1167         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
 1168         bz->reserved_bpages--;
 1169         bz->active_bpages++;
 1170         mtx_unlock(&bounce_lock);
 1171 
 1172         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1173                 /* Page offset needs to be preserved. */
 1174                 bpage->vaddr |= vaddr & PAGE_MASK;
 1175                 bpage->busaddr |= vaddr & PAGE_MASK;
 1176         }
 1177         bpage->datavaddr = vaddr;
 1178         bpage->datacount = size;
 1179         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
 1180         return (bpage->busaddr);
 1181 }
 1182 
 1183 static void
 1184 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
 1185 {
 1186         struct bus_dmamap *map;
 1187         struct bounce_zone *bz;
 1188 
 1189         bz = dmat->bounce_zone;
 1190         bpage->datavaddr = 0;
 1191         bpage->datacount = 0;
 1192         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
 1193                 /*
 1194                  * Reset the bounce page to start at offset 0.  Other uses
 1195                  * of this bounce page may need to store a full page of
 1196                  * data and/or assume it starts on a page boundary.
 1197                  */
 1198                 bpage->vaddr &= ~PAGE_MASK;
 1199                 bpage->busaddr &= ~PAGE_MASK;
 1200         }
 1201 
 1202         mtx_lock(&bounce_lock);
 1203         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
 1204         bz->free_bpages++;
 1205         bz->active_bpages--;
 1206         if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
 1207                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
 1208                         STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
 1209                         STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
 1210                                            map, links);
 1211                         busdma_swi_pending = 1;
 1212                         bz->total_deferred++;
 1213                         swi_sched(vm_ih, 0);
 1214                 }
 1215         }
 1216         mtx_unlock(&bounce_lock);
 1217 }
 1218 
 1219 void
 1220 busdma_swi(void)
 1221 {
 1222         bus_dma_tag_t dmat;
 1223         struct bus_dmamap *map;
 1224 
 1225         mtx_lock(&bounce_lock);
 1226         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
 1227                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
 1228                 mtx_unlock(&bounce_lock);
 1229                 dmat = map->dmat;
 1230                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
 1231                 bus_dmamap_load(map->dmat, map, map->buf, map->buflen,
 1232                                 map->callback, map->callback_arg, /*flags*/0);
 1233                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
 1234                 mtx_lock(&bounce_lock);
 1235         }
 1236         mtx_unlock(&bounce_lock);
 1237 }

Cache object: 7fdae8a7cddb77f1524cc92150e0beaa


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.