usb_busdma.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /* $FreeBSD$ */
    2 /*-
    3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    4  *
    5  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  */
   28 
   29 #ifdef USB_GLOBAL_INCLUDE_FILE
   30 #include USB_GLOBAL_INCLUDE_FILE
   31 #else
   32 #include <sys/stdint.h>
   33 #include <sys/stddef.h>
   34 #include <sys/param.h>
   35 #include <sys/queue.h>
   36 #include <sys/types.h>
   37 #include <sys/systm.h>
   38 #include <sys/kernel.h>
   39 #include <sys/bus.h>
   40 #include <sys/module.h>
   41 #include <sys/lock.h>
   42 #include <sys/mutex.h>
   43 #include <sys/condvar.h>
   44 #include <sys/sysctl.h>
   45 #include <sys/sx.h>
   46 #include <sys/unistd.h>
   47 #include <sys/callout.h>
   48 #include <sys/malloc.h>
   49 #include <sys/priv.h>
   50 
   51 #include <dev/usb/usb.h>
   52 #include <dev/usb/usbdi.h>
   53 #include <dev/usb/usbdi_util.h>
   54 
   55 #define USB_DEBUG_VAR usb_debug
   56 
   57 #include <dev/usb/usb_core.h>
   58 #include <dev/usb/usb_busdma.h>
   59 #include <dev/usb/usb_process.h>
   60 #include <dev/usb/usb_transfer.h>
   61 #include <dev/usb/usb_device.h>
   62 #include <dev/usb/usb_util.h>
   63 #include <dev/usb/usb_debug.h>
   64 
   65 #include <dev/usb/usb_controller.h>
   66 #include <dev/usb/usb_bus.h>
   67 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
   68 
   69 #if USB_HAVE_BUSDMA
   70 static void     usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
   71 static void     usb_dma_tag_destroy(struct usb_dma_tag *);
   72 static void     usb_dma_lock_cb(void *, bus_dma_lock_op_t);
   73 static void     usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
   74 static void     usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
   75 static void     usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
   76                     uint8_t);
   77 #endif
   78 
   79 /*------------------------------------------------------------------------*
   80  *  usbd_get_page - lookup DMA-able memory for the given offset
   81  *
   82  * NOTE: Only call this function when the "page_cache" structure has
   83  * been properly initialized !
   84  *------------------------------------------------------------------------*/
   85 void
   86 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
   87     struct usb_page_search *res)
   88 {
   89 #if USB_HAVE_BUSDMA
   90         struct usb_page *page;
   91 
   92         if (pc->page_start) {
   93                 /* Case 1 - something has been loaded into DMA */
   94 
   95                 if (pc->buffer) {
   96                         /* Case 1a - Kernel Virtual Address */
   97 
   98                         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
   99                 }
  100                 offset += pc->page_offset_buf;
  101 
  102                 /* compute destination page */
  103 
  104                 page = pc->page_start;
  105 
  106                 if (pc->ismultiseg) {
  107                         page += (offset / USB_PAGE_SIZE);
  108 
  109                         offset %= USB_PAGE_SIZE;
  110 
  111                         res->length = USB_PAGE_SIZE - offset;
  112                         res->physaddr = page->physaddr + offset;
  113                 } else {
  114                         res->length = (usb_size_t)-1;
  115                         res->physaddr = page->physaddr + offset;
  116                 }
  117                 if (!pc->buffer) {
  118                         /* Case 1b - Non Kernel Virtual Address */
  119 
  120                         res->buffer = USB_ADD_BYTES(page->buffer, offset);
  121                 }
  122                 return;
  123         }
  124 #endif
  125         /* Case 2 - Plain PIO */
  126 
  127         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
  128         res->length = (usb_size_t)-1;
  129 #if USB_HAVE_BUSDMA
  130         res->physaddr = 0;
  131 #endif
  132 }
  133 
  134 /*------------------------------------------------------------------------*
  135  *  usb_pc_buffer_is_aligned - verify alignment
  136  * 
  137  * This function is used to check if a page cache buffer is properly
  138  * aligned to reduce the use of bounce buffers in PIO mode.
  139  *------------------------------------------------------------------------*/
  140 uint8_t
  141 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset,
  142     usb_frlength_t len, usb_frlength_t mask)
  143 {
  144         struct usb_page_search buf_res;
  145 
  146         while (len != 0) {
  147                 usbd_get_page(pc, offset, &buf_res);
  148 
  149                 if (buf_res.length > len)
  150                         buf_res.length = len;
  151                 if (USB_P2U(buf_res.buffer) & mask)
  152                         return (0);
  153                 if (buf_res.length & mask)
  154                         return (0);
  155 
  156                 offset += buf_res.length;
  157                 len -= buf_res.length;
  158         }
  159         return (1);
  160 }
  161 
  162 /*------------------------------------------------------------------------*
  163  *  usbd_copy_in - copy directly to DMA-able memory
  164  *------------------------------------------------------------------------*/
  165 void
  166 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
  167     const void *ptr, usb_frlength_t len)
  168 {
  169         struct usb_page_search buf_res;
  170 
  171         while (len != 0) {
  172                 usbd_get_page(cache, offset, &buf_res);
  173 
  174                 if (buf_res.length > len) {
  175                         buf_res.length = len;
  176                 }
  177                 memcpy(buf_res.buffer, ptr, buf_res.length);
  178 
  179                 offset += buf_res.length;
  180                 len -= buf_res.length;
  181                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
  182         }
  183 }
  184 
  185 /*------------------------------------------------------------------------*
  186  *  usbd_copy_in_user - copy directly to DMA-able memory from userland
  187  *
  188  * Return values:
  189  *    0: Success
  190  * Else: Failure
  191  *------------------------------------------------------------------------*/
  192 #if USB_HAVE_USER_IO
  193 int
  194 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
  195     const void *ptr, usb_frlength_t len)
  196 {
  197         struct usb_page_search buf_res;
  198         int error;
  199 
  200         while (len != 0) {
  201                 usbd_get_page(cache, offset, &buf_res);
  202 
  203                 if (buf_res.length > len) {
  204                         buf_res.length = len;
  205                 }
  206                 error = copyin(ptr, buf_res.buffer, buf_res.length);
  207                 if (error)
  208                         return (error);
  209 
  210                 offset += buf_res.length;
  211                 len -= buf_res.length;
  212                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
  213         }
  214         return (0);                     /* success */
  215 }
  216 #endif
  217 
  218 /*------------------------------------------------------------------------*
  219  *  usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
  220  *------------------------------------------------------------------------*/
  221 #if USB_HAVE_MBUF
  222 struct usb_m_copy_in_arg {
  223         struct usb_page_cache *cache;
  224         usb_frlength_t dst_offset;
  225 };
  226 
  227 static int
  228 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
  229 {
  230         struct usb_m_copy_in_arg *ua = arg;
  231 
  232         usbd_copy_in(ua->cache, ua->dst_offset, src, count);
  233         ua->dst_offset += count;
  234         return (0);
  235 }
  236 
  237 void
  238 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
  239     struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
  240 {
  241         struct usb_m_copy_in_arg arg = {cache, dst_offset};
  242         (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
  243 }
  244 #endif
  245 
  246 /*------------------------------------------------------------------------*
  247  *  usb_uiomove - factored out code
  248  *------------------------------------------------------------------------*/
  249 #if USB_HAVE_USER_IO
  250 int
  251 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
  252     usb_frlength_t pc_offset, usb_frlength_t len)
  253 {
  254         struct usb_page_search res;
  255         int error = 0;
  256 
  257         while (len != 0) {
  258                 usbd_get_page(pc, pc_offset, &res);
  259 
  260                 if (res.length > len) {
  261                         res.length = len;
  262                 }
  263                 /*
  264                  * "uiomove()" can sleep so one needs to make a wrapper,
  265                  * exiting the mutex and checking things
  266                  */
  267                 error = uiomove(res.buffer, res.length, uio);
  268 
  269                 if (error) {
  270                         break;
  271                 }
  272                 pc_offset += res.length;
  273                 len -= res.length;
  274         }
  275         return (error);
  276 }
  277 #endif
  278 
  279 /*------------------------------------------------------------------------*
  280  *  usbd_copy_out - copy directly from DMA-able memory
  281  *------------------------------------------------------------------------*/
  282 void
  283 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
  284     void *ptr, usb_frlength_t len)
  285 {
  286         struct usb_page_search res;
  287 
  288         while (len != 0) {
  289                 usbd_get_page(cache, offset, &res);
  290 
  291                 if (res.length > len) {
  292                         res.length = len;
  293                 }
  294                 memcpy(ptr, res.buffer, res.length);
  295 
  296                 offset += res.length;
  297                 len -= res.length;
  298                 ptr = USB_ADD_BYTES(ptr, res.length);
  299         }
  300 }
  301 
  302 /*------------------------------------------------------------------------*
  303  *  usbd_copy_out_user - copy directly from DMA-able memory to userland
  304  *
  305  * Return values:
  306  *    0: Success
  307  * Else: Failure
  308  *------------------------------------------------------------------------*/
  309 #if USB_HAVE_USER_IO
  310 int
  311 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
  312     void *ptr, usb_frlength_t len)
  313 {
  314         struct usb_page_search res;
  315         int error;
  316 
  317         while (len != 0) {
  318                 usbd_get_page(cache, offset, &res);
  319 
  320                 if (res.length > len) {
  321                         res.length = len;
  322                 }
  323                 error = copyout(res.buffer, ptr, res.length);
  324                 if (error)
  325                         return (error);
  326 
  327                 offset += res.length;
  328                 len -= res.length;
  329                 ptr = USB_ADD_BYTES(ptr, res.length);
  330         }
  331         return (0);                     /* success */
  332 }
  333 #endif
  334 
  335 /*------------------------------------------------------------------------*
  336  *  usbd_frame_zero - zero DMA-able memory
  337  *------------------------------------------------------------------------*/
  338 void
  339 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
  340     usb_frlength_t len)
  341 {
  342         struct usb_page_search res;
  343 
  344         while (len != 0) {
  345                 usbd_get_page(cache, offset, &res);
  346 
  347                 if (res.length > len) {
  348                         res.length = len;
  349                 }
  350                 memset(res.buffer, 0, res.length);
  351 
  352                 offset += res.length;
  353                 len -= res.length;
  354         }
  355 }
  356 
  357 #if USB_HAVE_BUSDMA
  358 
  359 /*------------------------------------------------------------------------*
  360  *      usb_dma_lock_cb - dummy callback
  361  *------------------------------------------------------------------------*/
  362 static void
  363 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
  364 {
  365         /* we use "mtx_owned()" instead of this function */
  366 }
  367 
  368 /*------------------------------------------------------------------------*
  369  *      usb_dma_tag_create - allocate a DMA tag
  370  *
  371  * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
  372  * allow multi-segment mappings. Else all mappings are single-segment.
  373  *------------------------------------------------------------------------*/
  374 static void
  375 usb_dma_tag_create(struct usb_dma_tag *udt,
  376     usb_size_t size, usb_size_t align)
  377 {
  378         bus_dma_tag_t tag;
  379 
  380         if (bus_dma_tag_create
  381             ( /* parent    */ udt->tag_parent->tag,
  382              /* alignment */ align,
  383              /* boundary  */ 0,
  384              /* lowaddr   */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
  385              /* highaddr  */ BUS_SPACE_MAXADDR,
  386              /* filter    */ NULL,
  387              /* filterarg */ NULL,
  388              /* maxsize   */ size,
  389              /* nsegments */ (align == 1 && size > 1) ?
  390             (2 + (size / USB_PAGE_SIZE)) : 1,
  391              /* maxsegsz  */ (align == 1 && size > USB_PAGE_SIZE) ?
  392             USB_PAGE_SIZE : size,
  393              /* flags     */ BUS_DMA_KEEP_PG_OFFSET,
  394              /* lockfn    */ &usb_dma_lock_cb,
  395              /* lockarg   */ NULL,
  396             &tag)) {
  397                 tag = NULL;
  398         }
  399         udt->tag = tag;
  400 }
  401 
  402 /*------------------------------------------------------------------------*
  403  *      usb_dma_tag_free - free a DMA tag
  404  *------------------------------------------------------------------------*/
  405 static void
  406 usb_dma_tag_destroy(struct usb_dma_tag *udt)
  407 {
  408         bus_dma_tag_destroy(udt->tag);
  409 }
  410 
  411 /*------------------------------------------------------------------------*
  412  *      usb_pc_alloc_mem_cb - BUS-DMA callback function
  413  *------------------------------------------------------------------------*/
  414 static void
  415 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
  416     int nseg, int error)
  417 {
  418         usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
  419 }
  420 
  421 /*------------------------------------------------------------------------*
  422  *      usb_pc_load_mem_cb - BUS-DMA callback function
  423  *------------------------------------------------------------------------*/
  424 static void
  425 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
  426     int nseg, int error)
  427 {
  428         usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
  429 }
  430 
  431 /*------------------------------------------------------------------------*
  432  *      usb_pc_common_mem_cb - BUS-DMA callback function
  433  *------------------------------------------------------------------------*/
  434 static void
  435 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
  436     int nseg, int error, uint8_t isload)
  437 {
  438         struct usb_dma_parent_tag *uptag;
  439         struct usb_page_cache *pc;
  440         struct usb_page *pg;
  441         usb_size_t rem;
  442         bus_size_t off;
  443         uint8_t owned;
  444 
  445         pc = arg;
  446         uptag = pc->tag_parent;
  447 
  448         /*
  449          * XXX There is sometimes recursive locking here.
  450          * XXX We should try to find a better solution.
  451          * XXX Until further the "owned" variable does
  452          * XXX the trick.
  453          */
  454 
  455         if (error) {
  456                 goto done;
  457         }
  458 
  459         off = 0;
  460         pg = pc->page_start;
  461         pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE);
  462         rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
  463         pc->page_offset_buf = rem;
  464         pc->page_offset_end += rem;
  465 #ifdef USB_DEBUG
  466         if (nseg > 1) {
  467                 int x;
  468 
  469                 for (x = 0; x != nseg - 1; x++) {
  470                         if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) ==
  471                             ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1))))
  472                                 continue;
  473                         /*
  474                          * This check verifies there is no page offset
  475                          * hole between any of the segments. See the
  476                          * BUS_DMA_KEEP_PG_OFFSET flag.
  477                          */
  478                         DPRINTFN(0, "Page offset was not preserved\n");
  479                         error = 1;
  480                         goto done;
  481                 }
  482         }
  483 #endif
  484         while (pc->ismultiseg) {
  485                 off += USB_PAGE_SIZE;
  486                 if (off >= (segs->ds_len + rem)) {
  487                         /* page crossing */
  488                         nseg--;
  489                         segs++;
  490                         off = 0;
  491                         rem = 0;
  492                         if (nseg == 0)
  493                                 break;
  494                 }
  495                 pg++;
  496                 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE);
  497         }
  498 
  499 done:
  500         owned = mtx_owned(uptag->mtx);
  501         if (!owned)
  502                 USB_MTX_LOCK(uptag->mtx);
  503 
  504         uptag->dma_error = (error ? 1 : 0);
  505         if (isload) {
  506                 (uptag->func) (uptag);
  507         } else {
  508                 cv_broadcast(uptag->cv);
  509         }
  510         if (!owned)
  511                 USB_MTX_UNLOCK(uptag->mtx);
  512 }
  513 
  514 /*------------------------------------------------------------------------*
  515  *      usb_pc_alloc_mem - allocate DMA'able memory
  516  *
  517  * Returns:
  518  *    0: Success
  519  * Else: Failure
  520  *------------------------------------------------------------------------*/
  521 uint8_t
  522 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
  523     usb_size_t size, usb_size_t align)
  524 {
  525         struct usb_dma_parent_tag *uptag;
  526         struct usb_dma_tag *utag;
  527         bus_dmamap_t map;
  528         void *ptr;
  529         int err;
  530 
  531         uptag = pc->tag_parent;
  532 
  533         if (align != 1) {
  534                 /*
  535                  * The alignment must be greater or equal to the
  536                  * "size" else the object can be split between two
  537                  * memory pages and we get a problem!
  538                  */
  539                 while (align < size) {
  540                         align *= 2;
  541                         if (align == 0) {
  542                                 goto error;
  543                         }
  544                 }
  545 #if 1
  546                 /*
  547                  * XXX BUS-DMA workaround - FIXME later:
  548                  *
  549                  * We assume that the alignment at this point of
  550                  * the code is greater than or equal to the size and
  551                  * less than two times the size, so that if we double
  552                  * the size, the size will be greater than the
  553                  * alignment.
  554                  *
  555                  * The bus-dma system has a check for "alignment"
  556                  * being less than "size". If that check fails we end
  557                  * up using contigmalloc which is page based even for
  558                  * small allocations. Try to avoid that to save
  559                  * memory, hence we sometimes to a large number of
  560                  * small allocations!
  561                  */
  562                 if (size <= (USB_PAGE_SIZE / 2)) {
  563                         size *= 2;
  564                 }
  565 #endif
  566         }
  567         /* get the correct DMA tag */
  568         utag = usb_dma_tag_find(uptag, size, align);
  569         if (utag == NULL) {
  570                 goto error;
  571         }
  572         /* allocate memory */
  573         if (bus_dmamem_alloc(
  574             utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
  575                 goto error;
  576         }
  577         /* setup page cache */
  578         pc->buffer = ptr;
  579         pc->page_start = pg;
  580         pc->page_offset_buf = 0;
  581         pc->page_offset_end = size;
  582         pc->map = map;
  583         pc->tag = utag->tag;
  584         pc->ismultiseg = (align == 1);
  585 
  586         USB_MTX_LOCK(uptag->mtx);
  587 
  588         /* load memory into DMA */
  589         err = bus_dmamap_load(
  590             utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
  591             pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
  592 
  593         if (err == EINPROGRESS) {
  594                 cv_wait(uptag->cv, uptag->mtx);
  595                 err = 0;
  596         }
  597         USB_MTX_UNLOCK(uptag->mtx);
  598 
  599         if (err || uptag->dma_error) {
  600                 bus_dmamem_free(utag->tag, ptr, map);
  601                 goto error;
  602         }
  603         pc->isloaded = 1;
  604         memset(ptr, 0, size);
  605 
  606         usb_pc_cpu_flush(pc);
  607 
  608         return (0);
  609 
  610 error:
  611         /* reset most of the page cache */
  612         pc->buffer = NULL;
  613         pc->page_start = NULL;
  614         pc->page_offset_buf = 0;
  615         pc->page_offset_end = 0;
  616         pc->isloaded = 0;
  617         pc->map = NULL;
  618         pc->tag = NULL;
  619         return (1);
  620 }
  621 
  622 /*------------------------------------------------------------------------*
  623  *      usb_pc_free_mem - free DMA memory
  624  *
  625  * This function is NULL safe.
  626  *------------------------------------------------------------------------*/
  627 void
  628 usb_pc_free_mem(struct usb_page_cache *pc)
  629 {
  630         if (pc && pc->buffer) {
  631                 if (pc->isloaded)
  632                         bus_dmamap_unload(pc->tag, pc->map);
  633 
  634                 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
  635 
  636                 pc->buffer = NULL;
  637                 pc->isloaded = 0;
  638         }
  639 }
  640 
  641 /*------------------------------------------------------------------------*
  642  *      usb_pc_load_mem - load virtual memory into DMA
  643  *
  644  * Return values:
  645  * 0: Success
  646  * Else: Error
  647  *------------------------------------------------------------------------*/
  648 uint8_t
  649 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
  650 {
  651         /* setup page cache */
  652         pc->page_offset_buf = 0;
  653         pc->page_offset_end = size;
  654         pc->ismultiseg = 1;
  655 
  656         USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED);
  657 
  658         if (size > 0) {
  659                 if (sync) {
  660                         struct usb_dma_parent_tag *uptag;
  661                         int err;
  662 
  663                         uptag = pc->tag_parent;
  664 
  665                         /*
  666                          * We have to unload the previous loaded DMA
  667                          * pages before trying to load a new one!
  668                          */
  669                         if (pc->isloaded)
  670                                 bus_dmamap_unload(pc->tag, pc->map);
  671 
  672                         /*
  673                          * Try to load memory into DMA.
  674                          */
  675                         err = bus_dmamap_load(
  676                             pc->tag, pc->map, pc->buffer, size,
  677                             &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
  678                         if (err == EINPROGRESS) {
  679                                 cv_wait(uptag->cv, uptag->mtx);
  680                                 err = 0;
  681                         }
  682                         if (err || uptag->dma_error) {
  683                                 pc->isloaded = 0;
  684                                 return (1);
  685                         }
  686                 } else {
  687                         /*
  688                          * We have to unload the previous loaded DMA
  689                          * pages before trying to load a new one!
  690                          */
  691                         if (pc->isloaded)
  692                                 bus_dmamap_unload(pc->tag, pc->map);
  693 
  694                         /*
  695                          * Try to load memory into DMA. The callback
  696                          * will be called in all cases:
  697                          */
  698                         if (bus_dmamap_load(
  699                             pc->tag, pc->map, pc->buffer, size,
  700                             &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
  701                         }
  702                 }
  703                 pc->isloaded = 1;
  704         } else {
  705                 if (!sync) {
  706                         /*
  707                          * Call callback so that refcount is decremented
  708                          * properly:
  709                          */
  710                         pc->tag_parent->dma_error = 0;
  711                         (pc->tag_parent->func) (pc->tag_parent);
  712                 }
  713         }
  714         return (0);
  715 }
  716 
  717 /*------------------------------------------------------------------------*
  718  *      usb_pc_cpu_invalidate - invalidate CPU cache
  719  *------------------------------------------------------------------------*/
  720 void
  721 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
  722 {
  723         if (pc->page_offset_end == pc->page_offset_buf) {
  724                 /* nothing has been loaded into this page cache! */
  725                 return;
  726         }
  727 
  728         /*
  729          * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
  730          * same time, but in the future we should try to isolate the
  731          * different cases to optimise the code. --HPS
  732          */
  733         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
  734         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
  735 }
  736 
  737 /*------------------------------------------------------------------------*
  738  *      usb_pc_cpu_flush - flush CPU cache
  739  *------------------------------------------------------------------------*/
  740 void
  741 usb_pc_cpu_flush(struct usb_page_cache *pc)
  742 {
  743         if (pc->page_offset_end == pc->page_offset_buf) {
  744                 /* nothing has been loaded into this page cache! */
  745                 return;
  746         }
  747         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
  748 }
  749 
  750 /*------------------------------------------------------------------------*
  751  *      usb_pc_dmamap_create - create a DMA map
  752  *
  753  * Returns:
  754  *    0: Success
  755  * Else: Failure
  756  *------------------------------------------------------------------------*/
  757 uint8_t
  758 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
  759 {
  760         struct usb_xfer_root *info;
  761         struct usb_dma_tag *utag;
  762 
  763         /* get info */
  764         info = USB_DMATAG_TO_XROOT(pc->tag_parent);
  765 
  766         /* sanity check */
  767         if (info == NULL) {
  768                 goto error;
  769         }
  770         utag = usb_dma_tag_find(pc->tag_parent, size, 1);
  771         if (utag == NULL) {
  772                 goto error;
  773         }
  774         /* create DMA map */
  775         if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
  776                 goto error;
  777         }
  778         pc->tag = utag->tag;
  779         return 0;                       /* success */
  780 
  781 error:
  782         pc->map = NULL;
  783         pc->tag = NULL;
  784         return 1;                       /* failure */
  785 }
  786 
  787 /*------------------------------------------------------------------------*
  788  *      usb_pc_dmamap_destroy
  789  *
  790  * This function is NULL safe.
  791  *------------------------------------------------------------------------*/
  792 void
  793 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
  794 {
  795         if (pc && pc->tag) {
  796                 if (pc->isloaded)
  797                         bus_dmamap_unload(pc->tag, pc->map);
  798                 bus_dmamap_destroy(pc->tag, pc->map);
  799                 pc->tag = NULL;
  800                 pc->map = NULL;
  801         }
  802 }
  803 
  804 /*------------------------------------------------------------------------*
  805  *      usb_dma_tag_find - factored out code
  806  *------------------------------------------------------------------------*/
  807 struct usb_dma_tag *
  808 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
  809     usb_size_t size, usb_size_t align)
  810 {
  811         struct usb_dma_tag *udt;
  812         uint8_t nudt;
  813 
  814         USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
  815         USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
  816 
  817         udt = udpt->utag_first;
  818         nudt = udpt->utag_max;
  819 
  820         while (nudt--) {
  821                 if (udt->align == 0) {
  822                         usb_dma_tag_create(udt, size, align);
  823                         if (udt->tag == NULL) {
  824                                 return (NULL);
  825                         }
  826                         udt->align = align;
  827                         udt->size = size;
  828                         return (udt);
  829                 }
  830                 if ((udt->align == align) && (udt->size == size)) {
  831                         return (udt);
  832                 }
  833                 udt++;
  834         }
  835         return (NULL);
  836 }
  837 
  838 /*------------------------------------------------------------------------*
  839  *      usb_dma_tag_setup - initialise USB DMA tags
  840  *------------------------------------------------------------------------*/
  841 void
  842 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
  843     struct usb_dma_tag *udt, bus_dma_tag_t dmat,
  844     struct mtx *mtx, usb_dma_callback_t *func,
  845     uint8_t ndmabits, uint8_t nudt)
  846 {
  847         memset(udpt, 0, sizeof(*udpt));
  848 
  849         /* sanity checking */
  850         if ((nudt == 0) ||
  851             (ndmabits == 0) ||
  852             (mtx == NULL)) {
  853                 /* something is corrupt */
  854                 return;
  855         }
  856         /* initialise condition variable */
  857         cv_init(udpt->cv, "USB DMA CV");
  858 
  859         /* store some information */
  860         udpt->mtx = mtx;
  861         udpt->func = func;
  862         udpt->tag = dmat;
  863         udpt->utag_first = udt;
  864         udpt->utag_max = nudt;
  865         udpt->dma_bits = ndmabits;
  866 
  867         while (nudt--) {
  868                 memset(udt, 0, sizeof(*udt));
  869                 udt->tag_parent = udpt;
  870                 udt++;
  871         }
  872 }
  873 
  874 /*------------------------------------------------------------------------*
  875  *      usb_bus_tag_unsetup - factored out code
  876  *------------------------------------------------------------------------*/
  877 void
  878 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
  879 {
  880         struct usb_dma_tag *udt;
  881         uint8_t nudt;
  882 
  883         udt = udpt->utag_first;
  884         nudt = udpt->utag_max;
  885 
  886         while (nudt--) {
  887                 if (udt->align) {
  888                         /* destroy the USB DMA tag */
  889                         usb_dma_tag_destroy(udt);
  890                         udt->align = 0;
  891                 }
  892                 udt++;
  893         }
  894 
  895         if (udpt->utag_max) {
  896                 /* destroy the condition variable */
  897                 cv_destroy(udpt->cv);
  898         }
  899 }
  900 
  901 /*------------------------------------------------------------------------*
  902  *      usb_bdma_work_loop
  903  *
  904  * This function handles loading of virtual buffers into DMA and is
  905  * only called when "dma_refcount" is zero.
  906  *------------------------------------------------------------------------*/
  907 void
  908 usb_bdma_work_loop(struct usb_xfer_queue *pq)
  909 {
  910         struct usb_xfer_root *info;
  911         struct usb_xfer *xfer;
  912         usb_frcount_t nframes;
  913 
  914         xfer = pq->curr;
  915         info = xfer->xroot;
  916 
  917         USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED);
  918 
  919         if (xfer->error) {
  920                 /* some error happened */
  921                 USB_BUS_LOCK(info->bus);
  922                 usbd_transfer_done(xfer, 0);
  923                 USB_BUS_UNLOCK(info->bus);
  924                 return;
  925         }
  926         if (!xfer->flags_int.bdma_setup) {
  927                 struct usb_page *pg;
  928                 usb_frlength_t frlength_0;
  929                 uint8_t isread;
  930 
  931                 xfer->flags_int.bdma_setup = 1;
  932 
  933                 /* reset BUS-DMA load state */
  934 
  935                 info->dma_error = 0;
  936 
  937                 if (xfer->flags_int.isochronous_xfr) {
  938                         /* only one frame buffer */
  939                         nframes = 1;
  940                         frlength_0 = xfer->sumlen;
  941                 } else {
  942                         /* can be multiple frame buffers */
  943                         nframes = xfer->nframes;
  944                         frlength_0 = xfer->frlengths[0];
  945                 }
  946 
  947                 /*
  948                  * Set DMA direction first. This is needed to
  949                  * select the correct cache invalidate and cache
  950                  * flush operations.
  951                  */
  952                 isread = USB_GET_DATA_ISREAD(xfer);
  953                 pg = xfer->dma_page_ptr;
  954 
  955                 if (xfer->flags_int.control_xfr &&
  956                     xfer->flags_int.control_hdr) {
  957                         /* special case */
  958                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
  959                                 /* The device controller writes to memory */
  960                                 xfer->frbuffers[0].isread = 1;
  961                         } else {
  962                                 /* The host controller reads from memory */
  963                                 xfer->frbuffers[0].isread = 0;
  964                         }
  965                 } else {
  966                         /* default case */
  967                         xfer->frbuffers[0].isread = isread;
  968                 }
  969 
  970                 /*
  971                  * Setup the "page_start" pointer which points to an array of
  972                  * USB pages where information about the physical address of a
  973                  * page will be stored. Also initialise the "isread" field of
  974                  * the USB page caches.
  975                  */
  976                 xfer->frbuffers[0].page_start = pg;
  977 
  978                 info->dma_nframes = nframes;
  979                 info->dma_currframe = 0;
  980                 info->dma_frlength_0 = frlength_0;
  981 
  982                 pg += (frlength_0 / USB_PAGE_SIZE);
  983                 pg += 2;
  984 
  985                 while (--nframes > 0) {
  986                         xfer->frbuffers[nframes].isread = isread;
  987                         xfer->frbuffers[nframes].page_start = pg;
  988 
  989                         pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
  990                         pg += 2;
  991                 }
  992         }
  993         if (info->dma_error) {
  994                 USB_BUS_LOCK(info->bus);
  995                 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
  996                 USB_BUS_UNLOCK(info->bus);
  997                 return;
  998         }
  999         if (info->dma_currframe != info->dma_nframes) {
 1000                 if (info->dma_currframe == 0) {
 1001                         /* special case */
 1002                         usb_pc_load_mem(xfer->frbuffers,
 1003                             info->dma_frlength_0, 0);
 1004                 } else {
 1005                         /* default case */
 1006                         nframes = info->dma_currframe;
 1007                         usb_pc_load_mem(xfer->frbuffers + nframes,
 1008                             xfer->frlengths[nframes], 0);
 1009                 }
 1010 
 1011                 /* advance frame index */
 1012                 info->dma_currframe++;
 1013 
 1014                 return;
 1015         }
 1016         /* go ahead */
 1017         usb_bdma_pre_sync(xfer);
 1018 
 1019         /* start loading next USB transfer, if any */
 1020         usb_command_wrapper(pq, NULL);
 1021 
 1022         /* finally start the hardware */
 1023         usbd_pipe_enter(xfer);
 1024 }
 1025 
 1026 /*------------------------------------------------------------------------*
 1027  *      usb_bdma_done_event
 1028  *
 1029  * This function is called when the BUS-DMA has loaded virtual memory
 1030  * into DMA, if any.
 1031  *------------------------------------------------------------------------*/
 1032 void
 1033 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
 1034 {
 1035         struct usb_xfer_root *info;
 1036 
 1037         info = USB_DMATAG_TO_XROOT(udpt);
 1038 
 1039         USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED);
 1040 
 1041         /* copy error */
 1042         info->dma_error = udpt->dma_error;
 1043 
 1044         /* enter workloop again */
 1045         usb_command_wrapper(&info->dma_q,
 1046             info->dma_q.curr);
 1047 }
 1048 
 1049 /*------------------------------------------------------------------------*
 1050  *      usb_bdma_pre_sync
 1051  *
 1052  * This function handles DMA synchronisation that must be done before
 1053  * an USB transfer is started.
 1054  *------------------------------------------------------------------------*/
 1055 void
 1056 usb_bdma_pre_sync(struct usb_xfer *xfer)
 1057 {
 1058         struct usb_page_cache *pc;
 1059         usb_frcount_t nframes;
 1060 
 1061         if (xfer->flags_int.isochronous_xfr) {
 1062                 /* only one frame buffer */
 1063                 nframes = 1;
 1064         } else {
 1065                 /* can be multiple frame buffers */
 1066                 nframes = xfer->nframes;
 1067         }
 1068 
 1069         pc = xfer->frbuffers;
 1070 
 1071         while (nframes--) {
 1072                 if (pc->isread) {
 1073                         usb_pc_cpu_invalidate(pc);
 1074                 } else {
 1075                         usb_pc_cpu_flush(pc);
 1076                 }
 1077                 pc++;
 1078         }
 1079 }
 1080 
 1081 /*------------------------------------------------------------------------*
 1082  *      usb_bdma_post_sync
 1083  *
 1084  * This function handles DMA synchronisation that must be done after
 1085  * an USB transfer is complete.
 1086  *------------------------------------------------------------------------*/
 1087 void
 1088 usb_bdma_post_sync(struct usb_xfer *xfer)
 1089 {
 1090         struct usb_page_cache *pc;
 1091         usb_frcount_t nframes;
 1092 
 1093         if (xfer->flags_int.isochronous_xfr) {
 1094                 /* only one frame buffer */
 1095                 nframes = 1;
 1096         } else {
 1097                 /* can be multiple frame buffers */
 1098                 nframes = xfer->nframes;
 1099         }
 1100 
 1101         pc = xfer->frbuffers;
 1102 
 1103         while (nframes--) {
 1104                 if (pc->isread) {
 1105                         usb_pc_cpu_invalidate(pc);
 1106                 }
 1107                 pc++;
 1108         }
 1109 }
 1110 
 1111 #endif
Cache object: 1acce2a0aa708c40778b9551ec026358
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/usb/usb_busdma.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/usb/usb_busdma.c
