usb_busdma.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /* $FreeBSD: releng/11.2/sys/dev/usb/usb_busdma.c 331722 2018-03-29 02:50:57Z eadler $ */
    2 /*-
    3  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #ifdef USB_GLOBAL_INCLUDE_FILE
   28 #include USB_GLOBAL_INCLUDE_FILE
   29 #else
   30 #include <sys/stdint.h>
   31 #include <sys/stddef.h>
   32 #include <sys/param.h>
   33 #include <sys/queue.h>
   34 #include <sys/types.h>
   35 #include <sys/systm.h>
   36 #include <sys/kernel.h>
   37 #include <sys/bus.h>
   38 #include <sys/module.h>
   39 #include <sys/lock.h>
   40 #include <sys/mutex.h>
   41 #include <sys/condvar.h>
   42 #include <sys/sysctl.h>
   43 #include <sys/sx.h>
   44 #include <sys/unistd.h>
   45 #include <sys/callout.h>
   46 #include <sys/malloc.h>
   47 #include <sys/priv.h>
   48 
   49 #include <dev/usb/usb.h>
   50 #include <dev/usb/usbdi.h>
   51 #include <dev/usb/usbdi_util.h>
   52 
   53 #define USB_DEBUG_VAR usb_debug
   54 
   55 #include <dev/usb/usb_core.h>
   56 #include <dev/usb/usb_busdma.h>
   57 #include <dev/usb/usb_process.h>
   58 #include <dev/usb/usb_transfer.h>
   59 #include <dev/usb/usb_device.h>
   60 #include <dev/usb/usb_util.h>
   61 #include <dev/usb/usb_debug.h>
   62 
   63 #include <dev/usb/usb_controller.h>
   64 #include <dev/usb/usb_bus.h>
   65 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
   66 
   67 #if USB_HAVE_BUSDMA
   68 static void     usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
   69 static void     usb_dma_tag_destroy(struct usb_dma_tag *);
   70 static void     usb_dma_lock_cb(void *, bus_dma_lock_op_t);
   71 static void     usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
   72 static void     usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
   73 static void     usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
   74                     uint8_t);
   75 #endif
   76 
   77 /*------------------------------------------------------------------------*
   78  *  usbd_get_page - lookup DMA-able memory for the given offset
   79  *
   80  * NOTE: Only call this function when the "page_cache" structure has
   81  * been properly initialized !
   82  *------------------------------------------------------------------------*/
   83 void
   84 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
   85     struct usb_page_search *res)
   86 {
   87 #if USB_HAVE_BUSDMA
   88         struct usb_page *page;
   89 
   90         if (pc->page_start) {
   91 
   92                 /* Case 1 - something has been loaded into DMA */
   93 
   94                 if (pc->buffer) {
   95 
   96                         /* Case 1a - Kernel Virtual Address */
   97 
   98                         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
   99                 }
  100                 offset += pc->page_offset_buf;
  101 
  102                 /* compute destination page */
  103 
  104                 page = pc->page_start;
  105 
  106                 if (pc->ismultiseg) {
  107 
  108                         page += (offset / USB_PAGE_SIZE);
  109 
  110                         offset %= USB_PAGE_SIZE;
  111 
  112                         res->length = USB_PAGE_SIZE - offset;
  113                         res->physaddr = page->physaddr + offset;
  114                 } else {
  115                         res->length = (usb_size_t)-1;
  116                         res->physaddr = page->physaddr + offset;
  117                 }
  118                 if (!pc->buffer) {
  119 
  120                         /* Case 1b - Non Kernel Virtual Address */
  121 
  122                         res->buffer = USB_ADD_BYTES(page->buffer, offset);
  123                 }
  124                 return;
  125         }
  126 #endif
  127         /* Case 2 - Plain PIO */
  128 
  129         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
  130         res->length = (usb_size_t)-1;
  131 #if USB_HAVE_BUSDMA
  132         res->physaddr = 0;
  133 #endif
  134 }
  135 
  136 /*------------------------------------------------------------------------*
  137  *  usb_pc_buffer_is_aligned - verify alignment
  138  * 
  139  * This function is used to check if a page cache buffer is properly
  140  * aligned to reduce the use of bounce buffers in PIO mode.
  141  *------------------------------------------------------------------------*/
  142 uint8_t
  143 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset,
  144     usb_frlength_t len, usb_frlength_t mask)
  145 {
  146         struct usb_page_search buf_res;
  147 
  148         while (len != 0) {
  149 
  150                 usbd_get_page(pc, offset, &buf_res);
  151 
  152                 if (buf_res.length > len)
  153                         buf_res.length = len;
  154                 if (USB_P2U(buf_res.buffer) & mask)
  155                         return (0);
  156                 if (buf_res.length & mask)
  157                         return (0);
  158 
  159                 offset += buf_res.length;
  160                 len -= buf_res.length;
  161         }
  162         return (1);
  163 }
  164 
  165 /*------------------------------------------------------------------------*
  166  *  usbd_copy_in - copy directly to DMA-able memory
  167  *------------------------------------------------------------------------*/
  168 void
  169 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
  170     const void *ptr, usb_frlength_t len)
  171 {
  172         struct usb_page_search buf_res;
  173 
  174         while (len != 0) {
  175 
  176                 usbd_get_page(cache, offset, &buf_res);
  177 
  178                 if (buf_res.length > len) {
  179                         buf_res.length = len;
  180                 }
  181                 memcpy(buf_res.buffer, ptr, buf_res.length);
  182 
  183                 offset += buf_res.length;
  184                 len -= buf_res.length;
  185                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
  186         }
  187 }
  188 
  189 /*------------------------------------------------------------------------*
  190  *  usbd_copy_in_user - copy directly to DMA-able memory from userland
  191  *
  192  * Return values:
  193  *    0: Success
  194  * Else: Failure
  195  *------------------------------------------------------------------------*/
  196 #if USB_HAVE_USER_IO
  197 int
  198 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
  199     const void *ptr, usb_frlength_t len)
  200 {
  201         struct usb_page_search buf_res;
  202         int error;
  203 
  204         while (len != 0) {
  205 
  206                 usbd_get_page(cache, offset, &buf_res);
  207 
  208                 if (buf_res.length > len) {
  209                         buf_res.length = len;
  210                 }
  211                 error = copyin(ptr, buf_res.buffer, buf_res.length);
  212                 if (error)
  213                         return (error);
  214 
  215                 offset += buf_res.length;
  216                 len -= buf_res.length;
  217                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
  218         }
  219         return (0);                     /* success */
  220 }
  221 #endif
  222 
  223 /*------------------------------------------------------------------------*
  224  *  usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
  225  *------------------------------------------------------------------------*/
  226 #if USB_HAVE_MBUF
  227 struct usb_m_copy_in_arg {
  228         struct usb_page_cache *cache;
  229         usb_frlength_t dst_offset;
  230 };
  231 
  232 static int
  233 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
  234 {
  235         struct usb_m_copy_in_arg *ua = arg;
  236 
  237         usbd_copy_in(ua->cache, ua->dst_offset, src, count);
  238         ua->dst_offset += count;
  239         return (0);
  240 }
  241 
  242 void
  243 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
  244     struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
  245 {
  246         struct usb_m_copy_in_arg arg = {cache, dst_offset};
  247         (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
  248 }
  249 #endif
  250 
  251 /*------------------------------------------------------------------------*
  252  *  usb_uiomove - factored out code
  253  *------------------------------------------------------------------------*/
  254 #if USB_HAVE_USER_IO
  255 int
  256 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
  257     usb_frlength_t pc_offset, usb_frlength_t len)
  258 {
  259         struct usb_page_search res;
  260         int error = 0;
  261 
  262         while (len != 0) {
  263 
  264                 usbd_get_page(pc, pc_offset, &res);
  265 
  266                 if (res.length > len) {
  267                         res.length = len;
  268                 }
  269                 /*
  270                  * "uiomove()" can sleep so one needs to make a wrapper,
  271                  * exiting the mutex and checking things
  272                  */
  273                 error = uiomove(res.buffer, res.length, uio);
  274 
  275                 if (error) {
  276                         break;
  277                 }
  278                 pc_offset += res.length;
  279                 len -= res.length;
  280         }
  281         return (error);
  282 }
  283 #endif
  284 
  285 /*------------------------------------------------------------------------*
  286  *  usbd_copy_out - copy directly from DMA-able memory
  287  *------------------------------------------------------------------------*/
  288 void
  289 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
  290     void *ptr, usb_frlength_t len)
  291 {
  292         struct usb_page_search res;
  293 
  294         while (len != 0) {
  295 
  296                 usbd_get_page(cache, offset, &res);
  297 
  298                 if (res.length > len) {
  299                         res.length = len;
  300                 }
  301                 memcpy(ptr, res.buffer, res.length);
  302 
  303                 offset += res.length;
  304                 len -= res.length;
  305                 ptr = USB_ADD_BYTES(ptr, res.length);
  306         }
  307 }
  308 
  309 /*------------------------------------------------------------------------*
  310  *  usbd_copy_out_user - copy directly from DMA-able memory to userland
  311  *
  312  * Return values:
  313  *    0: Success
  314  * Else: Failure
  315  *------------------------------------------------------------------------*/
  316 #if USB_HAVE_USER_IO
  317 int
  318 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
  319     void *ptr, usb_frlength_t len)
  320 {
  321         struct usb_page_search res;
  322         int error;
  323 
  324         while (len != 0) {
  325 
  326                 usbd_get_page(cache, offset, &res);
  327 
  328                 if (res.length > len) {
  329                         res.length = len;
  330                 }
  331                 error = copyout(res.buffer, ptr, res.length);
  332                 if (error)
  333                         return (error);
  334 
  335                 offset += res.length;
  336                 len -= res.length;
  337                 ptr = USB_ADD_BYTES(ptr, res.length);
  338         }
  339         return (0);                     /* success */
  340 }
  341 #endif
  342 
  343 /*------------------------------------------------------------------------*
  344  *  usbd_frame_zero - zero DMA-able memory
  345  *------------------------------------------------------------------------*/
  346 void
  347 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
  348     usb_frlength_t len)
  349 {
  350         struct usb_page_search res;
  351 
  352         while (len != 0) {
  353 
  354                 usbd_get_page(cache, offset, &res);
  355 
  356                 if (res.length > len) {
  357                         res.length = len;
  358                 }
  359                 memset(res.buffer, 0, res.length);
  360 
  361                 offset += res.length;
  362                 len -= res.length;
  363         }
  364 }
  365 
  366 #if USB_HAVE_BUSDMA
  367 
  368 /*------------------------------------------------------------------------*
  369  *      usb_dma_lock_cb - dummy callback
  370  *------------------------------------------------------------------------*/
  371 static void
  372 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
  373 {
  374         /* we use "mtx_owned()" instead of this function */
  375 }
  376 
  377 /*------------------------------------------------------------------------*
  378  *      usb_dma_tag_create - allocate a DMA tag
  379  *
  380  * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
  381  * allow multi-segment mappings. Else all mappings are single-segment.
  382  *------------------------------------------------------------------------*/
  383 static void
  384 usb_dma_tag_create(struct usb_dma_tag *udt,
  385     usb_size_t size, usb_size_t align)
  386 {
  387         bus_dma_tag_t tag;
  388 
  389         if (bus_dma_tag_create
  390             ( /* parent    */ udt->tag_parent->tag,
  391              /* alignment */ align,
  392              /* boundary  */ 0,
  393              /* lowaddr   */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
  394              /* highaddr  */ BUS_SPACE_MAXADDR,
  395              /* filter    */ NULL,
  396              /* filterarg */ NULL,
  397              /* maxsize   */ size,
  398              /* nsegments */ (align == 1 && size > 1) ?
  399             (2 + (size / USB_PAGE_SIZE)) : 1,
  400              /* maxsegsz  */ (align == 1 && size > USB_PAGE_SIZE) ?
  401             USB_PAGE_SIZE : size,
  402              /* flags     */ BUS_DMA_KEEP_PG_OFFSET,
  403              /* lockfn    */ &usb_dma_lock_cb,
  404              /* lockarg   */ NULL,
  405             &tag)) {
  406                 tag = NULL;
  407         }
  408         udt->tag = tag;
  409 }
  410 
  411 /*------------------------------------------------------------------------*
  412  *      usb_dma_tag_free - free a DMA tag
  413  *------------------------------------------------------------------------*/
  414 static void
  415 usb_dma_tag_destroy(struct usb_dma_tag *udt)
  416 {
  417         bus_dma_tag_destroy(udt->tag);
  418 }
  419 
  420 /*------------------------------------------------------------------------*
  421  *      usb_pc_alloc_mem_cb - BUS-DMA callback function
  422  *------------------------------------------------------------------------*/
  423 static void
  424 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
  425     int nseg, int error)
  426 {
  427         usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
  428 }
  429 
  430 /*------------------------------------------------------------------------*
  431  *      usb_pc_load_mem_cb - BUS-DMA callback function
  432  *------------------------------------------------------------------------*/
  433 static void
  434 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
  435     int nseg, int error)
  436 {
  437         usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
  438 }
  439 
  440 /*------------------------------------------------------------------------*
  441  *      usb_pc_common_mem_cb - BUS-DMA callback function
  442  *------------------------------------------------------------------------*/
  443 static void
  444 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
  445     int nseg, int error, uint8_t isload)
  446 {
  447         struct usb_dma_parent_tag *uptag;
  448         struct usb_page_cache *pc;
  449         struct usb_page *pg;
  450         usb_size_t rem;
  451         bus_size_t off;
  452         uint8_t owned;
  453 
  454         pc = arg;
  455         uptag = pc->tag_parent;
  456 
  457         /*
  458          * XXX There is sometimes recursive locking here.
  459          * XXX We should try to find a better solution.
  460          * XXX Until further the "owned" variable does
  461          * XXX the trick.
  462          */
  463 
  464         if (error) {
  465                 goto done;
  466         }
  467 
  468         off = 0;
  469         pg = pc->page_start;
  470         pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE);
  471         rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
  472         pc->page_offset_buf = rem;
  473         pc->page_offset_end += rem;
  474 #ifdef USB_DEBUG
  475         if (nseg > 1) {
  476                 int x;
  477 
  478                 for (x = 0; x != nseg - 1; x++) {
  479                         if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) ==
  480                             ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1))))
  481                                 continue;
  482                         /*
  483                          * This check verifies there is no page offset
  484                          * hole between any of the segments. See the
  485                          * BUS_DMA_KEEP_PG_OFFSET flag.
  486                          */
  487                         DPRINTFN(0, "Page offset was not preserved\n");
  488                         error = 1;
  489                         goto done;
  490                 }
  491         }
  492 #endif
  493         while (pc->ismultiseg) {
  494                 off += USB_PAGE_SIZE;
  495                 if (off >= (segs->ds_len + rem)) {
  496                         /* page crossing */
  497                         nseg--;
  498                         segs++;
  499                         off = 0;
  500                         rem = 0;
  501                         if (nseg == 0)
  502                                 break;
  503                 }
  504                 pg++;
  505                 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE);
  506         }
  507 
  508 done:
  509         owned = mtx_owned(uptag->mtx);
  510         if (!owned)
  511                 mtx_lock(uptag->mtx);
  512 
  513         uptag->dma_error = (error ? 1 : 0);
  514         if (isload) {
  515                 (uptag->func) (uptag);
  516         } else {
  517                 cv_broadcast(uptag->cv);
  518         }
  519         if (!owned)
  520                 mtx_unlock(uptag->mtx);
  521 }
  522 
  523 /*------------------------------------------------------------------------*
  524  *      usb_pc_alloc_mem - allocate DMA'able memory
  525  *
  526  * Returns:
  527  *    0: Success
  528  * Else: Failure
  529  *------------------------------------------------------------------------*/
  530 uint8_t
  531 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
  532     usb_size_t size, usb_size_t align)
  533 {
  534         struct usb_dma_parent_tag *uptag;
  535         struct usb_dma_tag *utag;
  536         bus_dmamap_t map;
  537         void *ptr;
  538         int err;
  539 
  540         uptag = pc->tag_parent;
  541 
  542         if (align != 1) {
  543                 /*
  544                  * The alignment must be greater or equal to the
  545                  * "size" else the object can be split between two
  546                  * memory pages and we get a problem!
  547                  */
  548                 while (align < size) {
  549                         align *= 2;
  550                         if (align == 0) {
  551                                 goto error;
  552                         }
  553                 }
  554 #if 1
  555                 /*
  556                  * XXX BUS-DMA workaround - FIXME later:
  557                  *
  558                  * We assume that that the aligment at this point of
  559                  * the code is greater than or equal to the size and
  560                  * less than two times the size, so that if we double
  561                  * the size, the size will be greater than the
  562                  * alignment.
  563                  *
  564                  * The bus-dma system has a check for "alignment"
  565                  * being less than "size". If that check fails we end
  566                  * up using contigmalloc which is page based even for
  567                  * small allocations. Try to avoid that to save
  568                  * memory, hence we sometimes to a large number of
  569                  * small allocations!
  570                  */
  571                 if (size <= (USB_PAGE_SIZE / 2)) {
  572                         size *= 2;
  573                 }
  574 #endif
  575         }
  576         /* get the correct DMA tag */
  577         utag = usb_dma_tag_find(uptag, size, align);
  578         if (utag == NULL) {
  579                 goto error;
  580         }
  581         /* allocate memory */
  582         if (bus_dmamem_alloc(
  583             utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
  584                 goto error;
  585         }
  586         /* setup page cache */
  587         pc->buffer = ptr;
  588         pc->page_start = pg;
  589         pc->page_offset_buf = 0;
  590         pc->page_offset_end = size;
  591         pc->map = map;
  592         pc->tag = utag->tag;
  593         pc->ismultiseg = (align == 1);
  594 
  595         mtx_lock(uptag->mtx);
  596 
  597         /* load memory into DMA */
  598         err = bus_dmamap_load(
  599             utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
  600             pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
  601 
  602         if (err == EINPROGRESS) {
  603                 cv_wait(uptag->cv, uptag->mtx);
  604                 err = 0;
  605         }
  606         mtx_unlock(uptag->mtx);
  607 
  608         if (err || uptag->dma_error) {
  609                 bus_dmamem_free(utag->tag, ptr, map);
  610                 goto error;
  611         }
  612         memset(ptr, 0, size);
  613 
  614         usb_pc_cpu_flush(pc);
  615 
  616         return (0);
  617 
  618 error:
  619         /* reset most of the page cache */
  620         pc->buffer = NULL;
  621         pc->page_start = NULL;
  622         pc->page_offset_buf = 0;
  623         pc->page_offset_end = 0;
  624         pc->map = NULL;
  625         pc->tag = NULL;
  626         return (1);
  627 }
  628 
  629 /*------------------------------------------------------------------------*
  630  *      usb_pc_free_mem - free DMA memory
  631  *
  632  * This function is NULL safe.
  633  *------------------------------------------------------------------------*/
  634 void
  635 usb_pc_free_mem(struct usb_page_cache *pc)
  636 {
  637         if (pc && pc->buffer) {
  638 
  639                 bus_dmamap_unload(pc->tag, pc->map);
  640 
  641                 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
  642 
  643                 pc->buffer = NULL;
  644         }
  645 }
  646 
  647 /*------------------------------------------------------------------------*
  648  *      usb_pc_load_mem - load virtual memory into DMA
  649  *
  650  * Return values:
  651  * 0: Success
  652  * Else: Error
  653  *------------------------------------------------------------------------*/
  654 uint8_t
  655 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
  656 {
  657         /* setup page cache */
  658         pc->page_offset_buf = 0;
  659         pc->page_offset_end = size;
  660         pc->ismultiseg = 1;
  661 
  662         mtx_assert(pc->tag_parent->mtx, MA_OWNED);
  663 
  664         if (size > 0) {
  665                 if (sync) {
  666                         struct usb_dma_parent_tag *uptag;
  667                         int err;
  668 
  669                         uptag = pc->tag_parent;
  670 
  671                         /*
  672                          * We have to unload the previous loaded DMA
  673                          * pages before trying to load a new one!
  674                          */
  675                         bus_dmamap_unload(pc->tag, pc->map);
  676 
  677                         /*
  678                          * Try to load memory into DMA.
  679                          */
  680                         err = bus_dmamap_load(
  681                             pc->tag, pc->map, pc->buffer, size,
  682                             &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
  683                         if (err == EINPROGRESS) {
  684                                 cv_wait(uptag->cv, uptag->mtx);
  685                                 err = 0;
  686                         }
  687                         if (err || uptag->dma_error) {
  688                                 return (1);
  689                         }
  690                 } else {
  691 
  692                         /*
  693                          * We have to unload the previous loaded DMA
  694                          * pages before trying to load a new one!
  695                          */
  696                         bus_dmamap_unload(pc->tag, pc->map);
  697 
  698                         /*
  699                          * Try to load memory into DMA. The callback
  700                          * will be called in all cases:
  701                          */
  702                         if (bus_dmamap_load(
  703                             pc->tag, pc->map, pc->buffer, size,
  704                             &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
  705                         }
  706                 }
  707         } else {
  708                 if (!sync) {
  709                         /*
  710                          * Call callback so that refcount is decremented
  711                          * properly:
  712                          */
  713                         pc->tag_parent->dma_error = 0;
  714                         (pc->tag_parent->func) (pc->tag_parent);
  715                 }
  716         }
  717         return (0);
  718 }
  719 
  720 /*------------------------------------------------------------------------*
  721  *      usb_pc_cpu_invalidate - invalidate CPU cache
  722  *------------------------------------------------------------------------*/
  723 void
  724 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
  725 {
  726         if (pc->page_offset_end == pc->page_offset_buf) {
  727                 /* nothing has been loaded into this page cache! */
  728                 return;
  729         }
  730 
  731         /*
  732          * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
  733          * same time, but in the future we should try to isolate the
  734          * different cases to optimise the code. --HPS
  735          */
  736         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
  737         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
  738 }
  739 
  740 /*------------------------------------------------------------------------*
  741  *      usb_pc_cpu_flush - flush CPU cache
  742  *------------------------------------------------------------------------*/
  743 void
  744 usb_pc_cpu_flush(struct usb_page_cache *pc)
  745 {
  746         if (pc->page_offset_end == pc->page_offset_buf) {
  747                 /* nothing has been loaded into this page cache! */
  748                 return;
  749         }
  750         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
  751 }
  752 
  753 /*------------------------------------------------------------------------*
  754  *      usb_pc_dmamap_create - create a DMA map
  755  *
  756  * Returns:
  757  *    0: Success
  758  * Else: Failure
  759  *------------------------------------------------------------------------*/
  760 uint8_t
  761 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
  762 {
  763         struct usb_xfer_root *info;
  764         struct usb_dma_tag *utag;
  765 
  766         /* get info */
  767         info = USB_DMATAG_TO_XROOT(pc->tag_parent);
  768 
  769         /* sanity check */
  770         if (info == NULL) {
  771                 goto error;
  772         }
  773         utag = usb_dma_tag_find(pc->tag_parent, size, 1);
  774         if (utag == NULL) {
  775                 goto error;
  776         }
  777         /* create DMA map */
  778         if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
  779                 goto error;
  780         }
  781         pc->tag = utag->tag;
  782         return 0;                       /* success */
  783 
  784 error:
  785         pc->map = NULL;
  786         pc->tag = NULL;
  787         return 1;                       /* failure */
  788 }
  789 
  790 /*------------------------------------------------------------------------*
  791  *      usb_pc_dmamap_destroy
  792  *
  793  * This function is NULL safe.
  794  *------------------------------------------------------------------------*/
  795 void
  796 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
  797 {
  798         if (pc && pc->tag) {
  799                 bus_dmamap_destroy(pc->tag, pc->map);
  800                 pc->tag = NULL;
  801                 pc->map = NULL;
  802         }
  803 }
  804 
  805 /*------------------------------------------------------------------------*
  806  *      usb_dma_tag_find - factored out code
  807  *------------------------------------------------------------------------*/
  808 struct usb_dma_tag *
  809 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
  810     usb_size_t size, usb_size_t align)
  811 {
  812         struct usb_dma_tag *udt;
  813         uint8_t nudt;
  814 
  815         USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
  816         USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
  817 
  818         udt = udpt->utag_first;
  819         nudt = udpt->utag_max;
  820 
  821         while (nudt--) {
  822 
  823                 if (udt->align == 0) {
  824                         usb_dma_tag_create(udt, size, align);
  825                         if (udt->tag == NULL) {
  826                                 return (NULL);
  827                         }
  828                         udt->align = align;
  829                         udt->size = size;
  830                         return (udt);
  831                 }
  832                 if ((udt->align == align) && (udt->size == size)) {
  833                         return (udt);
  834                 }
  835                 udt++;
  836         }
  837         return (NULL);
  838 }
  839 
  840 /*------------------------------------------------------------------------*
  841  *      usb_dma_tag_setup - initialise USB DMA tags
  842  *------------------------------------------------------------------------*/
  843 void
  844 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
  845     struct usb_dma_tag *udt, bus_dma_tag_t dmat,
  846     struct mtx *mtx, usb_dma_callback_t *func,
  847     uint8_t ndmabits, uint8_t nudt)
  848 {
  849         memset(udpt, 0, sizeof(*udpt));
  850 
  851         /* sanity checking */
  852         if ((nudt == 0) ||
  853             (ndmabits == 0) ||
  854             (mtx == NULL)) {
  855                 /* something is corrupt */
  856                 return;
  857         }
  858         /* initialise condition variable */
  859         cv_init(udpt->cv, "USB DMA CV");
  860 
  861         /* store some information */
  862         udpt->mtx = mtx;
  863         udpt->func = func;
  864         udpt->tag = dmat;
  865         udpt->utag_first = udt;
  866         udpt->utag_max = nudt;
  867         udpt->dma_bits = ndmabits;
  868 
  869         while (nudt--) {
  870                 memset(udt, 0, sizeof(*udt));
  871                 udt->tag_parent = udpt;
  872                 udt++;
  873         }
  874 }
  875 
  876 /*------------------------------------------------------------------------*
  877  *      usb_bus_tag_unsetup - factored out code
  878  *------------------------------------------------------------------------*/
  879 void
  880 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
  881 {
  882         struct usb_dma_tag *udt;
  883         uint8_t nudt;
  884 
  885         udt = udpt->utag_first;
  886         nudt = udpt->utag_max;
  887 
  888         while (nudt--) {
  889 
  890                 if (udt->align) {
  891                         /* destroy the USB DMA tag */
  892                         usb_dma_tag_destroy(udt);
  893                         udt->align = 0;
  894                 }
  895                 udt++;
  896         }
  897 
  898         if (udpt->utag_max) {
  899                 /* destroy the condition variable */
  900                 cv_destroy(udpt->cv);
  901         }
  902 }
  903 
  904 /*------------------------------------------------------------------------*
  905  *      usb_bdma_work_loop
  906  *
  907  * This function handles loading of virtual buffers into DMA and is
  908  * only called when "dma_refcount" is zero.
  909  *------------------------------------------------------------------------*/
  910 void
  911 usb_bdma_work_loop(struct usb_xfer_queue *pq)
  912 {
  913         struct usb_xfer_root *info;
  914         struct usb_xfer *xfer;
  915         usb_frcount_t nframes;
  916 
  917         xfer = pq->curr;
  918         info = xfer->xroot;
  919 
  920         mtx_assert(info->xfer_mtx, MA_OWNED);
  921 
  922         if (xfer->error) {
  923                 /* some error happened */
  924                 USB_BUS_LOCK(info->bus);
  925                 usbd_transfer_done(xfer, 0);
  926                 USB_BUS_UNLOCK(info->bus);
  927                 return;
  928         }
  929         if (!xfer->flags_int.bdma_setup) {
  930                 struct usb_page *pg;
  931                 usb_frlength_t frlength_0;
  932                 uint8_t isread;
  933 
  934                 xfer->flags_int.bdma_setup = 1;
  935 
  936                 /* reset BUS-DMA load state */
  937 
  938                 info->dma_error = 0;
  939 
  940                 if (xfer->flags_int.isochronous_xfr) {
  941                         /* only one frame buffer */
  942                         nframes = 1;
  943                         frlength_0 = xfer->sumlen;
  944                 } else {
  945                         /* can be multiple frame buffers */
  946                         nframes = xfer->nframes;
  947                         frlength_0 = xfer->frlengths[0];
  948                 }
  949 
  950                 /*
  951                  * Set DMA direction first. This is needed to
  952                  * select the correct cache invalidate and cache
  953                  * flush operations.
  954                  */
  955                 isread = USB_GET_DATA_ISREAD(xfer);
  956                 pg = xfer->dma_page_ptr;
  957 
  958                 if (xfer->flags_int.control_xfr &&
  959                     xfer->flags_int.control_hdr) {
  960                         /* special case */
  961                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
  962                                 /* The device controller writes to memory */
  963                                 xfer->frbuffers[0].isread = 1;
  964                         } else {
  965                                 /* The host controller reads from memory */
  966                                 xfer->frbuffers[0].isread = 0;
  967                         }
  968                 } else {
  969                         /* default case */
  970                         xfer->frbuffers[0].isread = isread;
  971                 }
  972 
  973                 /*
  974                  * Setup the "page_start" pointer which points to an array of
  975                  * USB pages where information about the physical address of a
  976                  * page will be stored. Also initialise the "isread" field of
  977                  * the USB page caches.
  978                  */
  979                 xfer->frbuffers[0].page_start = pg;
  980 
  981                 info->dma_nframes = nframes;
  982                 info->dma_currframe = 0;
  983                 info->dma_frlength_0 = frlength_0;
  984 
  985                 pg += (frlength_0 / USB_PAGE_SIZE);
  986                 pg += 2;
  987 
  988                 while (--nframes > 0) {
  989                         xfer->frbuffers[nframes].isread = isread;
  990                         xfer->frbuffers[nframes].page_start = pg;
  991 
  992                         pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
  993                         pg += 2;
  994                 }
  995 
  996         }
  997         if (info->dma_error) {
  998                 USB_BUS_LOCK(info->bus);
  999                 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
 1000                 USB_BUS_UNLOCK(info->bus);
 1001                 return;
 1002         }
 1003         if (info->dma_currframe != info->dma_nframes) {
 1004 
 1005                 if (info->dma_currframe == 0) {
 1006                         /* special case */
 1007                         usb_pc_load_mem(xfer->frbuffers,
 1008                             info->dma_frlength_0, 0);
 1009                 } else {
 1010                         /* default case */
 1011                         nframes = info->dma_currframe;
 1012                         usb_pc_load_mem(xfer->frbuffers + nframes,
 1013                             xfer->frlengths[nframes], 0);
 1014                 }
 1015 
 1016                 /* advance frame index */
 1017                 info->dma_currframe++;
 1018 
 1019                 return;
 1020         }
 1021         /* go ahead */
 1022         usb_bdma_pre_sync(xfer);
 1023 
 1024         /* start loading next USB transfer, if any */
 1025         usb_command_wrapper(pq, NULL);
 1026 
 1027         /* finally start the hardware */
 1028         usbd_pipe_enter(xfer);
 1029 }
 1030 
 1031 /*------------------------------------------------------------------------*
 1032  *      usb_bdma_done_event
 1033  *
 1034  * This function is called when the BUS-DMA has loaded virtual memory
 1035  * into DMA, if any.
 1036  *------------------------------------------------------------------------*/
 1037 void
 1038 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
 1039 {
 1040         struct usb_xfer_root *info;
 1041 
 1042         info = USB_DMATAG_TO_XROOT(udpt);
 1043 
 1044         mtx_assert(info->xfer_mtx, MA_OWNED);
 1045 
 1046         /* copy error */
 1047         info->dma_error = udpt->dma_error;
 1048 
 1049         /* enter workloop again */
 1050         usb_command_wrapper(&info->dma_q,
 1051             info->dma_q.curr);
 1052 }
 1053 
 1054 /*------------------------------------------------------------------------*
 1055  *      usb_bdma_pre_sync
 1056  *
 1057  * This function handles DMA synchronisation that must be done before
 1058  * an USB transfer is started.
 1059  *------------------------------------------------------------------------*/
 1060 void
 1061 usb_bdma_pre_sync(struct usb_xfer *xfer)
 1062 {
 1063         struct usb_page_cache *pc;
 1064         usb_frcount_t nframes;
 1065 
 1066         if (xfer->flags_int.isochronous_xfr) {
 1067                 /* only one frame buffer */
 1068                 nframes = 1;
 1069         } else {
 1070                 /* can be multiple frame buffers */
 1071                 nframes = xfer->nframes;
 1072         }
 1073 
 1074         pc = xfer->frbuffers;
 1075 
 1076         while (nframes--) {
 1077 
 1078                 if (pc->isread) {
 1079                         usb_pc_cpu_invalidate(pc);
 1080                 } else {
 1081                         usb_pc_cpu_flush(pc);
 1082                 }
 1083                 pc++;
 1084         }
 1085 }
 1086 
 1087 /*------------------------------------------------------------------------*
 1088  *      usb_bdma_post_sync
 1089  *
 1090  * This function handles DMA synchronisation that must be done after
 1091  * an USB transfer is complete.
 1092  *------------------------------------------------------------------------*/
 1093 void
 1094 usb_bdma_post_sync(struct usb_xfer *xfer)
 1095 {
 1096         struct usb_page_cache *pc;
 1097         usb_frcount_t nframes;
 1098 
 1099         if (xfer->flags_int.isochronous_xfr) {
 1100                 /* only one frame buffer */
 1101                 nframes = 1;
 1102         } else {
 1103                 /* can be multiple frame buffers */
 1104                 nframes = xfer->nframes;
 1105         }
 1106 
 1107         pc = xfer->frbuffers;
 1108 
 1109         while (nframes--) {
 1110                 if (pc->isread) {
 1111                         usb_pc_cpu_invalidate(pc);
 1112                 }
 1113                 pc++;
 1114         }
 1115 }
 1116 
 1117 #endif
Cache object: 0dc64789e5432e2eb5978014db70841d
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/usb/usb_busdma.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/usb/usb_busdma.c
