subr_bus_dma.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2012 EMC Corp.
    3  * All rights reserved.
    4  *
    5  * Copyright (c) 1997, 1998 Justin T. Gibbs.
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/11.0/sys/kern/subr_bus_dma.c 293698 2016-01-11 20:38:39Z cperciva $");
   32 
   33 #include "opt_bus.h"
   34 
   35 #include <sys/param.h>
   36 #include <sys/conf.h>
   37 #include <sys/systm.h>
   38 #include <sys/bio.h>
   39 #include <sys/bus.h>
   40 #include <sys/callout.h>
   41 #include <sys/mbuf.h>
   42 #include <sys/memdesc.h>
   43 #include <sys/proc.h>
   44 #include <sys/uio.h>
   45 
   46 #include <vm/vm.h>
   47 #include <vm/vm_page.h>
   48 #include <vm/vm_map.h>
   49 #include <vm/pmap.h>
   50 
   51 #include <cam/cam.h>
   52 #include <cam/cam_ccb.h>
   53 
   54 #include <machine/bus.h>
   55 
   56 /*
   57  * Load up data starting at offset within a region specified by a
   58  * list of virtual address ranges until either length or the region
   59  * are exhausted.
   60  */
   61 static int
   62 _bus_dmamap_load_vlist(bus_dma_tag_t dmat, bus_dmamap_t map,
   63     bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs,
   64     int flags, size_t offset, size_t length)
   65 {
   66         int error;
   67 
   68         error = 0;
   69         for (; sglist_cnt > 0 && length != 0; sglist_cnt--, list++) {
   70                 char *addr;
   71                 size_t ds_len;
   72 
   73                 KASSERT((offset < list->ds_len),
   74                     ("Invalid mid-segment offset"));
   75                 addr = (char *)(uintptr_t)list->ds_addr + offset;
   76                 ds_len = list->ds_len - offset;
   77                 offset = 0;
   78                 if (ds_len > length)
   79                         ds_len = length;
   80                 length -= ds_len;
   81                 KASSERT((ds_len != 0), ("Segment length is zero"));
   82                 error = _bus_dmamap_load_buffer(dmat, map, addr, ds_len, pmap,
   83                     flags, NULL, nsegs);
   84                 if (error)
   85                         break;
   86         }
   87         return (error);
   88 }
   89 
   90 /*
   91  * Load a list of physical addresses.
   92  */
   93 static int
   94 _bus_dmamap_load_plist(bus_dma_tag_t dmat, bus_dmamap_t map,
   95     bus_dma_segment_t *list, int sglist_cnt, int *nsegs, int flags)
   96 {
   97         int error;
   98 
   99         error = 0;
  100         for (; sglist_cnt > 0; sglist_cnt--, list++) {
  101                 error = _bus_dmamap_load_phys(dmat, map,
  102                     (vm_paddr_t)list->ds_addr, list->ds_len, flags, NULL,
  103                     nsegs);
  104                 if (error)
  105                         break;
  106         }
  107         return (error);
  108 }
  109 
  110 /*
  111  * Load an mbuf chain.
  112  */
  113 static int
  114 _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
  115     struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags)
  116 {
  117         struct mbuf *m;
  118         int error;
  119 
  120         error = 0;
  121         for (m = m0; m != NULL && error == 0; m = m->m_next) {
  122                 if (m->m_len > 0) {
  123                         error = _bus_dmamap_load_buffer(dmat, map, m->m_data,
  124                             m->m_len, kernel_pmap, flags | BUS_DMA_LOAD_MBUF,
  125                             segs, nsegs);
  126                 }
  127         }
  128         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  129             __func__, dmat, flags, error, *nsegs);
  130         return (error);
  131 }
  132 
  133 /*
  134  * Load from block io.
  135  */
  136 static int
  137 _bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio,
  138     int *nsegs, int flags)
  139 {
  140 
  141         if ((bio->bio_flags & BIO_VLIST) != 0) {
  142                 bus_dma_segment_t *segs = (bus_dma_segment_t *)bio->bio_data;
  143                 return (_bus_dmamap_load_vlist(dmat, map, segs, bio->bio_ma_n,
  144                     kernel_pmap, nsegs, flags, bio->bio_ma_offset,
  145                     bio->bio_bcount));
  146         }
  147 
  148         if ((bio->bio_flags & BIO_UNMAPPED) != 0)
  149                 return (_bus_dmamap_load_ma(dmat, map, bio->bio_ma,
  150                     bio->bio_bcount, bio->bio_ma_offset, flags, NULL, nsegs));
  151 
  152         return (_bus_dmamap_load_buffer(dmat, map, bio->bio_data,
  153             bio->bio_bcount, kernel_pmap, flags, NULL, nsegs));
  154 }
  155 
  156 int
  157 bus_dmamap_load_ma_triv(bus_dma_tag_t dmat, bus_dmamap_t map,
  158     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
  159     bus_dma_segment_t *segs, int *segp)
  160 {
  161         vm_paddr_t paddr;
  162         bus_size_t len;
  163         int error, i;
  164 
  165         error = 0;
  166         for (i = 0; tlen > 0; i++, tlen -= len) {
  167                 len = min(PAGE_SIZE - ma_offs, tlen);
  168                 paddr = VM_PAGE_TO_PHYS(ma[i]) + ma_offs;
  169                 error = _bus_dmamap_load_phys(dmat, map, paddr, len,
  170                     flags, segs, segp);
  171                 if (error != 0)
  172                         break;
  173                 ma_offs = 0;
  174         }
  175         return (error);
  176 }
  177 
  178 /*
  179  * Load a cam control block.
  180  */
  181 static int
  182 _bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb,
  183                     int *nsegs, int flags)
  184 {
  185         struct ccb_hdr *ccb_h;
  186         void *data_ptr;
  187         int error;
  188         uint32_t dxfer_len;
  189         uint16_t sglist_cnt;
  190 
  191         error = 0;
  192         ccb_h = &ccb->ccb_h;
  193         switch (ccb_h->func_code) {
  194         case XPT_SCSI_IO: {
  195                 struct ccb_scsiio *csio;
  196 
  197                 csio = &ccb->csio;
  198                 data_ptr = csio->data_ptr;
  199                 dxfer_len = csio->dxfer_len;
  200                 sglist_cnt = csio->sglist_cnt;
  201                 break;
  202         }
  203         case XPT_CONT_TARGET_IO: {
  204                 struct ccb_scsiio *ctio;
  205 
  206                 ctio = &ccb->ctio;
  207                 data_ptr = ctio->data_ptr;
  208                 dxfer_len = ctio->dxfer_len;
  209                 sglist_cnt = ctio->sglist_cnt;
  210                 break;
  211         }
  212         case XPT_ATA_IO: {
  213                 struct ccb_ataio *ataio;
  214 
  215                 ataio = &ccb->ataio;
  216                 data_ptr = ataio->data_ptr;
  217                 dxfer_len = ataio->dxfer_len;
  218                 sglist_cnt = 0;
  219                 break;
  220         }
  221         default:
  222                 panic("_bus_dmamap_load_ccb: Unsupported func code %d",
  223                     ccb_h->func_code);
  224         }
  225 
  226         switch ((ccb_h->flags & CAM_DATA_MASK)) {
  227         case CAM_DATA_VADDR:
  228                 error = _bus_dmamap_load_buffer(dmat, map, data_ptr, dxfer_len,
  229                     kernel_pmap, flags, NULL, nsegs);
  230                 break;
  231         case CAM_DATA_PADDR:
  232                 error = _bus_dmamap_load_phys(dmat, map,
  233                     (vm_paddr_t)(uintptr_t)data_ptr, dxfer_len, flags, NULL,
  234                     nsegs);
  235                 break;
  236         case CAM_DATA_SG:
  237                 error = _bus_dmamap_load_vlist(dmat, map,
  238                     (bus_dma_segment_t *)data_ptr, sglist_cnt, kernel_pmap,
  239                     nsegs, flags, 0, dxfer_len);
  240                 break;
  241         case CAM_DATA_SG_PADDR:
  242                 error = _bus_dmamap_load_plist(dmat, map,
  243                     (bus_dma_segment_t *)data_ptr, sglist_cnt, nsegs, flags);
  244                 break;
  245         case CAM_DATA_BIO:
  246                 error = _bus_dmamap_load_bio(dmat, map, (struct bio *)data_ptr,
  247                     nsegs, flags);
  248                 break;
  249         default:
  250                 panic("_bus_dmamap_load_ccb: flags 0x%X unimplemented",
  251                     ccb_h->flags);
  252         }
  253         return (error);
  254 }
  255 
  256 /*
  257  * Load a uio.
  258  */
  259 static int
  260 _bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
  261     int *nsegs, int flags)
  262 {
  263         bus_size_t resid;
  264         bus_size_t minlen;
  265         struct iovec *iov;
  266         pmap_t pmap;
  267         caddr_t addr;
  268         int error, i;
  269 
  270         if (uio->uio_segflg == UIO_USERSPACE) {
  271                 KASSERT(uio->uio_td != NULL,
  272                         ("bus_dmamap_load_uio: USERSPACE but no proc"));
  273                 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
  274         } else
  275                 pmap = kernel_pmap;
  276         resid = uio->uio_resid;
  277         iov = uio->uio_iov;
  278         error = 0;
  279 
  280         for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
  281                 /*
  282                  * Now at the first iovec to load.  Load each iovec
  283                  * until we have exhausted the residual count.
  284                  */
  285 
  286                 addr = (caddr_t) iov[i].iov_base;
  287                 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
  288                 if (minlen > 0) {
  289                         error = _bus_dmamap_load_buffer(dmat, map, addr,
  290                             minlen, pmap, flags, NULL, nsegs);
  291                         resid -= minlen;
  292                 }
  293         }
  294 
  295         return (error);
  296 }
  297 
  298 /*
  299  * Map the buffer buf into bus space using the dmamap map.
  300  */
  301 int
  302 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
  303     bus_size_t buflen, bus_dmamap_callback_t *callback,
  304     void *callback_arg, int flags)
  305 {
  306         bus_dma_segment_t *segs;
  307         struct memdesc mem;
  308         int error;
  309         int nsegs;
  310 
  311         if ((flags & BUS_DMA_NOWAIT) == 0) {
  312                 mem = memdesc_vaddr(buf, buflen);
  313                 _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg);
  314         }
  315 
  316         nsegs = -1;
  317         error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, kernel_pmap,
  318             flags, NULL, &nsegs);
  319         nsegs++;
  320 
  321         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  322             __func__, dmat, flags, error, nsegs);
  323 
  324         if (error == EINPROGRESS)
  325                 return (error);
  326 
  327         segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
  328         if (error)
  329                 (*callback)(callback_arg, segs, 0, error);
  330         else
  331                 (*callback)(callback_arg, segs, nsegs, 0);
  332 
  333         /*
  334          * Return ENOMEM to the caller so that it can pass it up the stack.
  335          * This error only happens when NOWAIT is set, so deferral is disabled.
  336          */
  337         if (error == ENOMEM)
  338                 return (error);
  339 
  340         return (0);
  341 }
  342 
  343 int
  344 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
  345     bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
  346 {
  347         bus_dma_segment_t *segs;
  348         int nsegs, error;
  349 
  350         M_ASSERTPKTHDR(m0);
  351 
  352         flags |= BUS_DMA_NOWAIT;
  353         nsegs = -1;
  354         error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags);
  355         ++nsegs;
  356 
  357         segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
  358         if (error)
  359                 (*callback)(callback_arg, segs, 0, 0, error);
  360         else
  361                 (*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error);
  362 
  363         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  364             __func__, dmat, flags, error, nsegs);
  365         return (error);
  366 }
  367 
  368 int
  369 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
  370     bus_dma_segment_t *segs, int *nsegs, int flags)
  371 {
  372         int error;
  373 
  374         flags |= BUS_DMA_NOWAIT;
  375         *nsegs = -1;
  376         error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags);
  377         ++*nsegs;
  378         _bus_dmamap_complete(dmat, map, segs, *nsegs, error);
  379         return (error);
  380 }
  381 
  382 int
  383 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
  384     bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
  385 {
  386         bus_dma_segment_t *segs;
  387         int nsegs, error;
  388 
  389         flags |= BUS_DMA_NOWAIT;
  390         nsegs = -1;
  391         error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags);
  392         nsegs++;
  393 
  394         segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
  395         if (error)
  396                 (*callback)(callback_arg, segs, 0, 0, error);
  397         else
  398                 (*callback)(callback_arg, segs, nsegs, uio->uio_resid, error);
  399 
  400         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  401             __func__, dmat, flags, error, nsegs);
  402         return (error);
  403 }
  404 
  405 int
  406 bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb,
  407                     bus_dmamap_callback_t *callback, void *callback_arg,
  408                     int flags)
  409 {
  410         bus_dma_segment_t *segs;
  411         struct ccb_hdr *ccb_h;
  412         struct memdesc mem;
  413         int error;
  414         int nsegs;
  415 
  416         ccb_h = &ccb->ccb_h;
  417         if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) {
  418                 callback(callback_arg, NULL, 0, 0);
  419                 return (0);
  420         }
  421         if ((flags & BUS_DMA_NOWAIT) == 0) {
  422                 mem = memdesc_ccb(ccb);
  423                 _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg);
  424         }
  425         nsegs = -1;
  426         error = _bus_dmamap_load_ccb(dmat, map, ccb, &nsegs, flags);
  427         nsegs++;
  428 
  429         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  430             __func__, dmat, flags, error, nsegs);
  431 
  432         if (error == EINPROGRESS)
  433                 return (error);
  434 
  435         segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
  436         if (error)
  437                 (*callback)(callback_arg, segs, 0, error);
  438         else
  439                 (*callback)(callback_arg, segs, nsegs, error);
  440         /*
  441          * Return ENOMEM to the caller so that it can pass it up the stack.
  442          * This error only happens when NOWAIT is set, so deferral is disabled.
  443          */
  444         if (error == ENOMEM)
  445                 return (error);
  446 
  447         return (0);
  448 }
  449 
  450 int
  451 bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio,
  452                     bus_dmamap_callback_t *callback, void *callback_arg,
  453                     int flags)
  454 {
  455         bus_dma_segment_t *segs;
  456         struct memdesc mem;
  457         int error;
  458         int nsegs;
  459 
  460         if ((flags & BUS_DMA_NOWAIT) == 0) {
  461                 mem = memdesc_bio(bio);
  462                 _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg);
  463         }
  464         nsegs = -1;
  465         error = _bus_dmamap_load_bio(dmat, map, bio, &nsegs, flags);
  466         nsegs++;
  467 
  468         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  469             __func__, dmat, flags, error, nsegs);
  470 
  471         if (error == EINPROGRESS)
  472                 return (error);
  473 
  474         segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
  475         if (error)
  476                 (*callback)(callback_arg, segs, 0, error);
  477         else
  478                 (*callback)(callback_arg, segs, nsegs, error);
  479         /*
  480          * Return ENOMEM to the caller so that it can pass it up the stack.
  481          * This error only happens when NOWAIT is set, so deferral is disabled.
  482          */
  483         if (error == ENOMEM)
  484                 return (error);
  485 
  486         return (0);
  487 }
  488 
  489 int
  490 bus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map,
  491     struct memdesc *mem, bus_dmamap_callback_t *callback,
  492     void *callback_arg, int flags)
  493 {
  494         bus_dma_segment_t *segs;
  495         int error;
  496         int nsegs;
  497 
  498         if ((flags & BUS_DMA_NOWAIT) == 0)
  499                 _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg);
  500 
  501         nsegs = -1;
  502         error = 0;
  503         switch (mem->md_type) {
  504         case MEMDESC_VADDR:
  505                 error = _bus_dmamap_load_buffer(dmat, map, mem->u.md_vaddr,
  506                     mem->md_opaque, kernel_pmap, flags, NULL, &nsegs);
  507                 break;
  508         case MEMDESC_PADDR:
  509                 error = _bus_dmamap_load_phys(dmat, map, mem->u.md_paddr,
  510                     mem->md_opaque, flags, NULL, &nsegs);
  511                 break;
  512         case MEMDESC_VLIST:
  513                 error = _bus_dmamap_load_vlist(dmat, map, mem->u.md_list,
  514                     mem->md_opaque, kernel_pmap, &nsegs, flags, 0, SIZE_T_MAX);
  515                 break;
  516         case MEMDESC_PLIST:
  517                 error = _bus_dmamap_load_plist(dmat, map, mem->u.md_list,
  518                     mem->md_opaque, &nsegs, flags);
  519                 break;
  520         case MEMDESC_BIO:
  521                 error = _bus_dmamap_load_bio(dmat, map, mem->u.md_bio,
  522                     &nsegs, flags);
  523                 break;
  524         case MEMDESC_UIO:
  525                 error = _bus_dmamap_load_uio(dmat, map, mem->u.md_uio,
  526                     &nsegs, flags);
  527                 break;
  528         case MEMDESC_MBUF:
  529                 error = _bus_dmamap_load_mbuf_sg(dmat, map, mem->u.md_mbuf,
  530                     NULL, &nsegs, flags);
  531                 break;
  532         case MEMDESC_CCB:
  533                 error = _bus_dmamap_load_ccb(dmat, map, mem->u.md_ccb, &nsegs,
  534                     flags);
  535                 break;
  536         }
  537         nsegs++;
  538 
  539         CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
  540             __func__, dmat, flags, error, nsegs);
  541 
  542         if (error == EINPROGRESS)
  543                 return (error);
  544 
  545         segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
  546         if (error)
  547                 (*callback)(callback_arg, segs, 0, error);
  548         else
  549                 (*callback)(callback_arg, segs, nsegs, 0);
  550 
  551         /*
  552          * Return ENOMEM to the caller so that it can pass it up the stack.
  553          * This error only happens when NOWAIT is set, so deferral is disabled.
  554          */
  555         if (error == ENOMEM)
  556                 return (error);
  557 
  558         return (0);
  559 }
Cache object: 75e865e0b1930802acdbb6486b9e57ee
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/subr_bus_dma.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_bus_dma.c
