The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/busdma_dmar.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2013 The FreeBSD Foundation
    3  * All rights reserved.
    4  *
    5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    6  * under sponsorship from the FreeBSD Foundation.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/10.1/sys/x86/iommu/busdma_dmar.c 263747 2014-03-25 20:17:57Z kib $");
   32 
   33 #include <sys/param.h>
   34 #include <sys/systm.h>
   35 #include <sys/malloc.h>
   36 #include <sys/bus.h>
   37 #include <sys/conf.h>
   38 #include <sys/interrupt.h>
   39 #include <sys/kernel.h>
   40 #include <sys/ktr.h>
   41 #include <sys/lock.h>
   42 #include <sys/proc.h>
   43 #include <sys/memdesc.h>
   44 #include <sys/mutex.h>
   45 #include <sys/sysctl.h>
   46 #include <sys/rman.h>
   47 #include <sys/taskqueue.h>
   48 #include <sys/tree.h>
   49 #include <sys/uio.h>
   50 #include <dev/pci/pcireg.h>
   51 #include <dev/pci/pcivar.h>
   52 #include <vm/vm.h>
   53 #include <vm/vm_extern.h>
   54 #include <vm/vm_kern.h>
   55 #include <vm/vm_object.h>
   56 #include <vm/vm_page.h>
   57 #include <vm/vm_map.h>
   58 #include <machine/atomic.h>
   59 #include <machine/bus.h>
   60 #include <machine/md_var.h>
   61 #include <machine/specialreg.h>
   62 #include <x86/include/busdma_impl.h>
   63 #include <x86/iommu/intel_reg.h>
   64 #include <x86/iommu/busdma_dmar.h>
   65 #include <x86/iommu/intel_dmar.h>
   66 
   67 /*
   68  * busdma_dmar.c, the implementation of the busdma(9) interface using
   69  * DMAR units from Intel VT-d.
   70  */
   71 
   72 static bool
   73 dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
   74 {
   75         char str[128], *env;
   76 
   77         snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d.bounce",
   78             domain, bus, slot, func);
   79         env = getenv(str);
   80         if (env == NULL)
   81                 return (false);
   82         freeenv(env);
   83         return (true);
   84 }
   85 
   86 /*
   87  * Given original device, find the requester ID that will be seen by
   88  * the DMAR unit and used for page table lookup.  PCI bridges may take
   89  * ownership of transactions from downstream devices, so it may not be
   90  * the same as the BSF of the target device.  In those cases, all
   91  * devices downstream of the bridge must share a single mapping
   92  * domain, and must collectively be assigned to use either DMAR or
   93  * bounce mapping.
   94  */
   95 static device_t
   96 dmar_get_requester(device_t dev, int *bus, int *slot, int *func)
   97 {
   98         devclass_t pci_class;
   99         device_t pci, pcib, requester;
  100         int cap_offset;
  101 
  102         pci_class = devclass_find("pci");
  103         requester = dev;
  104 
  105         *bus = pci_get_bus(dev);
  106         *slot = pci_get_slot(dev);
  107         *func = pci_get_function(dev);
  108 
  109         /*
  110          * Walk the bridge hierarchy from the target device to the
  111          * host port to find the translating bridge nearest the DMAR
  112          * unit.
  113          */
  114         for (;;) {
  115                 pci = device_get_parent(dev);
  116                 KASSERT(pci != NULL, ("NULL parent for pci%d:%d:%d:%d",
  117                     pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
  118                     pci_get_function(dev)));
  119                 KASSERT(device_get_devclass(pci) == pci_class,
  120                     ("Non-pci parent for pci%d:%d:%d:%d",
  121                     pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
  122                     pci_get_function(dev)));
  123 
  124                 pcib = device_get_parent(pci);
  125                 KASSERT(pcib != NULL, ("NULL bridge for pci%d:%d:%d:%d",
  126                     pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
  127                     pci_get_function(dev)));
  128 
  129                 /*
  130                  * The parent of our "bridge" isn't another PCI bus,
  131                  * so pcib isn't a PCI->PCI bridge but rather a host
  132                  * port, and the requester ID won't be translated
  133                  * further.
  134                  */
  135                 if (device_get_devclass(device_get_parent(pcib)) != pci_class)
  136                         break;
  137 
  138                 if (pci_find_cap(dev, PCIY_EXPRESS, &cap_offset) != 0) {
  139                         /*
  140                          * Device is not PCIe, it cannot be seen as a
  141                          * requester by DMAR unit.
  142                          */
  143                         requester = pcib;
  144 
  145                         /* Check whether the bus above is PCIe. */
  146                         if (pci_find_cap(pcib, PCIY_EXPRESS,
  147                             &cap_offset) == 0) {
  148                                 /*
  149                                  * The current device is not PCIe, but
  150                                  * the bridge above it is.  This is a
  151                                  * PCIe->PCI bridge.  Assume that the
  152                                  * requester ID will be the secondary
  153                                  * bus number with slot and function
  154                                  * set to zero.
  155                                  *
  156                                  * XXX: Doesn't handle the case where
  157                                  * the bridge is PCIe->PCI-X, and the
  158                                  * bridge will only take ownership of
  159                                  * requests in some cases.  We should
  160                                  * provide context entries with the
  161                                  * same page tables for taken and
  162                                  * non-taken transactions.
  163                                  */
  164                                 *bus = pci_get_bus(dev);
  165                                 *slot = *func = 0;
  166                         } else {
  167                                 /*
  168                                  * Neither the device nor the bridge
  169                                  * above it are PCIe.  This is a
  170                                  * conventional PCI->PCI bridge, which
  171                                  * will use the bridge's BSF as the
  172                                  * requester ID.
  173                                  */
  174                                 *bus = pci_get_bus(pcib);
  175                                 *slot = pci_get_slot(pcib);
  176                                 *func = pci_get_function(pcib);
  177                         }
  178                 }
  179                 /*
  180                  * Do not stop the loop even if the target device is
  181                  * PCIe, because it is possible (but unlikely) to have
  182                  * a PCI->PCIe bridge somewhere in the hierarchy.
  183                  */
  184 
  185                 dev = pcib;
  186         }
  187         return (requester);
  188 }
  189 
  190 struct dmar_ctx *
  191 dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
  192 {
  193         device_t requester;
  194         struct dmar_ctx *ctx;
  195         bool disabled;
  196         int bus, slot, func;
  197 
  198         requester = dmar_get_requester(dev, &bus, &slot, &func);
  199 
  200         /*
  201          * If the user requested the IOMMU disabled for the device, we
  202          * cannot disable the DMAR, due to possibility of other
  203          * devices on the same DMAR still requiring translation.
  204          * Instead provide the identity mapping for the device
  205          * context.
  206          */
  207         disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(dev), bus,
  208             slot, func);
  209         ctx = dmar_get_ctx(dmar, requester, bus, slot, func, disabled, rmrr);
  210         if (ctx == NULL)
  211                 return (NULL);
  212         if (disabled) {
  213                 /*
  214                  * Keep the first reference on context, release the
  215                  * later refs.
  216                  */
  217                 DMAR_LOCK(dmar);
  218                 if ((ctx->flags & DMAR_CTX_DISABLED) == 0) {
  219                         ctx->flags |= DMAR_CTX_DISABLED;
  220                         DMAR_UNLOCK(dmar);
  221                 } else {
  222                         dmar_free_ctx_locked(dmar, ctx);
  223                 }
  224                 ctx = NULL;
  225         }
  226         return (ctx);
  227 }
  228 
  229 bus_dma_tag_t
  230 dmar_get_dma_tag(device_t dev, device_t child)
  231 {
  232         struct dmar_unit *dmar;
  233         struct dmar_ctx *ctx;
  234         bus_dma_tag_t res;
  235 
  236         dmar = dmar_find(child);
  237         /* Not in scope of any DMAR ? */
  238         if (dmar == NULL)
  239                 return (NULL);
  240         dmar_quirks_pre_use(dmar);
  241         dmar_instantiate_rmrr_ctxs(dmar);
  242 
  243         ctx = dmar_instantiate_ctx(dmar, child, false);
  244         res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag;
  245         return (res);
  246 }
  247 
  248 static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map");
  249 
  250 static void dmar_bus_schedule_dmamap(struct dmar_unit *unit,
  251     struct bus_dmamap_dmar *map);
  252 
  253 static int
  254 dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  255     bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
  256     bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
  257     int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  258     void *lockfuncarg, bus_dma_tag_t *dmat)
  259 {
  260         struct bus_dma_tag_dmar *newtag, *oldtag;
  261         int error;
  262 
  263         *dmat = NULL;
  264         error = common_bus_dma_tag_create(parent != NULL ?
  265             &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment,
  266             boundary, lowaddr, highaddr, filter, filterarg, maxsize,
  267             nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
  268             sizeof(struct bus_dma_tag_dmar), (void **)&newtag);
  269         if (error != 0)
  270                 goto out;
  271 
  272         oldtag = (struct bus_dma_tag_dmar *)parent;
  273         newtag->common.impl = &bus_dma_dmar_impl;
  274         newtag->ctx = oldtag->ctx;
  275         newtag->owner = oldtag->owner;
  276 
  277         *dmat = (bus_dma_tag_t)newtag;
  278 out:
  279         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  280             __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
  281             error);
  282         return (error);
  283 }
  284 
  285 static int
  286 dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
  287 {
  288         struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent;
  289         int error;
  290 
  291         error = 0;
  292         dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1;
  293 
  294         if (dmat != NULL) {
  295                 if (dmat->map_count != 0) {
  296                         error = EBUSY;
  297                         goto out;
  298                 }
  299                 while (dmat != NULL) {
  300                         parent = (struct bus_dma_tag_dmar *)dmat->common.parent;
  301                         if (atomic_fetchadd_int(&dmat->common.ref_count, -1) ==
  302                             1) {
  303                                 if (dmat == &dmat->ctx->ctx_tag)
  304                                         dmar_free_ctx(dmat->ctx);
  305                                 free(dmat->segments, M_DMAR_DMAMAP);
  306                                 free(dmat, M_DEVBUF);
  307                                 dmat = parent;
  308                         } else
  309                                 dmat = NULL;
  310                 }
  311         }
  312 out:
  313         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
  314         return (error);
  315 }
  316 
  317 static int
  318 dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  319 {
  320         struct bus_dma_tag_dmar *tag;
  321         struct bus_dmamap_dmar *map;
  322 
  323         tag = (struct bus_dma_tag_dmar *)dmat;
  324         map = malloc(sizeof(*map), M_DMAR_DMAMAP, M_NOWAIT | M_ZERO);
  325         if (map == NULL) {
  326                 *mapp = NULL;
  327                 return (ENOMEM);
  328         }
  329         if (tag->segments == NULL) {
  330                 tag->segments = malloc(sizeof(bus_dma_segment_t) *
  331                     tag->common.nsegments, M_DMAR_DMAMAP, M_NOWAIT);
  332                 if (tag->segments == NULL) {
  333                         free(map, M_DMAR_DMAMAP);
  334                         *mapp = NULL;
  335                         return (ENOMEM);
  336                 }
  337         }
  338         TAILQ_INIT(&map->map_entries);
  339         map->tag = tag;
  340         map->locked = true;
  341         map->cansleep = false;
  342         tag->map_count++;
  343         *mapp = (bus_dmamap_t)map;
  344 
  345         return (0);
  346 }
  347 
  348 static int
  349 dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
  350 {
  351         struct bus_dma_tag_dmar *tag;
  352         struct bus_dmamap_dmar *map;
  353 
  354         tag = (struct bus_dma_tag_dmar *)dmat;
  355         map = (struct bus_dmamap_dmar *)map1;
  356         if (map != NULL) {
  357                 DMAR_CTX_LOCK(tag->ctx);
  358                 if (!TAILQ_EMPTY(&map->map_entries)) {
  359                         DMAR_CTX_UNLOCK(tag->ctx);
  360                         return (EBUSY);
  361                 }
  362                 DMAR_CTX_UNLOCK(tag->ctx);
  363                 free(map, M_DMAR_DMAMAP);
  364         }
  365         tag->map_count--;
  366         return (0);
  367 }
  368 
  369 
  370 static int
  371 dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
  372     bus_dmamap_t *mapp)
  373 {
  374         struct bus_dma_tag_dmar *tag;
  375         struct bus_dmamap_dmar *map;
  376         int error, mflags;
  377         vm_memattr_t attr;
  378 
  379         error = dmar_bus_dmamap_create(dmat, flags, mapp);
  380         if (error != 0)
  381                 return (error);
  382 
  383         mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
  384         mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
  385         attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
  386             VM_MEMATTR_DEFAULT;
  387 
  388         tag = (struct bus_dma_tag_dmar *)dmat;
  389         map = (struct bus_dmamap_dmar *)*mapp;
  390 
  391         if (tag->common.maxsize < PAGE_SIZE &&
  392             tag->common.alignment <= tag->common.maxsize &&
  393             attr == VM_MEMATTR_DEFAULT) {
  394                 *vaddr = malloc(tag->common.maxsize, M_DEVBUF, mflags);
  395                 map->flags |= BUS_DMAMAP_DMAR_MALLOC;
  396         } else {
  397                 *vaddr = (void *)kmem_alloc_attr(kernel_arena,
  398                     tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR,
  399                     attr);
  400                 map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC;
  401         }
  402         if (*vaddr == NULL) {
  403                 dmar_bus_dmamap_destroy(dmat, *mapp);
  404                 *mapp = NULL;
  405                 return (ENOMEM);
  406         }
  407         return (0);
  408 }
  409 
  410 static void
  411 dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
  412 {
  413         struct bus_dma_tag_dmar *tag;
  414         struct bus_dmamap_dmar *map;
  415 
  416         tag = (struct bus_dma_tag_dmar *)dmat;
  417         map = (struct bus_dmamap_dmar *)map1;
  418 
  419         if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) {
  420                 free(vaddr, M_DEVBUF);
  421                 map->flags &= ~BUS_DMAMAP_DMAR_MALLOC;
  422         } else {
  423                 KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0,
  424                     ("dmar_bus_dmamem_free for non alloced map %p", map));
  425                 kmem_free(kernel_arena, (vm_offset_t)vaddr, tag->common.maxsize);
  426                 map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC;
  427         }
  428 
  429         dmar_bus_dmamap_destroy(dmat, map1);
  430 }
  431 
  432 static int
  433 dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag,
  434     struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
  435     int flags, bus_dma_segment_t *segs, int *segp,
  436     struct dmar_map_entries_tailq *unroll_list)
  437 {
  438         struct dmar_ctx *ctx;
  439         struct dmar_map_entry *entry;
  440         dmar_gaddr_t size;
  441         bus_size_t buflen1;
  442         int error, idx, gas_flags, seg;
  443 
  444         if (segs == NULL)
  445                 segs = tag->segments;
  446         ctx = tag->ctx;
  447         seg = *segp;
  448         error = 0;
  449         idx = 0;
  450         while (buflen > 0) {
  451                 seg++;
  452                 if (seg >= tag->common.nsegments) {
  453                         error = EFBIG;
  454                         break;
  455                 }
  456                 buflen1 = buflen > tag->common.maxsegsz ?
  457                     tag->common.maxsegsz : buflen;
  458                 buflen -= buflen1;
  459                 size = round_page(offset + buflen1);
  460 
  461                 /*
  462                  * (Too) optimistically allow split if there are more
  463                  * then one segments left.
  464                  */
  465                 gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0;
  466                 if (seg + 1 < tag->common.nsegments)
  467                         gas_flags |= DMAR_GM_CANSPLIT;
  468 
  469                 error = dmar_gas_map(ctx, &tag->common, size,
  470                     DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE,
  471                     gas_flags, ma + idx, &entry);
  472                 if (error != 0)
  473                         break;
  474                 if ((gas_flags & DMAR_GM_CANSPLIT) != 0) {
  475                         KASSERT(size >= entry->end - entry->start,
  476                             ("split increased entry size %jx %jx %jx",
  477                             (uintmax_t)size, (uintmax_t)entry->start,
  478                             (uintmax_t)entry->end));
  479                         size = entry->end - entry->start;
  480                         if (buflen1 > size)
  481                                 buflen1 = size;
  482                 } else {
  483                         KASSERT(entry->end - entry->start == size,
  484                             ("no split allowed %jx %jx %jx",
  485                             (uintmax_t)size, (uintmax_t)entry->start,
  486                             (uintmax_t)entry->end));
  487                 }
  488 
  489                 KASSERT(((entry->start + offset) & (tag->common.alignment - 1))
  490                     == 0,
  491                     ("alignment failed: ctx %p start 0x%jx offset %x "
  492                     "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
  493                     (uintmax_t)tag->common.alignment));
  494                 KASSERT(entry->end <= tag->common.lowaddr ||
  495                     entry->start >= tag->common.highaddr,
  496                     ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
  497                     "lowaddr 0x%jx highaddr 0x%jx", ctx,
  498                     (uintmax_t)entry->start, (uintmax_t)entry->end,
  499                     (uintmax_t)tag->common.lowaddr,
  500                     (uintmax_t)tag->common.highaddr));
  501                 KASSERT(dmar_test_boundary(entry->start, entry->end -
  502                     entry->start, tag->common.boundary),
  503                     ("boundary failed: ctx %p start 0x%jx end 0x%jx "
  504                     "boundary 0x%jx", ctx, (uintmax_t)entry->start,
  505                     (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
  506                 KASSERT(buflen1 <= tag->common.maxsegsz,
  507                     ("segment too large: ctx %p start 0x%jx end 0x%jx "
  508                     "maxsegsz 0x%jx", ctx, (uintmax_t)entry->start,
  509                     (uintmax_t)entry->end, (uintmax_t)tag->common.maxsegsz));
  510 
  511                 DMAR_CTX_LOCK(ctx);
  512                 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
  513                 entry->flags |= DMAR_MAP_ENTRY_MAP;
  514                 DMAR_CTX_UNLOCK(ctx);
  515                 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);
  516 
  517                 segs[seg].ds_addr = entry->start + offset;
  518                 segs[seg].ds_len = buflen1;
  519 
  520                 idx += OFF_TO_IDX(trunc_page(offset + buflen1));
  521                 offset += buflen1;
  522                 offset &= DMAR_PAGE_MASK;
  523         }
  524         if (error == 0)
  525                 *segp = seg;
  526         return (error);
  527 }
  528 
  529 static int
  530 dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag,
  531     struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
  532     int flags, bus_dma_segment_t *segs, int *segp)
  533 {
  534         struct dmar_ctx *ctx;
  535         struct dmar_map_entry *entry, *entry1;
  536         struct dmar_map_entries_tailq unroll_list;
  537         int error;
  538 
  539         ctx = tag->ctx;
  540         atomic_add_long(&ctx->loads, 1);
  541 
  542         TAILQ_INIT(&unroll_list);
  543         error = dmar_bus_dmamap_load_something1(tag, map, ma, offset,
  544             buflen, flags, segs, segp, &unroll_list);
  545         if (error != 0) {
  546                 /*
  547                  * The busdma interface does not allow us to report
  548                  * partial buffer load, so unfortunately we have to
  549                  * revert all work done.
  550                  */
  551                 DMAR_CTX_LOCK(ctx);
  552                 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link,
  553                     entry1) {
  554                         /*
  555                          * No entries other than what we have created
  556                          * during the failed run might have been
  557                          * inserted there in between, since we own ctx
  558                          * pglock.
  559                          */
  560                         TAILQ_REMOVE(&map->map_entries, entry, dmamap_link);
  561                         TAILQ_REMOVE(&unroll_list, entry, unroll_link);
  562                         TAILQ_INSERT_TAIL(&ctx->unload_entries, entry,
  563                             dmamap_link);
  564                 }
  565                 DMAR_CTX_UNLOCK(ctx);
  566                 taskqueue_enqueue(ctx->dmar->delayed_taskqueue,
  567                     &ctx->unload_task);
  568         }
  569 
  570         if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
  571             !map->cansleep)
  572                 error = EINPROGRESS;
  573         if (error == EINPROGRESS)
  574                 dmar_bus_schedule_dmamap(ctx->dmar, map);
  575         return (error);
  576 }
  577 
  578 static int
  579 dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
  580     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
  581     bus_dma_segment_t *segs, int *segp)
  582 {
  583         struct bus_dma_tag_dmar *tag;
  584         struct bus_dmamap_dmar *map;
  585 
  586         tag = (struct bus_dma_tag_dmar *)dmat;
  587         map = (struct bus_dmamap_dmar *)map1;
  588         return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
  589             flags, segs, segp));
  590 }
  591 
  592 static int
  593 dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
  594     vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
  595     int *segp)
  596 {
  597         struct bus_dma_tag_dmar *tag;
  598         struct bus_dmamap_dmar *map;
  599         vm_page_t *ma;
  600         vm_paddr_t pstart, pend;
  601         int error, i, ma_cnt, offset;
  602 
  603         tag = (struct bus_dma_tag_dmar *)dmat;
  604         map = (struct bus_dmamap_dmar *)map1;
  605         pstart = trunc_page(buf);
  606         pend = round_page(buf + buflen);
  607         offset = buf & PAGE_MASK;
  608         ma_cnt = OFF_TO_IDX(pend - pstart);
  609         ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
  610             M_WAITOK : M_NOWAIT);
  611         if (ma == NULL)
  612                 return (ENOMEM);
  613         for (i = 0; i < ma_cnt; i++)
  614                 ma[i] = PHYS_TO_VM_PAGE(pstart + i * PAGE_SIZE);
  615         error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
  616             flags, segs, segp);
  617         free(ma, M_DEVBUF);
  618         return (error);
  619 }
  620 
  621 static int
  622 dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
  623     bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
  624     int *segp)
  625 {
  626         struct bus_dma_tag_dmar *tag;
  627         struct bus_dmamap_dmar *map;
  628         vm_page_t *ma, fma;
  629         vm_paddr_t pstart, pend, paddr;
  630         int error, i, ma_cnt, offset;
  631 
  632         tag = (struct bus_dma_tag_dmar *)dmat;
  633         map = (struct bus_dmamap_dmar *)map1;
  634         pstart = trunc_page((vm_offset_t)buf);
  635         pend = round_page((vm_offset_t)buf + buflen);
  636         offset = (vm_offset_t)buf & PAGE_MASK;
  637         ma_cnt = OFF_TO_IDX(pend - pstart);
  638         ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
  639             M_WAITOK : M_NOWAIT);
  640         if (ma == NULL)
  641                 return (ENOMEM);
  642         if (dumping) {
  643                 /*
  644                  * If dumping, do not attempt to call
  645                  * PHYS_TO_VM_PAGE() at all.  It may return non-NULL
  646                  * but the vm_page returned might be not initialized,
  647                  * e.g. for the kernel itself.
  648                  */
  649                 KASSERT(pmap == kernel_pmap, ("non-kernel address write"));
  650                 fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF,
  651                     M_ZERO | (map->cansleep ? M_WAITOK : M_NOWAIT));
  652                 if (fma == NULL) {
  653                         free(ma, M_DEVBUF);
  654                         return (ENOMEM);
  655                 }
  656                 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
  657                         paddr = pmap_kextract(pstart);
  658                         vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
  659                         ma[i] = &fma[i];
  660                 }
  661         } else {
  662                 fma = NULL;
  663                 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
  664                         if (pmap == kernel_pmap)
  665                                 paddr = pmap_kextract(pstart);
  666                         else
  667                                 paddr = pmap_extract(pmap, pstart);
  668                         ma[i] = PHYS_TO_VM_PAGE(paddr);
  669                         KASSERT(VM_PAGE_TO_PHYS(ma[i]) == paddr,
  670                             ("PHYS_TO_VM_PAGE failed %jx %jx m %p",
  671                             (uintmax_t)paddr, (uintmax_t)VM_PAGE_TO_PHYS(ma[i]),
  672                             ma[i]));
  673                 }
  674         }
  675         error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
  676             flags, segs, segp);
  677         free(ma, M_DEVBUF);
  678         free(fma, M_DEVBUF);
  679         return (error);
  680 }
  681 
  682 static void
  683 dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
  684     struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
  685 {
  686         struct bus_dmamap_dmar *map;
  687 
  688         if (map1 == NULL)
  689                 return;
  690         map = (struct bus_dmamap_dmar *)map1;
  691         map->mem = *mem;
  692         map->tag = (struct bus_dma_tag_dmar *)dmat;
  693         map->callback = callback;
  694         map->callback_arg = callback_arg;
  695 }
  696 
  697 static bus_dma_segment_t *
  698 dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
  699     bus_dma_segment_t *segs, int nsegs, int error)
  700 {
  701         struct bus_dma_tag_dmar *tag;
  702         struct bus_dmamap_dmar *map;
  703 
  704         tag = (struct bus_dma_tag_dmar *)dmat;
  705         map = (struct bus_dmamap_dmar *)map1;
  706 
  707         if (!map->locked) {
  708                 KASSERT(map->cansleep,
  709                     ("map not locked and not sleepable context %p", map));
  710 
  711                 /*
  712                  * We are called from the delayed context.  Relock the
  713                  * driver.
  714                  */
  715                 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
  716                 map->locked = true;
  717         }
  718 
  719         if (segs == NULL)
  720                 segs = tag->segments;
  721         return (segs);
  722 }
  723 
  724 /*
  725  * The limitations of busdma KPI forces the dmar to perform the actual
  726  * unload, consisting of the unmapping of the map entries page tables,
  727  * from the delayed context on i386, since page table page mapping
  728  * might require a sleep to be successfull.  The unfortunate
  729  * consequence is that the DMA requests can be served some time after
  730  * the bus_dmamap_unload() call returned.
  731  *
  732  * On amd64, we assume that sf allocation cannot fail.
  733  */
  734 static void
  735 dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
  736 {
  737         struct bus_dma_tag_dmar *tag;
  738         struct bus_dmamap_dmar *map;
  739         struct dmar_ctx *ctx;
  740 #if defined(__amd64__)
  741         struct dmar_map_entries_tailq entries;
  742 #endif
  743 
  744         tag = (struct bus_dma_tag_dmar *)dmat;
  745         map = (struct bus_dmamap_dmar *)map1;
  746         ctx = tag->ctx;
  747         atomic_add_long(&ctx->unloads, 1);
  748 
  749 #if defined(__i386__)
  750         DMAR_CTX_LOCK(ctx);
  751         TAILQ_CONCAT(&ctx->unload_entries, &map->map_entries, dmamap_link);
  752         DMAR_CTX_UNLOCK(ctx);
  753         taskqueue_enqueue(ctx->dmar->delayed_taskqueue, &ctx->unload_task);
  754 #else /* defined(__amd64__) */
  755         TAILQ_INIT(&entries);
  756         DMAR_CTX_LOCK(ctx);
  757         TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
  758         DMAR_CTX_UNLOCK(ctx);
  759         THREAD_NO_SLEEPING();
  760         dmar_ctx_unload(ctx, &entries, false);
  761         THREAD_SLEEPING_OK();
  762         KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx));
  763 #endif
  764 }
  765 
  766 static void
  767 dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
  768     bus_dmasync_op_t op)
  769 {
  770 }
  771 
  772 struct bus_dma_impl bus_dma_dmar_impl = {
  773         .tag_create = dmar_bus_dma_tag_create,
  774         .tag_destroy = dmar_bus_dma_tag_destroy,
  775         .map_create = dmar_bus_dmamap_create,
  776         .map_destroy = dmar_bus_dmamap_destroy,
  777         .mem_alloc = dmar_bus_dmamem_alloc,
  778         .mem_free = dmar_bus_dmamem_free,
  779         .load_phys = dmar_bus_dmamap_load_phys,
  780         .load_buffer = dmar_bus_dmamap_load_buffer,
  781         .load_ma = dmar_bus_dmamap_load_ma,
  782         .map_waitok = dmar_bus_dmamap_waitok,
  783         .map_complete = dmar_bus_dmamap_complete,
  784         .map_unload = dmar_bus_dmamap_unload,
  785         .map_sync = dmar_bus_dmamap_sync
  786 };
  787 
  788 static void
  789 dmar_bus_task_dmamap(void *arg, int pending)
  790 {
  791         struct bus_dma_tag_dmar *tag;
  792         struct bus_dmamap_dmar *map;
  793         struct dmar_unit *unit;
  794         struct dmar_ctx *ctx;
  795 
  796         unit = arg;
  797         DMAR_LOCK(unit);
  798         while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
  799                 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
  800                 DMAR_UNLOCK(unit);
  801                 tag = map->tag;
  802                 ctx = map->tag->ctx;
  803                 map->cansleep = true;
  804                 map->locked = false;
  805                 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
  806                     &map->mem, map->callback, map->callback_arg,
  807                     BUS_DMA_WAITOK);
  808                 map->cansleep = false;
  809                 if (map->locked) {
  810                         (tag->common.lockfunc)(tag->common.lockfuncarg,
  811                             BUS_DMA_UNLOCK);
  812                 } else
  813                         map->locked = true;
  814                 map->cansleep = false;
  815                 DMAR_LOCK(unit);
  816         }
  817         DMAR_UNLOCK(unit);
  818 }
  819 
  820 static void
  821 dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map)
  822 {
  823         struct dmar_ctx *ctx;
  824 
  825         ctx = map->tag->ctx;
  826         map->locked = false;
  827         DMAR_LOCK(unit);
  828         TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
  829         DMAR_UNLOCK(unit);
  830         taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
  831 }
  832 
  833 int
  834 dmar_init_busdma(struct dmar_unit *unit)
  835 {
  836 
  837         TAILQ_INIT(&unit->delayed_maps);
  838         TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit);
  839         unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK,
  840             taskqueue_thread_enqueue, &unit->delayed_taskqueue);
  841         taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
  842             "dmar%d busdma taskq", unit->unit);
  843         return (0);
  844 }
  845 
  846 void
  847 dmar_fini_busdma(struct dmar_unit *unit)
  848 {
  849 
  850         if (unit->delayed_taskqueue == NULL)
  851                 return;
  852 
  853         taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
  854         taskqueue_free(unit->delayed_taskqueue);
  855         unit->delayed_taskqueue = NULL;
  856 }

Cache object: be84b0b56025bd65ac6ef8ffb3ecaafe


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.