The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/busdma_dmar.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2013 The FreeBSD Foundation
    3  * All rights reserved.
    4  *
    5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    6  * under sponsorship from the FreeBSD Foundation.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD$");
   32 
   33 #include <sys/param.h>
   34 #include <sys/systm.h>
   35 #include <sys/malloc.h>
   36 #include <sys/bus.h>
   37 #include <sys/conf.h>
   38 #include <sys/interrupt.h>
   39 #include <sys/kernel.h>
   40 #include <sys/ktr.h>
   41 #include <sys/lock.h>
   42 #include <sys/proc.h>
   43 #include <sys/memdesc.h>
   44 #include <sys/mutex.h>
   45 #include <sys/sysctl.h>
   46 #include <sys/rman.h>
   47 #include <sys/taskqueue.h>
   48 #include <sys/tree.h>
   49 #include <sys/uio.h>
   50 #include <dev/pci/pcireg.h>
   51 #include <dev/pci/pcivar.h>
   52 #include <vm/vm.h>
   53 #include <vm/vm_extern.h>
   54 #include <vm/vm_kern.h>
   55 #include <vm/vm_object.h>
   56 #include <vm/vm_page.h>
   57 #include <vm/vm_map.h>
   58 #include <machine/atomic.h>
   59 #include <machine/bus.h>
   60 #include <machine/md_var.h>
   61 #include <machine/specialreg.h>
   62 #include <x86/include/busdma_impl.h>
   63 #include <x86/iommu/intel_reg.h>
   64 #include <x86/iommu/busdma_dmar.h>
   65 #include <x86/iommu/intel_dmar.h>
   66 
   67 /*
   68  * busdma_dmar.c, the implementation of the busdma(9) interface using
   69  * DMAR units from Intel VT-d.
   70  */
   71 
   72 static bool
   73 dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
   74 {
   75         char str[128], *env;
   76 
   77         snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d.bounce",
   78             domain, bus, slot, func);
   79         env = getenv(str);
   80         if (env == NULL)
   81                 return (false);
   82         freeenv(env);
   83         return (true);
   84 }
   85 
   86 /*
   87  * Given original device, find the requester ID that will be seen by
   88  * the DMAR unit and used for page table lookup.  PCI bridges may take
   89  * ownership of transactions from downstream devices, so it may not be
   90  * the same as the BSF of the target device.  In those cases, all
   91  * devices downstream of the bridge must share a single mapping
   92  * domain, and must collectively be assigned to use either DMAR or
   93  * bounce mapping.
   94  */
   95 static device_t
   96 dmar_get_requester(device_t dev, uint16_t *rid)
   97 {
   98         devclass_t pci_class;
   99         device_t l, pci, pcib, pcip, pcibp, requester;
  100         int cap_offset;
  101         uint16_t pcie_flags;
  102         bool bridge_is_pcie;
  103 
  104         pci_class = devclass_find("pci");
  105         l = requester = dev;
  106 
  107         *rid = pci_get_rid(dev);
  108 
  109         /*
  110          * Walk the bridge hierarchy from the target device to the
  111          * host port to find the translating bridge nearest the DMAR
  112          * unit.
  113          */
  114         for (;;) {
  115                 pci = device_get_parent(l);
  116                 KASSERT(pci != NULL, ("dmar_get_requester(%s): NULL parent "
  117                     "for %s", device_get_name(dev), device_get_name(l)));
  118                 KASSERT(device_get_devclass(pci) == pci_class,
  119                     ("dmar_get_requester(%s): non-pci parent %s for %s",
  120                     device_get_name(dev), device_get_name(pci),
  121                     device_get_name(l)));
  122 
  123                 pcib = device_get_parent(pci);
  124                 KASSERT(pcib != NULL, ("dmar_get_requester(%s): NULL bridge "
  125                     "for %s", device_get_name(dev), device_get_name(pci)));
  126 
  127                 /*
  128                  * The parent of our "bridge" isn't another PCI bus,
  129                  * so pcib isn't a PCI->PCI bridge but rather a host
  130                  * port, and the requester ID won't be translated
  131                  * further.
  132                  */
  133                 pcip = device_get_parent(pcib);
  134                 if (device_get_devclass(pcip) != pci_class)
  135                         break;
  136                 pcibp = device_get_parent(pcip);
  137 
  138                 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
  139                         /*
  140                          * Do not stop the loop even if the target
  141                          * device is PCIe, because it is possible (but
  142                          * unlikely) to have a PCI->PCIe bridge
  143                          * somewhere in the hierarchy.
  144                          */
  145                         l = pcib;
  146                 } else {
  147                         /*
  148                          * Device is not PCIe, it cannot be seen as a
  149                          * requester by DMAR unit.  Check whether the
  150                          * bridge is PCIe.
  151                          */
  152                         bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
  153                             &cap_offset) == 0;
  154                         requester = pcib;
  155 
  156                         /*
  157                          * Check for a buggy PCIe/PCI bridge that
  158                          * doesn't report the express capability.  If
  159                          * the bridge above it is express but isn't a
  160                          * PCI bridge, then we know pcib is actually a
  161                          * PCIe/PCI bridge.
  162                          */
  163                         if (!bridge_is_pcie && pci_find_cap(pcibp,
  164                             PCIY_EXPRESS, &cap_offset) == 0) {
  165                                 pcie_flags = pci_read_config(pcibp,
  166                                     cap_offset + PCIER_FLAGS, 2);
  167                                 if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
  168                                     PCIEM_TYPE_PCI_BRIDGE)
  169                                         bridge_is_pcie = true;
  170                         }
  171 
  172                         if (bridge_is_pcie) {
  173                                 /*
  174                                  * The current device is not PCIe, but
  175                                  * the bridge above it is.  This is a
  176                                  * PCIe->PCI bridge.  Assume that the
  177                                  * requester ID will be the secondary
  178                                  * bus number with slot and function
  179                                  * set to zero.
  180                                  *
  181                                  * XXX: Doesn't handle the case where
  182                                  * the bridge is PCIe->PCI-X, and the
  183                                  * bridge will only take ownership of
  184                                  * requests in some cases.  We should
  185                                  * provide context entries with the
  186                                  * same page tables for taken and
  187                                  * non-taken transactions.
  188                                  */
  189                                 *rid = PCI_RID(pci_get_bus(l), 0, 0);
  190                                 l = pcibp;
  191                         } else {
  192                                 /*
  193                                  * Neither the device nor the bridge
  194                                  * above it are PCIe.  This is a
  195                                  * conventional PCI->PCI bridge, which
  196                                  * will use the bridge's BSF as the
  197                                  * requester ID.
  198                                  */
  199                                 *rid = pci_get_rid(pcib);
  200                                 l = pcib;
  201                         }
  202                 }
  203         }
  204         return (requester);
  205 }
  206 
  207 struct dmar_ctx *
  208 dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
  209 {
  210         device_t requester;
  211         struct dmar_ctx *ctx;
  212         bool disabled;
  213         uint16_t rid;
  214 
  215         requester = dmar_get_requester(dev, &rid);
  216 
  217         /*
  218          * If the user requested the IOMMU disabled for the device, we
  219          * cannot disable the DMAR, due to possibility of other
  220          * devices on the same DMAR still requiring translation.
  221          * Instead provide the identity mapping for the device
  222          * context.
  223          */
  224         disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester), 
  225             pci_get_bus(requester), pci_get_slot(requester), 
  226             pci_get_function(requester));
  227         ctx = dmar_get_ctx(dmar, requester, rid, disabled, rmrr);
  228         if (ctx == NULL)
  229                 return (NULL);
  230         if (disabled) {
  231                 /*
  232                  * Keep the first reference on context, release the
  233                  * later refs.
  234                  */
  235                 DMAR_LOCK(dmar);
  236                 if ((ctx->flags & DMAR_CTX_DISABLED) == 0) {
  237                         ctx->flags |= DMAR_CTX_DISABLED;
  238                         DMAR_UNLOCK(dmar);
  239                 } else {
  240                         dmar_free_ctx_locked(dmar, ctx);
  241                 }
  242                 ctx = NULL;
  243         }
  244         return (ctx);
  245 }
  246 
  247 bus_dma_tag_t
  248 dmar_get_dma_tag(device_t dev, device_t child)
  249 {
  250         struct dmar_unit *dmar;
  251         struct dmar_ctx *ctx;
  252         bus_dma_tag_t res;
  253 
  254         dmar = dmar_find(child);
  255         /* Not in scope of any DMAR ? */
  256         if (dmar == NULL)
  257                 return (NULL);
  258         dmar_quirks_pre_use(dmar);
  259         dmar_instantiate_rmrr_ctxs(dmar);
  260 
  261         ctx = dmar_instantiate_ctx(dmar, child, false);
  262         res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag;
  263         return (res);
  264 }
  265 
  266 static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map");
  267 
  268 static void dmar_bus_schedule_dmamap(struct dmar_unit *unit,
  269     struct bus_dmamap_dmar *map);
  270 
  271 static int
  272 dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  273     bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
  274     bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
  275     int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  276     void *lockfuncarg, bus_dma_tag_t *dmat)
  277 {
  278         struct bus_dma_tag_dmar *newtag, *oldtag;
  279         int error;
  280 
  281         *dmat = NULL;
  282         error = common_bus_dma_tag_create(parent != NULL ?
  283             &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment,
  284             boundary, lowaddr, highaddr, filter, filterarg, maxsize,
  285             nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
  286             sizeof(struct bus_dma_tag_dmar), (void **)&newtag);
  287         if (error != 0)
  288                 goto out;
  289 
  290         oldtag = (struct bus_dma_tag_dmar *)parent;
  291         newtag->common.impl = &bus_dma_dmar_impl;
  292         newtag->ctx = oldtag->ctx;
  293         newtag->owner = oldtag->owner;
  294 
  295         *dmat = (bus_dma_tag_t)newtag;
  296 out:
  297         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  298             __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
  299             error);
  300         return (error);
  301 }
  302 
  303 static int
  304 dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
  305 {
  306         struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent;
  307         int error;
  308 
  309         error = 0;
  310         dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1;
  311 
  312         if (dmat != NULL) {
  313                 if (dmat->map_count != 0) {
  314                         error = EBUSY;
  315                         goto out;
  316                 }
  317                 while (dmat != NULL) {
  318                         parent = (struct bus_dma_tag_dmar *)dmat->common.parent;
  319                         if (atomic_fetchadd_int(&dmat->common.ref_count, -1) ==
  320                             1) {
  321                                 if (dmat == &dmat->ctx->ctx_tag)
  322                                         dmar_free_ctx(dmat->ctx);
  323                                 free(dmat->segments, M_DMAR_DMAMAP);
  324                                 free(dmat, M_DEVBUF);
  325                                 dmat = parent;
  326                         } else
  327                                 dmat = NULL;
  328                 }
  329         }
  330 out:
  331         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
  332         return (error);
  333 }
  334 
  335 static int
  336 dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  337 {
  338         struct bus_dma_tag_dmar *tag;
  339         struct bus_dmamap_dmar *map;
  340 
  341         tag = (struct bus_dma_tag_dmar *)dmat;
  342         map = malloc(sizeof(*map), M_DMAR_DMAMAP, M_NOWAIT | M_ZERO);
  343         if (map == NULL) {
  344                 *mapp = NULL;
  345                 return (ENOMEM);
  346         }
  347         if (tag->segments == NULL) {
  348                 tag->segments = malloc(sizeof(bus_dma_segment_t) *
  349                     tag->common.nsegments, M_DMAR_DMAMAP, M_NOWAIT);
  350                 if (tag->segments == NULL) {
  351                         free(map, M_DMAR_DMAMAP);
  352                         *mapp = NULL;
  353                         return (ENOMEM);
  354                 }
  355         }
  356         TAILQ_INIT(&map->map_entries);
  357         map->tag = tag;
  358         map->locked = true;
  359         map->cansleep = false;
  360         tag->map_count++;
  361         *mapp = (bus_dmamap_t)map;
  362 
  363         return (0);
  364 }
  365 
  366 static int
  367 dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
  368 {
  369         struct bus_dma_tag_dmar *tag;
  370         struct bus_dmamap_dmar *map;
  371 
  372         tag = (struct bus_dma_tag_dmar *)dmat;
  373         map = (struct bus_dmamap_dmar *)map1;
  374         if (map != NULL) {
  375                 DMAR_CTX_LOCK(tag->ctx);
  376                 if (!TAILQ_EMPTY(&map->map_entries)) {
  377                         DMAR_CTX_UNLOCK(tag->ctx);
  378                         return (EBUSY);
  379                 }
  380                 DMAR_CTX_UNLOCK(tag->ctx);
  381                 free(map, M_DMAR_DMAMAP);
  382         }
  383         tag->map_count--;
  384         return (0);
  385 }
  386 
  387 
  388 static int
  389 dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
  390     bus_dmamap_t *mapp)
  391 {
  392         struct bus_dma_tag_dmar *tag;
  393         struct bus_dmamap_dmar *map;
  394         int error, mflags;
  395         vm_memattr_t attr;
  396 
  397         error = dmar_bus_dmamap_create(dmat, flags, mapp);
  398         if (error != 0)
  399                 return (error);
  400 
  401         mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
  402         mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
  403         attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
  404             VM_MEMATTR_DEFAULT;
  405 
  406         tag = (struct bus_dma_tag_dmar *)dmat;
  407         map = (struct bus_dmamap_dmar *)*mapp;
  408 
  409         if (tag->common.maxsize < PAGE_SIZE &&
  410             tag->common.alignment <= tag->common.maxsize &&
  411             attr == VM_MEMATTR_DEFAULT) {
  412                 *vaddr = malloc(tag->common.maxsize, M_DEVBUF, mflags);
  413                 map->flags |= BUS_DMAMAP_DMAR_MALLOC;
  414         } else {
  415                 *vaddr = (void *)kmem_alloc_attr(kernel_arena,
  416                     tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR,
  417                     attr);
  418                 map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC;
  419         }
  420         if (*vaddr == NULL) {
  421                 dmar_bus_dmamap_destroy(dmat, *mapp);
  422                 *mapp = NULL;
  423                 return (ENOMEM);
  424         }
  425         return (0);
  426 }
  427 
  428 static void
  429 dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
  430 {
  431         struct bus_dma_tag_dmar *tag;
  432         struct bus_dmamap_dmar *map;
  433 
  434         tag = (struct bus_dma_tag_dmar *)dmat;
  435         map = (struct bus_dmamap_dmar *)map1;
  436 
  437         if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) {
  438                 free(vaddr, M_DEVBUF);
  439                 map->flags &= ~BUS_DMAMAP_DMAR_MALLOC;
  440         } else {
  441                 KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0,
  442                     ("dmar_bus_dmamem_free for non alloced map %p", map));
  443                 kmem_free(kernel_arena, (vm_offset_t)vaddr, tag->common.maxsize);
  444                 map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC;
  445         }
  446 
  447         dmar_bus_dmamap_destroy(dmat, map1);
  448 }
  449 
  450 static int
  451 dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag,
  452     struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
  453     int flags, bus_dma_segment_t *segs, int *segp,
  454     struct dmar_map_entries_tailq *unroll_list)
  455 {
  456         struct dmar_ctx *ctx;
  457         struct dmar_map_entry *entry;
  458         dmar_gaddr_t size;
  459         bus_size_t buflen1;
  460         int error, idx, gas_flags, seg;
  461 
  462         KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset));
  463         if (segs == NULL)
  464                 segs = tag->segments;
  465         ctx = tag->ctx;
  466         seg = *segp;
  467         error = 0;
  468         idx = 0;
  469         while (buflen > 0) {
  470                 seg++;
  471                 if (seg >= tag->common.nsegments) {
  472                         error = EFBIG;
  473                         break;
  474                 }
  475                 buflen1 = buflen > tag->common.maxsegsz ?
  476                     tag->common.maxsegsz : buflen;
  477                 size = round_page(offset + buflen1);
  478 
  479                 /*
  480                  * (Too) optimistically allow split if there are more
  481                  * then one segments left.
  482                  */
  483                 gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0;
  484                 if (seg + 1 < tag->common.nsegments)
  485                         gas_flags |= DMAR_GM_CANSPLIT;
  486 
  487                 error = dmar_gas_map(ctx, &tag->common, size, offset,
  488                     DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE,
  489                     gas_flags, ma + idx, &entry);
  490                 if (error != 0)
  491                         break;
  492                 if ((gas_flags & DMAR_GM_CANSPLIT) != 0) {
  493                         KASSERT(size >= entry->end - entry->start,
  494                             ("split increased entry size %jx %jx %jx",
  495                             (uintmax_t)size, (uintmax_t)entry->start,
  496                             (uintmax_t)entry->end));
  497                         size = entry->end - entry->start;
  498                         if (buflen1 > size)
  499                                 buflen1 = size;
  500                 } else {
  501                         KASSERT(entry->end - entry->start == size,
  502                             ("no split allowed %jx %jx %jx",
  503                             (uintmax_t)size, (uintmax_t)entry->start,
  504                             (uintmax_t)entry->end));
  505                 }
  506                 if (offset + buflen1 > size)
  507                         buflen1 = size - offset;
  508                 if (buflen1 > tag->common.maxsegsz)
  509                         buflen1 = tag->common.maxsegsz;
  510 
  511                 KASSERT(((entry->start + offset) & (tag->common.alignment - 1))
  512                     == 0,
  513                     ("alignment failed: ctx %p start 0x%jx offset %x "
  514                     "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
  515                     (uintmax_t)tag->common.alignment));
  516                 KASSERT(entry->end <= tag->common.lowaddr ||
  517                     entry->start >= tag->common.highaddr,
  518                     ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
  519                     "lowaddr 0x%jx highaddr 0x%jx", ctx,
  520                     (uintmax_t)entry->start, (uintmax_t)entry->end,
  521                     (uintmax_t)tag->common.lowaddr,
  522                     (uintmax_t)tag->common.highaddr));
  523                 KASSERT(dmar_test_boundary(entry->start + offset, buflen1,
  524                     tag->common.boundary),
  525                     ("boundary failed: ctx %p start 0x%jx end 0x%jx "
  526                     "boundary 0x%jx", ctx, (uintmax_t)entry->start,
  527                     (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
  528                 KASSERT(buflen1 <= tag->common.maxsegsz,
  529                     ("segment too large: ctx %p start 0x%jx end 0x%jx "
  530                     "buflen1 0x%jx maxsegsz 0x%jx", ctx,
  531                     (uintmax_t)entry->start, (uintmax_t)entry->end,
  532                     (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));
  533 
  534                 DMAR_CTX_LOCK(ctx);
  535                 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
  536                 entry->flags |= DMAR_MAP_ENTRY_MAP;
  537                 DMAR_CTX_UNLOCK(ctx);
  538                 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);
  539 
  540                 segs[seg].ds_addr = entry->start + offset;
  541                 segs[seg].ds_len = buflen1;
  542 
  543                 idx += OFF_TO_IDX(trunc_page(offset + buflen1));
  544                 offset += buflen1;
  545                 offset &= DMAR_PAGE_MASK;
  546                 buflen -= buflen1;
  547         }
  548         if (error == 0)
  549                 *segp = seg;
  550         return (error);
  551 }
  552 
  553 static int
  554 dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag,
  555     struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
  556     int flags, bus_dma_segment_t *segs, int *segp)
  557 {
  558         struct dmar_ctx *ctx;
  559         struct dmar_map_entry *entry, *entry1;
  560         struct dmar_map_entries_tailq unroll_list;
  561         int error;
  562 
  563         ctx = tag->ctx;
  564         atomic_add_long(&ctx->loads, 1);
  565 
  566         TAILQ_INIT(&unroll_list);
  567         error = dmar_bus_dmamap_load_something1(tag, map, ma, offset,
  568             buflen, flags, segs, segp, &unroll_list);
  569         if (error != 0) {
  570                 /*
  571                  * The busdma interface does not allow us to report
  572                  * partial buffer load, so unfortunately we have to
  573                  * revert all work done.
  574                  */
  575                 DMAR_CTX_LOCK(ctx);
  576                 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link,
  577                     entry1) {
  578                         /*
  579                          * No entries other than what we have created
  580                          * during the failed run might have been
  581                          * inserted there in between, since we own ctx
  582                          * pglock.
  583                          */
  584                         TAILQ_REMOVE(&map->map_entries, entry, dmamap_link);
  585                         TAILQ_REMOVE(&unroll_list, entry, unroll_link);
  586                         TAILQ_INSERT_TAIL(&ctx->unload_entries, entry,
  587                             dmamap_link);
  588                 }
  589                 DMAR_CTX_UNLOCK(ctx);
  590                 taskqueue_enqueue(ctx->dmar->delayed_taskqueue,
  591                     &ctx->unload_task);
  592         }
  593 
  594         if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
  595             !map->cansleep)
  596                 error = EINPROGRESS;
  597         if (error == EINPROGRESS)
  598                 dmar_bus_schedule_dmamap(ctx->dmar, map);
  599         return (error);
  600 }
  601 
  602 static int
  603 dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
  604     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
  605     bus_dma_segment_t *segs, int *segp)
  606 {
  607         struct bus_dma_tag_dmar *tag;
  608         struct bus_dmamap_dmar *map;
  609 
  610         tag = (struct bus_dma_tag_dmar *)dmat;
  611         map = (struct bus_dmamap_dmar *)map1;
  612         return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
  613             flags, segs, segp));
  614 }
  615 
  616 static int
  617 dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
  618     vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
  619     int *segp)
  620 {
  621         struct bus_dma_tag_dmar *tag;
  622         struct bus_dmamap_dmar *map;
  623         vm_page_t *ma;
  624         vm_paddr_t pstart, pend;
  625         int error, i, ma_cnt, offset;
  626 
  627         tag = (struct bus_dma_tag_dmar *)dmat;
  628         map = (struct bus_dmamap_dmar *)map1;
  629         pstart = trunc_page(buf);
  630         pend = round_page(buf + buflen);
  631         offset = buf & PAGE_MASK;
  632         ma_cnt = OFF_TO_IDX(pend - pstart);
  633         ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
  634             M_WAITOK : M_NOWAIT);
  635         if (ma == NULL)
  636                 return (ENOMEM);
  637         for (i = 0; i < ma_cnt; i++)
  638                 ma[i] = PHYS_TO_VM_PAGE(pstart + i * PAGE_SIZE);
  639         error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
  640             flags, segs, segp);
  641         free(ma, M_DEVBUF);
  642         return (error);
  643 }
  644 
  645 static int
  646 dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
  647     bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
  648     int *segp)
  649 {
  650         struct bus_dma_tag_dmar *tag;
  651         struct bus_dmamap_dmar *map;
  652         vm_page_t *ma, fma;
  653         vm_paddr_t pstart, pend, paddr;
  654         int error, i, ma_cnt, offset;
  655 
  656         tag = (struct bus_dma_tag_dmar *)dmat;
  657         map = (struct bus_dmamap_dmar *)map1;
  658         pstart = trunc_page((vm_offset_t)buf);
  659         pend = round_page((vm_offset_t)buf + buflen);
  660         offset = (vm_offset_t)buf & PAGE_MASK;
  661         ma_cnt = OFF_TO_IDX(pend - pstart);
  662         ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
  663             M_WAITOK : M_NOWAIT);
  664         if (ma == NULL)
  665                 return (ENOMEM);
  666         if (dumping) {
  667                 /*
  668                  * If dumping, do not attempt to call
  669                  * PHYS_TO_VM_PAGE() at all.  It may return non-NULL
  670                  * but the vm_page returned might be not initialized,
  671                  * e.g. for the kernel itself.
  672                  */
  673                 KASSERT(pmap == kernel_pmap, ("non-kernel address write"));
  674                 fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF,
  675                     M_ZERO | (map->cansleep ? M_WAITOK : M_NOWAIT));
  676                 if (fma == NULL) {
  677                         free(ma, M_DEVBUF);
  678                         return (ENOMEM);
  679                 }
  680                 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
  681                         paddr = pmap_kextract(pstart);
  682                         vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
  683                         ma[i] = &fma[i];
  684                 }
  685         } else {
  686                 fma = NULL;
  687                 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
  688                         if (pmap == kernel_pmap)
  689                                 paddr = pmap_kextract(pstart);
  690                         else
  691                                 paddr = pmap_extract(pmap, pstart);
  692                         ma[i] = PHYS_TO_VM_PAGE(paddr);
  693                         KASSERT(VM_PAGE_TO_PHYS(ma[i]) == paddr,
  694                             ("PHYS_TO_VM_PAGE failed %jx %jx m %p",
  695                             (uintmax_t)paddr, (uintmax_t)VM_PAGE_TO_PHYS(ma[i]),
  696                             ma[i]));
  697                 }
  698         }
  699         error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
  700             flags, segs, segp);
  701         free(ma, M_DEVBUF);
  702         free(fma, M_DEVBUF);
  703         return (error);
  704 }
  705 
  706 static void
  707 dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
  708     struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
  709 {
  710         struct bus_dmamap_dmar *map;
  711 
  712         if (map1 == NULL)
  713                 return;
  714         map = (struct bus_dmamap_dmar *)map1;
  715         map->mem = *mem;
  716         map->tag = (struct bus_dma_tag_dmar *)dmat;
  717         map->callback = callback;
  718         map->callback_arg = callback_arg;
  719 }
  720 
  721 static bus_dma_segment_t *
  722 dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
  723     bus_dma_segment_t *segs, int nsegs, int error)
  724 {
  725         struct bus_dma_tag_dmar *tag;
  726         struct bus_dmamap_dmar *map;
  727 
  728         tag = (struct bus_dma_tag_dmar *)dmat;
  729         map = (struct bus_dmamap_dmar *)map1;
  730 
  731         if (!map->locked) {
  732                 KASSERT(map->cansleep,
  733                     ("map not locked and not sleepable context %p", map));
  734 
  735                 /*
  736                  * We are called from the delayed context.  Relock the
  737                  * driver.
  738                  */
  739                 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
  740                 map->locked = true;
  741         }
  742 
  743         if (segs == NULL)
  744                 segs = tag->segments;
  745         return (segs);
  746 }
  747 
  748 /*
  749  * The limitations of busdma KPI forces the dmar to perform the actual
  750  * unload, consisting of the unmapping of the map entries page tables,
  751  * from the delayed context on i386, since page table page mapping
  752  * might require a sleep to be successfull.  The unfortunate
  753  * consequence is that the DMA requests can be served some time after
  754  * the bus_dmamap_unload() call returned.
  755  *
  756  * On amd64, we assume that sf allocation cannot fail.
  757  */
  758 static void
  759 dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
  760 {
  761         struct bus_dma_tag_dmar *tag;
  762         struct bus_dmamap_dmar *map;
  763         struct dmar_ctx *ctx;
  764 #if defined(__amd64__)
  765         struct dmar_map_entries_tailq entries;
  766 #endif
  767 
  768         tag = (struct bus_dma_tag_dmar *)dmat;
  769         map = (struct bus_dmamap_dmar *)map1;
  770         ctx = tag->ctx;
  771         atomic_add_long(&ctx->unloads, 1);
  772 
  773 #if defined(__i386__)
  774         DMAR_CTX_LOCK(ctx);
  775         TAILQ_CONCAT(&ctx->unload_entries, &map->map_entries, dmamap_link);
  776         DMAR_CTX_UNLOCK(ctx);
  777         taskqueue_enqueue(ctx->dmar->delayed_taskqueue, &ctx->unload_task);
  778 #else /* defined(__amd64__) */
  779         TAILQ_INIT(&entries);
  780         DMAR_CTX_LOCK(ctx);
  781         TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
  782         DMAR_CTX_UNLOCK(ctx);
  783         THREAD_NO_SLEEPING();
  784         dmar_ctx_unload(ctx, &entries, false);
  785         THREAD_SLEEPING_OK();
  786         KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx));
  787 #endif
  788 }
  789 
  790 static void
  791 dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
  792     bus_dmasync_op_t op)
  793 {
  794 }
  795 
  796 struct bus_dma_impl bus_dma_dmar_impl = {
  797         .tag_create = dmar_bus_dma_tag_create,
  798         .tag_destroy = dmar_bus_dma_tag_destroy,
  799         .map_create = dmar_bus_dmamap_create,
  800         .map_destroy = dmar_bus_dmamap_destroy,
  801         .mem_alloc = dmar_bus_dmamem_alloc,
  802         .mem_free = dmar_bus_dmamem_free,
  803         .load_phys = dmar_bus_dmamap_load_phys,
  804         .load_buffer = dmar_bus_dmamap_load_buffer,
  805         .load_ma = dmar_bus_dmamap_load_ma,
  806         .map_waitok = dmar_bus_dmamap_waitok,
  807         .map_complete = dmar_bus_dmamap_complete,
  808         .map_unload = dmar_bus_dmamap_unload,
  809         .map_sync = dmar_bus_dmamap_sync
  810 };
  811 
  812 static void
  813 dmar_bus_task_dmamap(void *arg, int pending)
  814 {
  815         struct bus_dma_tag_dmar *tag;
  816         struct bus_dmamap_dmar *map;
  817         struct dmar_unit *unit;
  818 
  819         unit = arg;
  820         DMAR_LOCK(unit);
  821         while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
  822                 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
  823                 DMAR_UNLOCK(unit);
  824                 tag = map->tag;
  825                 map->cansleep = true;
  826                 map->locked = false;
  827                 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
  828                     &map->mem, map->callback, map->callback_arg,
  829                     BUS_DMA_WAITOK);
  830                 map->cansleep = false;
  831                 if (map->locked) {
  832                         (tag->common.lockfunc)(tag->common.lockfuncarg,
  833                             BUS_DMA_UNLOCK);
  834                 } else
  835                         map->locked = true;
  836                 map->cansleep = false;
  837                 DMAR_LOCK(unit);
  838         }
  839         DMAR_UNLOCK(unit);
  840 }
  841 
  842 static void
  843 dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map)
  844 {
  845 
  846         map->locked = false;
  847         DMAR_LOCK(unit);
  848         TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
  849         DMAR_UNLOCK(unit);
  850         taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
  851 }
  852 
  853 int
  854 dmar_init_busdma(struct dmar_unit *unit)
  855 {
  856 
  857         TAILQ_INIT(&unit->delayed_maps);
  858         TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit);
  859         unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK,
  860             taskqueue_thread_enqueue, &unit->delayed_taskqueue);
  861         taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
  862             "dmar%d busdma taskq", unit->unit);
  863         return (0);
  864 }
  865 
  866 void
  867 dmar_fini_busdma(struct dmar_unit *unit)
  868 {
  869 
  870         if (unit->delayed_taskqueue == NULL)
  871                 return;
  872 
  873         taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
  874         taskqueue_free(unit->delayed_taskqueue);
  875         unit->delayed_taskqueue = NULL;
  876 }

Cache object: 4b445f86c8f78f1cc7c4097de5e34ab9


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.