The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/busdma_dmar.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2013 The FreeBSD Foundation
    3  * All rights reserved.
    4  *
    5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    6  * under sponsorship from the FreeBSD Foundation.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/11.1/sys/x86/iommu/busdma_dmar.c 316392 2017-04-02 07:11:15Z kib $");
   32 
   33 #include <sys/param.h>
   34 #include <sys/systm.h>
   35 #include <sys/malloc.h>
   36 #include <sys/bus.h>
   37 #include <sys/conf.h>
   38 #include <sys/interrupt.h>
   39 #include <sys/kernel.h>
   40 #include <sys/ktr.h>
   41 #include <sys/lock.h>
   42 #include <sys/proc.h>
   43 #include <sys/memdesc.h>
   44 #include <sys/mutex.h>
   45 #include <sys/sysctl.h>
   46 #include <sys/rman.h>
   47 #include <sys/taskqueue.h>
   48 #include <sys/tree.h>
   49 #include <sys/uio.h>
   50 #include <sys/vmem.h>
   51 #include <dev/pci/pcireg.h>
   52 #include <dev/pci/pcivar.h>
   53 #include <vm/vm.h>
   54 #include <vm/vm_extern.h>
   55 #include <vm/vm_kern.h>
   56 #include <vm/vm_object.h>
   57 #include <vm/vm_page.h>
   58 #include <vm/vm_map.h>
   59 #include <machine/atomic.h>
   60 #include <machine/bus.h>
   61 #include <machine/md_var.h>
   62 #include <machine/specialreg.h>
   63 #include <x86/include/busdma_impl.h>
   64 #include <x86/iommu/intel_reg.h>
   65 #include <x86/iommu/busdma_dmar.h>
   66 #include <x86/iommu/intel_dmar.h>
   67 
   68 /*
   69  * busdma_dmar.c, the implementation of the busdma(9) interface using
   70  * DMAR units from Intel VT-d.
   71  */
   72 
   73 static bool
   74 dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
   75 {
   76         char str[128], *env;
   77         int default_bounce;
   78         bool ret;
   79         static const char bounce_str[] = "bounce";
   80         static const char dmar_str[] = "dmar";
   81 
   82         default_bounce = 0;
   83         env = kern_getenv("hw.busdma.default");
   84         if (env != NULL) {
   85                 if (strcmp(env, bounce_str) == 0)
   86                         default_bounce = 1;
   87                 else if (strcmp(env, dmar_str) == 0)
   88                         default_bounce = 0;
   89                 freeenv(env);
   90         }
   91 
   92         snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d",
   93             domain, bus, slot, func);
   94         env = kern_getenv(str);
   95         if (env == NULL)
   96                 return (default_bounce != 0);
   97         if (strcmp(env, bounce_str) == 0)
   98                 ret = true;
   99         else if (strcmp(env, dmar_str) == 0)
  100                 ret = false;
  101         else
  102                 ret = default_bounce != 0;
  103         freeenv(env);
  104         return (ret);
  105 }
  106 
  107 /*
  108  * Given original device, find the requester ID that will be seen by
  109  * the DMAR unit and used for page table lookup.  PCI bridges may take
  110  * ownership of transactions from downstream devices, so it may not be
  111  * the same as the BSF of the target device.  In those cases, all
  112  * devices downstream of the bridge must share a single mapping
  113  * domain, and must collectively be assigned to use either DMAR or
  114  * bounce mapping.
  115  */
  116 device_t
  117 dmar_get_requester(device_t dev, uint16_t *rid)
  118 {
  119         devclass_t pci_class;
  120         device_t l, pci, pcib, pcip, pcibp, requester;
  121         int cap_offset;
  122         uint16_t pcie_flags;
  123         bool bridge_is_pcie;
  124 
  125         pci_class = devclass_find("pci");
  126         l = requester = dev;
  127 
  128         *rid = pci_get_rid(dev);
  129 
  130         /*
  131          * Walk the bridge hierarchy from the target device to the
  132          * host port to find the translating bridge nearest the DMAR
  133          * unit.
  134          */
  135         for (;;) {
  136                 pci = device_get_parent(l);
  137                 KASSERT(pci != NULL, ("dmar_get_requester(%s): NULL parent "
  138                     "for %s", device_get_name(dev), device_get_name(l)));
  139                 KASSERT(device_get_devclass(pci) == pci_class,
  140                     ("dmar_get_requester(%s): non-pci parent %s for %s",
  141                     device_get_name(dev), device_get_name(pci),
  142                     device_get_name(l)));
  143 
  144                 pcib = device_get_parent(pci);
  145                 KASSERT(pcib != NULL, ("dmar_get_requester(%s): NULL bridge "
  146                     "for %s", device_get_name(dev), device_get_name(pci)));
  147 
  148                 /*
  149                  * The parent of our "bridge" isn't another PCI bus,
  150                  * so pcib isn't a PCI->PCI bridge but rather a host
  151                  * port, and the requester ID won't be translated
  152                  * further.
  153                  */
  154                 pcip = device_get_parent(pcib);
  155                 if (device_get_devclass(pcip) != pci_class)
  156                         break;
  157                 pcibp = device_get_parent(pcip);
  158 
  159                 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
  160                         /*
  161                          * Do not stop the loop even if the target
  162                          * device is PCIe, because it is possible (but
  163                          * unlikely) to have a PCI->PCIe bridge
  164                          * somewhere in the hierarchy.
  165                          */
  166                         l = pcib;
  167                 } else {
  168                         /*
  169                          * Device is not PCIe, it cannot be seen as a
  170                          * requester by DMAR unit.  Check whether the
  171                          * bridge is PCIe.
  172                          */
  173                         bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
  174                             &cap_offset) == 0;
  175                         requester = pcib;
  176 
  177                         /*
  178                          * Check for a buggy PCIe/PCI bridge that
  179                          * doesn't report the express capability.  If
  180                          * the bridge above it is express but isn't a
  181                          * PCI bridge, then we know pcib is actually a
  182                          * PCIe/PCI bridge.
  183                          */
  184                         if (!bridge_is_pcie && pci_find_cap(pcibp,
  185                             PCIY_EXPRESS, &cap_offset) == 0) {
  186                                 pcie_flags = pci_read_config(pcibp,
  187                                     cap_offset + PCIER_FLAGS, 2);
  188                                 if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
  189                                     PCIEM_TYPE_PCI_BRIDGE)
  190                                         bridge_is_pcie = true;
  191                         }
  192 
  193                         if (bridge_is_pcie) {
  194                                 /*
  195                                  * The current device is not PCIe, but
  196                                  * the bridge above it is.  This is a
  197                                  * PCIe->PCI bridge.  Assume that the
  198                                  * requester ID will be the secondary
  199                                  * bus number with slot and function
  200                                  * set to zero.
  201                                  *
  202                                  * XXX: Doesn't handle the case where
  203                                  * the bridge is PCIe->PCI-X, and the
  204                                  * bridge will only take ownership of
  205                                  * requests in some cases.  We should
  206                                  * provide context entries with the
  207                                  * same page tables for taken and
  208                                  * non-taken transactions.
  209                                  */
  210                                 *rid = PCI_RID(pci_get_bus(l), 0, 0);
  211                                 l = pcibp;
  212                         } else {
  213                                 /*
  214                                  * Neither the device nor the bridge
  215                                  * above it are PCIe.  This is a
  216                                  * conventional PCI->PCI bridge, which
  217                                  * will use the bridge's BSF as the
  218                                  * requester ID.
  219                                  */
  220                                 *rid = pci_get_rid(pcib);
  221                                 l = pcib;
  222                         }
  223                 }
  224         }
  225         return (requester);
  226 }
  227 
  228 struct dmar_ctx *
  229 dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
  230 {
  231         device_t requester;
  232         struct dmar_ctx *ctx;
  233         bool disabled;
  234         uint16_t rid;
  235 
  236         requester = dmar_get_requester(dev, &rid);
  237 
  238         /*
  239          * If the user requested the IOMMU disabled for the device, we
  240          * cannot disable the DMAR, due to possibility of other
  241          * devices on the same DMAR still requiring translation.
  242          * Instead provide the identity mapping for the device
  243          * context.
  244          */
  245         disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester), 
  246             pci_get_bus(requester), pci_get_slot(requester), 
  247             pci_get_function(requester));
  248         ctx = dmar_get_ctx_for_dev(dmar, requester, rid, disabled, rmrr);
  249         if (ctx == NULL)
  250                 return (NULL);
  251         if (disabled) {
  252                 /*
  253                  * Keep the first reference on context, release the
  254                  * later refs.
  255                  */
  256                 DMAR_LOCK(dmar);
  257                 if ((ctx->flags & DMAR_CTX_DISABLED) == 0) {
  258                         ctx->flags |= DMAR_CTX_DISABLED;
  259                         DMAR_UNLOCK(dmar);
  260                 } else {
  261                         dmar_free_ctx_locked(dmar, ctx);
  262                 }
  263                 ctx = NULL;
  264         }
  265         return (ctx);
  266 }
  267 
  268 bus_dma_tag_t
  269 dmar_get_dma_tag(device_t dev, device_t child)
  270 {
  271         struct dmar_unit *dmar;
  272         struct dmar_ctx *ctx;
  273         bus_dma_tag_t res;
  274 
  275         dmar = dmar_find(child);
  276         /* Not in scope of any DMAR ? */
  277         if (dmar == NULL)
  278                 return (NULL);
  279         if (!dmar->dma_enabled)
  280                 return (NULL);
  281         dmar_quirks_pre_use(dmar);
  282         dmar_instantiate_rmrr_ctxs(dmar);
  283 
  284         ctx = dmar_instantiate_ctx(dmar, child, false);
  285         res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag;
  286         return (res);
  287 }
  288 
  289 static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map");
  290 
  291 static void dmar_bus_schedule_dmamap(struct dmar_unit *unit,
  292     struct bus_dmamap_dmar *map);
  293 
  294 static int
  295 dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
  296     bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
  297     bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
  298     int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
  299     void *lockfuncarg, bus_dma_tag_t *dmat)
  300 {
  301         struct bus_dma_tag_dmar *newtag, *oldtag;
  302         int error;
  303 
  304         *dmat = NULL;
  305         error = common_bus_dma_tag_create(parent != NULL ?
  306             &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment,
  307             boundary, lowaddr, highaddr, filter, filterarg, maxsize,
  308             nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
  309             sizeof(struct bus_dma_tag_dmar), (void **)&newtag);
  310         if (error != 0)
  311                 goto out;
  312 
  313         oldtag = (struct bus_dma_tag_dmar *)parent;
  314         newtag->common.impl = &bus_dma_dmar_impl;
  315         newtag->ctx = oldtag->ctx;
  316         newtag->owner = oldtag->owner;
  317 
  318         *dmat = (bus_dma_tag_t)newtag;
  319 out:
  320         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
  321             __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
  322             error);
  323         return (error);
  324 }
  325 
  326 static int
  327 dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
  328 {
  329         struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent;
  330         int error;
  331 
  332         error = 0;
  333         dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1;
  334 
  335         if (dmat != NULL) {
  336                 if (dmat->map_count != 0) {
  337                         error = EBUSY;
  338                         goto out;
  339                 }
  340                 while (dmat != NULL) {
  341                         parent = (struct bus_dma_tag_dmar *)dmat->common.parent;
  342                         if (atomic_fetchadd_int(&dmat->common.ref_count, -1) ==
  343                             1) {
  344                                 if (dmat == &dmat->ctx->ctx_tag)
  345                                         dmar_free_ctx(dmat->ctx);
  346                                 free(dmat->segments, M_DMAR_DMAMAP);
  347                                 free(dmat, M_DEVBUF);
  348                                 dmat = parent;
  349                         } else
  350                                 dmat = NULL;
  351                 }
  352         }
  353 out:
  354         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
  355         return (error);
  356 }
  357 
  358 static int
  359 dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
  360 {
  361         struct bus_dma_tag_dmar *tag;
  362         struct bus_dmamap_dmar *map;
  363 
  364         tag = (struct bus_dma_tag_dmar *)dmat;
  365         map = malloc(sizeof(*map), M_DMAR_DMAMAP, M_NOWAIT | M_ZERO);
  366         if (map == NULL) {
  367                 *mapp = NULL;
  368                 return (ENOMEM);
  369         }
  370         if (tag->segments == NULL) {
  371                 tag->segments = malloc(sizeof(bus_dma_segment_t) *
  372                     tag->common.nsegments, M_DMAR_DMAMAP, M_NOWAIT);
  373                 if (tag->segments == NULL) {
  374                         free(map, M_DMAR_DMAMAP);
  375                         *mapp = NULL;
  376                         return (ENOMEM);
  377                 }
  378         }
  379         TAILQ_INIT(&map->map_entries);
  380         map->tag = tag;
  381         map->locked = true;
  382         map->cansleep = false;
  383         tag->map_count++;
  384         *mapp = (bus_dmamap_t)map;
  385 
  386         return (0);
  387 }
  388 
  389 static int
  390 dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
  391 {
  392         struct bus_dma_tag_dmar *tag;
  393         struct bus_dmamap_dmar *map;
  394         struct dmar_domain *domain;
  395 
  396         tag = (struct bus_dma_tag_dmar *)dmat;
  397         map = (struct bus_dmamap_dmar *)map1;
  398         if (map != NULL) {
  399                 domain = tag->ctx->domain;
  400                 DMAR_DOMAIN_LOCK(domain);
  401                 if (!TAILQ_EMPTY(&map->map_entries)) {
  402                         DMAR_DOMAIN_UNLOCK(domain);
  403                         return (EBUSY);
  404                 }
  405                 DMAR_DOMAIN_UNLOCK(domain);
  406                 free(map, M_DMAR_DMAMAP);
  407         }
  408         tag->map_count--;
  409         return (0);
  410 }
  411 
  412 
  413 static int
  414 dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
  415     bus_dmamap_t *mapp)
  416 {
  417         struct bus_dma_tag_dmar *tag;
  418         struct bus_dmamap_dmar *map;
  419         int error, mflags;
  420         vm_memattr_t attr;
  421 
  422         error = dmar_bus_dmamap_create(dmat, flags, mapp);
  423         if (error != 0)
  424                 return (error);
  425 
  426         mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
  427         mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
  428         attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
  429             VM_MEMATTR_DEFAULT;
  430 
  431         tag = (struct bus_dma_tag_dmar *)dmat;
  432         map = (struct bus_dmamap_dmar *)*mapp;
  433 
  434         if (tag->common.maxsize < PAGE_SIZE &&
  435             tag->common.alignment <= tag->common.maxsize &&
  436             attr == VM_MEMATTR_DEFAULT) {
  437                 *vaddr = malloc(tag->common.maxsize, M_DEVBUF, mflags);
  438                 map->flags |= BUS_DMAMAP_DMAR_MALLOC;
  439         } else {
  440                 *vaddr = (void *)kmem_alloc_attr(kernel_arena,
  441                     tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR,
  442                     attr);
  443                 map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC;
  444         }
  445         if (*vaddr == NULL) {
  446                 dmar_bus_dmamap_destroy(dmat, *mapp);
  447                 *mapp = NULL;
  448                 return (ENOMEM);
  449         }
  450         return (0);
  451 }
  452 
  453 static void
  454 dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
  455 {
  456         struct bus_dma_tag_dmar *tag;
  457         struct bus_dmamap_dmar *map;
  458 
  459         tag = (struct bus_dma_tag_dmar *)dmat;
  460         map = (struct bus_dmamap_dmar *)map1;
  461 
  462         if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) {
  463                 free(vaddr, M_DEVBUF);
  464                 map->flags &= ~BUS_DMAMAP_DMAR_MALLOC;
  465         } else {
  466                 KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0,
  467                     ("dmar_bus_dmamem_free for non alloced map %p", map));
  468                 kmem_free(kernel_arena, (vm_offset_t)vaddr, tag->common.maxsize);
  469                 map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC;
  470         }
  471 
  472         dmar_bus_dmamap_destroy(dmat, map1);
  473 }
  474 
  475 static int
  476 dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag,
  477     struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
  478     int flags, bus_dma_segment_t *segs, int *segp,
  479     struct dmar_map_entries_tailq *unroll_list)
  480 {
  481         struct dmar_ctx *ctx;
  482         struct dmar_domain *domain;
  483         struct dmar_map_entry *entry;
  484         dmar_gaddr_t size;
  485         bus_size_t buflen1;
  486         int error, idx, gas_flags, seg;
  487 
  488         KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset));
  489         if (segs == NULL)
  490                 segs = tag->segments;
  491         ctx = tag->ctx;
  492         domain = ctx->domain;
  493         seg = *segp;
  494         error = 0;
  495         idx = 0;
  496         while (buflen > 0) {
  497                 seg++;
  498                 if (seg >= tag->common.nsegments) {
  499                         error = EFBIG;
  500                         break;
  501                 }
  502                 buflen1 = buflen > tag->common.maxsegsz ?
  503                     tag->common.maxsegsz : buflen;
  504                 size = round_page(offset + buflen1);
  505 
  506                 /*
  507                  * (Too) optimistically allow split if there are more
  508                  * then one segments left.
  509                  */
  510                 gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0;
  511                 if (seg + 1 < tag->common.nsegments)
  512                         gas_flags |= DMAR_GM_CANSPLIT;
  513 
  514                 error = dmar_gas_map(domain, &tag->common, size, offset,
  515                     DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE,
  516                     gas_flags, ma + idx, &entry);
  517                 if (error != 0)
  518                         break;
  519                 if ((gas_flags & DMAR_GM_CANSPLIT) != 0) {
  520                         KASSERT(size >= entry->end - entry->start,
  521                             ("split increased entry size %jx %jx %jx",
  522                             (uintmax_t)size, (uintmax_t)entry->start,
  523                             (uintmax_t)entry->end));
  524                         size = entry->end - entry->start;
  525                         if (buflen1 > size)
  526                                 buflen1 = size;
  527                 } else {
  528                         KASSERT(entry->end - entry->start == size,
  529                             ("no split allowed %jx %jx %jx",
  530                             (uintmax_t)size, (uintmax_t)entry->start,
  531                             (uintmax_t)entry->end));
  532                 }
  533                 if (offset + buflen1 > size)
  534                         buflen1 = size - offset;
  535                 if (buflen1 > tag->common.maxsegsz)
  536                         buflen1 = tag->common.maxsegsz;
  537 
  538                 KASSERT(((entry->start + offset) & (tag->common.alignment - 1))
  539                     == 0,
  540                     ("alignment failed: ctx %p start 0x%jx offset %x "
  541                     "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
  542                     (uintmax_t)tag->common.alignment));
  543                 KASSERT(entry->end <= tag->common.lowaddr ||
  544                     entry->start >= tag->common.highaddr,
  545                     ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
  546                     "lowaddr 0x%jx highaddr 0x%jx", ctx,
  547                     (uintmax_t)entry->start, (uintmax_t)entry->end,
  548                     (uintmax_t)tag->common.lowaddr,
  549                     (uintmax_t)tag->common.highaddr));
  550                 KASSERT(dmar_test_boundary(entry->start + offset, buflen1,
  551                     tag->common.boundary),
  552                     ("boundary failed: ctx %p start 0x%jx end 0x%jx "
  553                     "boundary 0x%jx", ctx, (uintmax_t)entry->start,
  554                     (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
  555                 KASSERT(buflen1 <= tag->common.maxsegsz,
  556                     ("segment too large: ctx %p start 0x%jx end 0x%jx "
  557                     "buflen1 0x%jx maxsegsz 0x%jx", ctx,
  558                     (uintmax_t)entry->start, (uintmax_t)entry->end,
  559                     (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));
  560 
  561                 DMAR_DOMAIN_LOCK(domain);
  562                 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
  563                 entry->flags |= DMAR_MAP_ENTRY_MAP;
  564                 DMAR_DOMAIN_UNLOCK(domain);
  565                 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);
  566 
  567                 segs[seg].ds_addr = entry->start + offset;
  568                 segs[seg].ds_len = buflen1;
  569 
  570                 idx += OFF_TO_IDX(trunc_page(offset + buflen1));
  571                 offset += buflen1;
  572                 offset &= DMAR_PAGE_MASK;
  573                 buflen -= buflen1;
  574         }
  575         if (error == 0)
  576                 *segp = seg;
  577         return (error);
  578 }
  579 
  580 static int
  581 dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag,
  582     struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
  583     int flags, bus_dma_segment_t *segs, int *segp)
  584 {
  585         struct dmar_ctx *ctx;
  586         struct dmar_domain *domain;
  587         struct dmar_map_entry *entry, *entry1;
  588         struct dmar_map_entries_tailq unroll_list;
  589         int error;
  590 
  591         ctx = tag->ctx;
  592         domain = ctx->domain;
  593         atomic_add_long(&ctx->loads, 1);
  594 
  595         TAILQ_INIT(&unroll_list);
  596         error = dmar_bus_dmamap_load_something1(tag, map, ma, offset,
  597             buflen, flags, segs, segp, &unroll_list);
  598         if (error != 0) {
  599                 /*
  600                  * The busdma interface does not allow us to report
  601                  * partial buffer load, so unfortunately we have to
  602                  * revert all work done.
  603                  */
  604                 DMAR_DOMAIN_LOCK(domain);
  605                 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link,
  606                     entry1) {
  607                         /*
  608                          * No entries other than what we have created
  609                          * during the failed run might have been
  610                          * inserted there in between, since we own ctx
  611                          * pglock.
  612                          */
  613                         TAILQ_REMOVE(&map->map_entries, entry, dmamap_link);
  614                         TAILQ_REMOVE(&unroll_list, entry, unroll_link);
  615                         TAILQ_INSERT_TAIL(&domain->unload_entries, entry,
  616                             dmamap_link);
  617                 }
  618                 DMAR_DOMAIN_UNLOCK(domain);
  619                 taskqueue_enqueue(domain->dmar->delayed_taskqueue,
  620                     &domain->unload_task);
  621         }
  622 
  623         if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
  624             !map->cansleep)
  625                 error = EINPROGRESS;
  626         if (error == EINPROGRESS)
  627                 dmar_bus_schedule_dmamap(domain->dmar, map);
  628         return (error);
  629 }
  630 
  631 static int
  632 dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
  633     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
  634     bus_dma_segment_t *segs, int *segp)
  635 {
  636         struct bus_dma_tag_dmar *tag;
  637         struct bus_dmamap_dmar *map;
  638 
  639         tag = (struct bus_dma_tag_dmar *)dmat;
  640         map = (struct bus_dmamap_dmar *)map1;
  641         return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
  642             flags, segs, segp));
  643 }
  644 
  645 static int
  646 dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
  647     vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
  648     int *segp)
  649 {
  650         struct bus_dma_tag_dmar *tag;
  651         struct bus_dmamap_dmar *map;
  652         vm_page_t *ma;
  653         vm_paddr_t pstart, pend;
  654         int error, i, ma_cnt, offset;
  655 
  656         tag = (struct bus_dma_tag_dmar *)dmat;
  657         map = (struct bus_dmamap_dmar *)map1;
  658         pstart = trunc_page(buf);
  659         pend = round_page(buf + buflen);
  660         offset = buf & PAGE_MASK;
  661         ma_cnt = OFF_TO_IDX(pend - pstart);
  662         ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
  663             M_WAITOK : M_NOWAIT);
  664         if (ma == NULL)
  665                 return (ENOMEM);
  666         for (i = 0; i < ma_cnt; i++)
  667                 ma[i] = PHYS_TO_VM_PAGE(pstart + i * PAGE_SIZE);
  668         error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
  669             flags, segs, segp);
  670         free(ma, M_DEVBUF);
  671         return (error);
  672 }
  673 
  674 static int
  675 dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
  676     bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
  677     int *segp)
  678 {
  679         struct bus_dma_tag_dmar *tag;
  680         struct bus_dmamap_dmar *map;
  681         vm_page_t *ma, fma;
  682         vm_paddr_t pstart, pend, paddr;
  683         int error, i, ma_cnt, offset;
  684 
  685         tag = (struct bus_dma_tag_dmar *)dmat;
  686         map = (struct bus_dmamap_dmar *)map1;
  687         pstart = trunc_page((vm_offset_t)buf);
  688         pend = round_page((vm_offset_t)buf + buflen);
  689         offset = (vm_offset_t)buf & PAGE_MASK;
  690         ma_cnt = OFF_TO_IDX(pend - pstart);
  691         ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
  692             M_WAITOK : M_NOWAIT);
  693         if (ma == NULL)
  694                 return (ENOMEM);
  695         if (dumping) {
  696                 /*
  697                  * If dumping, do not attempt to call
  698                  * PHYS_TO_VM_PAGE() at all.  It may return non-NULL
  699                  * but the vm_page returned might be not initialized,
  700                  * e.g. for the kernel itself.
  701                  */
  702                 KASSERT(pmap == kernel_pmap, ("non-kernel address write"));
  703                 fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF,
  704                     M_ZERO | (map->cansleep ? M_WAITOK : M_NOWAIT));
  705                 if (fma == NULL) {
  706                         free(ma, M_DEVBUF);
  707                         return (ENOMEM);
  708                 }
  709                 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
  710                         paddr = pmap_kextract(pstart);
  711                         vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
  712                         ma[i] = &fma[i];
  713                 }
  714         } else {
  715                 fma = NULL;
  716                 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
  717                         if (pmap == kernel_pmap)
  718                                 paddr = pmap_kextract(pstart);
  719                         else
  720                                 paddr = pmap_extract(pmap, pstart);
  721                         ma[i] = PHYS_TO_VM_PAGE(paddr);
  722                         KASSERT(VM_PAGE_TO_PHYS(ma[i]) == paddr,
  723                             ("PHYS_TO_VM_PAGE failed %jx %jx m %p",
  724                             (uintmax_t)paddr, (uintmax_t)VM_PAGE_TO_PHYS(ma[i]),
  725                             ma[i]));
  726                 }
  727         }
  728         error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
  729             flags, segs, segp);
  730         free(ma, M_DEVBUF);
  731         free(fma, M_DEVBUF);
  732         return (error);
  733 }
  734 
  735 static void
  736 dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
  737     struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
  738 {
  739         struct bus_dmamap_dmar *map;
  740 
  741         if (map1 == NULL)
  742                 return;
  743         map = (struct bus_dmamap_dmar *)map1;
  744         map->mem = *mem;
  745         map->tag = (struct bus_dma_tag_dmar *)dmat;
  746         map->callback = callback;
  747         map->callback_arg = callback_arg;
  748 }
  749 
  750 static bus_dma_segment_t *
  751 dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
  752     bus_dma_segment_t *segs, int nsegs, int error)
  753 {
  754         struct bus_dma_tag_dmar *tag;
  755         struct bus_dmamap_dmar *map;
  756 
  757         tag = (struct bus_dma_tag_dmar *)dmat;
  758         map = (struct bus_dmamap_dmar *)map1;
  759 
  760         if (!map->locked) {
  761                 KASSERT(map->cansleep,
  762                     ("map not locked and not sleepable context %p", map));
  763 
  764                 /*
  765                  * We are called from the delayed context.  Relock the
  766                  * driver.
  767                  */
  768                 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
  769                 map->locked = true;
  770         }
  771 
  772         if (segs == NULL)
  773                 segs = tag->segments;
  774         return (segs);
  775 }
  776 
  777 /*
  778  * The limitations of busdma KPI forces the dmar to perform the actual
  779  * unload, consisting of the unmapping of the map entries page tables,
  780  * from the delayed context on i386, since page table page mapping
  781  * might require a sleep to be successfull.  The unfortunate
  782  * consequence is that the DMA requests can be served some time after
  783  * the bus_dmamap_unload() call returned.
  784  *
  785  * On amd64, we assume that sf allocation cannot fail.
  786  */
  787 static void
  788 dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
  789 {
  790         struct bus_dma_tag_dmar *tag;
  791         struct bus_dmamap_dmar *map;
  792         struct dmar_ctx *ctx;
  793         struct dmar_domain *domain;
  794 #if defined(__amd64__)
  795         struct dmar_map_entries_tailq entries;
  796 #endif
  797 
  798         tag = (struct bus_dma_tag_dmar *)dmat;
  799         map = (struct bus_dmamap_dmar *)map1;
  800         ctx = tag->ctx;
  801         domain = ctx->domain;
  802         atomic_add_long(&ctx->unloads, 1);
  803 
  804 #if defined(__i386__)
  805         DMAR_DOMAIN_LOCK(domain);
  806         TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link);
  807         DMAR_DOMAIN_UNLOCK(domain);
  808         taskqueue_enqueue(domain->dmar->delayed_taskqueue,
  809             &domain->unload_task);
  810 #else /* defined(__amd64__) */
  811         TAILQ_INIT(&entries);
  812         DMAR_DOMAIN_LOCK(domain);
  813         TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
  814         DMAR_DOMAIN_UNLOCK(domain);
  815         THREAD_NO_SLEEPING();
  816         dmar_domain_unload(domain, &entries, false);
  817         THREAD_SLEEPING_OK();
  818         KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx));
  819 #endif
  820 }
  821 
  822 static void
  823 dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
  824     bus_dmasync_op_t op)
  825 {
  826 }
  827 
  828 struct bus_dma_impl bus_dma_dmar_impl = {
  829         .tag_create = dmar_bus_dma_tag_create,
  830         .tag_destroy = dmar_bus_dma_tag_destroy,
  831         .map_create = dmar_bus_dmamap_create,
  832         .map_destroy = dmar_bus_dmamap_destroy,
  833         .mem_alloc = dmar_bus_dmamem_alloc,
  834         .mem_free = dmar_bus_dmamem_free,
  835         .load_phys = dmar_bus_dmamap_load_phys,
  836         .load_buffer = dmar_bus_dmamap_load_buffer,
  837         .load_ma = dmar_bus_dmamap_load_ma,
  838         .map_waitok = dmar_bus_dmamap_waitok,
  839         .map_complete = dmar_bus_dmamap_complete,
  840         .map_unload = dmar_bus_dmamap_unload,
  841         .map_sync = dmar_bus_dmamap_sync
  842 };
  843 
  844 static void
  845 dmar_bus_task_dmamap(void *arg, int pending)
  846 {
  847         struct bus_dma_tag_dmar *tag;
  848         struct bus_dmamap_dmar *map;
  849         struct dmar_unit *unit;
  850 
  851         unit = arg;
  852         DMAR_LOCK(unit);
  853         while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
  854                 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
  855                 DMAR_UNLOCK(unit);
  856                 tag = map->tag;
  857                 map->cansleep = true;
  858                 map->locked = false;
  859                 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
  860                     &map->mem, map->callback, map->callback_arg,
  861                     BUS_DMA_WAITOK);
  862                 map->cansleep = false;
  863                 if (map->locked) {
  864                         (tag->common.lockfunc)(tag->common.lockfuncarg,
  865                             BUS_DMA_UNLOCK);
  866                 } else
  867                         map->locked = true;
  868                 map->cansleep = false;
  869                 DMAR_LOCK(unit);
  870         }
  871         DMAR_UNLOCK(unit);
  872 }
  873 
  874 static void
  875 dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map)
  876 {
  877 
  878         map->locked = false;
  879         DMAR_LOCK(unit);
  880         TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
  881         DMAR_UNLOCK(unit);
  882         taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
  883 }
  884 
  885 int
  886 dmar_init_busdma(struct dmar_unit *unit)
  887 {
  888 
  889         unit->dma_enabled = 1;
  890         TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
  891         TAILQ_INIT(&unit->delayed_maps);
  892         TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit);
  893         unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK,
  894             taskqueue_thread_enqueue, &unit->delayed_taskqueue);
  895         taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
  896             "dmar%d busdma taskq", unit->unit);
  897         return (0);
  898 }
  899 
  900 void
  901 dmar_fini_busdma(struct dmar_unit *unit)
  902 {
  903 
  904         if (unit->delayed_taskqueue == NULL)
  905                 return;
  906 
  907         taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
  908         taskqueue_free(unit->delayed_taskqueue);
  909         unit->delayed_taskqueue = NULL;
  910 }

Cache object: 2cd875e0ea30e98b72d2cf2f96c3b820


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.