The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/intel_ctx.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2013 The FreeBSD Foundation
    5  * All rights reserved.
    6  *
    7  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    8  * under sponsorship from the FreeBSD Foundation.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   29  * SUCH DAMAGE.
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 
   35 #include <sys/param.h>
   36 #include <sys/systm.h>
   37 #include <sys/malloc.h>
   38 #include <sys/bus.h>
   39 #include <sys/interrupt.h>
   40 #include <sys/kernel.h>
   41 #include <sys/ktr.h>
   42 #include <sys/limits.h>
   43 #include <sys/lock.h>
   44 #include <sys/memdesc.h>
   45 #include <sys/mutex.h>
   46 #include <sys/proc.h>
   47 #include <sys/rwlock.h>
   48 #include <sys/rman.h>
   49 #include <sys/sysctl.h>
   50 #include <sys/taskqueue.h>
   51 #include <sys/tree.h>
   52 #include <sys/uio.h>
   53 #include <sys/vmem.h>
   54 #include <vm/vm.h>
   55 #include <vm/vm_extern.h>
   56 #include <vm/vm_kern.h>
   57 #include <vm/vm_object.h>
   58 #include <vm/vm_page.h>
   59 #include <vm/vm_pager.h>
   60 #include <vm/vm_map.h>
   61 #include <contrib/dev/acpica/include/acpi.h>
   62 #include <contrib/dev/acpica/include/accommon.h>
   63 #include <dev/pci/pcireg.h>
   64 #include <dev/pci/pcivar.h>
   65 #include <machine/atomic.h>
   66 #include <machine/bus.h>
   67 #include <machine/md_var.h>
   68 #include <machine/specialreg.h>
   69 #include <x86/include/busdma_impl.h>
   70 #include <dev/iommu/busdma_iommu.h>
   71 #include <x86/iommu/intel_reg.h>
   72 #include <x86/iommu/intel_dmar.h>
   73 
   74 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
   75 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain");
   76 
   77 static void dmar_unref_domain_locked(struct dmar_unit *dmar,
   78     struct dmar_domain *domain);
   79 static void dmar_domain_destroy(struct dmar_domain *domain);
   80 
   81 static void
   82 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
   83 {
   84         struct sf_buf *sf;
   85         dmar_root_entry_t *re;
   86         vm_page_t ctxm;
   87 
   88         /*
   89          * Allocated context page must be linked.
   90          */
   91         ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC);
   92         if (ctxm != NULL)
   93                 return;
   94 
   95         /*
   96          * Page not present, allocate and link.  Note that other
   97          * thread might execute this sequence in parallel.  This
   98          * should be safe, because the context entries written by both
   99          * threads are equal.
  100          */
  101         TD_PREP_PINNED_ASSERT;
  102         ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO |
  103             IOMMU_PGF_WAITOK);
  104         re = dmar_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf);
  105         re += bus;
  106         dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK &
  107             VM_PAGE_TO_PHYS(ctxm)));
  108         dmar_flush_root_to_ram(dmar, re);
  109         dmar_unmap_pgtbl(sf);
  110         TD_PINNED_ASSERT;
  111 }
  112 
  113 static dmar_ctx_entry_t *
  114 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp)
  115 {
  116         struct dmar_unit *dmar;
  117         dmar_ctx_entry_t *ctxp;
  118 
  119         dmar = CTX2DMAR(ctx);
  120 
  121         ctxp = dmar_map_pgtbl(dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->context.rid),
  122             IOMMU_PGF_NOALLOC | IOMMU_PGF_WAITOK, sfp);
  123         ctxp += ctx->context.rid & 0xff;
  124         return (ctxp);
  125 }
  126 
  127 static void
  128 device_tag_init(struct dmar_ctx *ctx, device_t dev)
  129 {
  130         struct dmar_domain *domain;
  131         bus_addr_t maxaddr;
  132 
  133         domain = CTX2DOM(ctx);
  134         maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR);
  135         ctx->context.tag->common.ref_count = 1; /* Prevent free */
  136         ctx->context.tag->common.impl = &bus_dma_iommu_impl;
  137         ctx->context.tag->common.boundary = 0;
  138         ctx->context.tag->common.lowaddr = maxaddr;
  139         ctx->context.tag->common.highaddr = maxaddr;
  140         ctx->context.tag->common.maxsize = maxaddr;
  141         ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED;
  142         ctx->context.tag->common.maxsegsz = maxaddr;
  143         ctx->context.tag->ctx = CTX2IOCTX(ctx);
  144         ctx->context.tag->owner = dev;
  145 }
  146 
  147 static void
  148 ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain,
  149     vm_page_t ctx_root)
  150 {
  151         /*
  152          * For update due to move, the store is not atomic.  It is
  153          * possible that DMAR read upper doubleword, while low
  154          * doubleword is not yet updated.  The domain id is stored in
  155          * the upper doubleword, while the table pointer in the lower.
  156          *
  157          * There is no good solution, for the same reason it is wrong
  158          * to clear P bit in the ctx entry for update.
  159          */
  160         dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) |
  161             domain->awlvl);
  162         if (ctx_root == NULL) {
  163                 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P);
  164         } else {
  165                 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR |
  166                     (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) |
  167                     DMAR_CTX1_P);
  168         }
  169 }
  170 
  171 static void
  172 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move,
  173     int busno)
  174 {
  175         struct dmar_unit *unit;
  176         struct dmar_domain *domain;
  177         vm_page_t ctx_root;
  178         int i;
  179 
  180         domain = CTX2DOM(ctx);
  181         unit = DOM2DMAR(domain);
  182         KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0),
  183             ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
  184             unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner),
  185             pci_get_function(ctx->context.tag->owner),
  186             ctxp->ctx1, ctxp->ctx2));
  187 
  188         if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 &&
  189             (unit->hw_ecap & DMAR_ECAP_PT) != 0) {
  190                 KASSERT(domain->pgtbl_obj == NULL,
  191                     ("ctx %p non-null pgtbl_obj", ctx));
  192                 ctx_root = NULL;
  193         } else {
  194                 ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0,
  195                     IOMMU_PGF_NOALLOC);
  196         }
  197 
  198         if (iommu_is_buswide_ctx(DMAR2IOMMU(unit), busno)) {
  199                 MPASS(!move);
  200                 for (i = 0; i <= PCI_BUSMAX; i++) {
  201                         ctx_id_entry_init_one(&ctxp[i], domain, ctx_root);
  202                 }
  203         } else {
  204                 ctx_id_entry_init_one(ctxp, domain, ctx_root);
  205         }
  206         dmar_flush_ctx_to_ram(unit, ctxp);
  207 }
  208 
  209 static int
  210 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force)
  211 {
  212         int error;
  213 
  214         /*
  215          * If dmar declares Caching Mode as Set, follow 11.5 "Caching
  216          * Mode Consideration" and do the (global) invalidation of the
  217          * negative TLB entries.
  218          */
  219         if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force)
  220                 return (0);
  221         if (dmar->qi_enabled) {
  222                 dmar_qi_invalidate_ctx_glob_locked(dmar);
  223                 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)
  224                         dmar_qi_invalidate_iotlb_glob_locked(dmar);
  225                 return (0);
  226         }
  227         error = dmar_inv_ctx_glob(dmar);
  228         if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force))
  229                 error = dmar_inv_iotlb_glob(dmar);
  230         return (error);
  231 }
  232 
  233 static int
  234 domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus,
  235     int slot, int func, int dev_domain, int dev_busno,
  236     const void *dev_path, int dev_path_len)
  237 {
  238         struct iommu_map_entries_tailq rmrr_entries;
  239         struct iommu_map_entry *entry, *entry1;
  240         vm_page_t *ma;
  241         iommu_gaddr_t start, end;
  242         vm_pindex_t size, i;
  243         int error, error1;
  244 
  245         error = 0;
  246         TAILQ_INIT(&rmrr_entries);
  247         dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path,
  248             dev_path_len, &rmrr_entries);
  249         TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {
  250                 /*
  251                  * VT-d specification requires that the start of an
  252                  * RMRR entry is 4k-aligned.  Buggy BIOSes put
  253                  * anything into the start and end fields.  Truncate
  254                  * and round as neccesary.
  255                  *
  256                  * We also allow the overlapping RMRR entries, see
  257                  * iommu_gas_alloc_region().
  258                  */
  259                 start = entry->start;
  260                 end = entry->end;
  261                 if (bootverbose)
  262                         printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n",
  263                             domain->iodom.iommu->unit, bus, slot, func,
  264                             (uintmax_t)start, (uintmax_t)end);
  265                 entry->start = trunc_page(start);
  266                 entry->end = round_page(end);
  267                 if (entry->start == entry->end) {
  268                         /* Workaround for some AMI (?) BIOSes */
  269                         if (bootverbose) {
  270                                 if (dev != NULL)
  271                                         device_printf(dev, "");
  272                                 printf("pci%d:%d:%d ", bus, slot, func);
  273                                 printf("BIOS bug: dmar%d RMRR "
  274                                     "region (%jx, %jx) corrected\n",
  275                                     domain->iodom.iommu->unit, start, end);
  276                         }
  277                         entry->end += DMAR_PAGE_SIZE * 0x20;
  278                 }
  279                 size = OFF_TO_IDX(entry->end - entry->start);
  280                 ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
  281                 for (i = 0; i < size; i++) {
  282                         ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
  283                             VM_MEMATTR_DEFAULT);
  284                 }
  285                 error1 = iommu_gas_map_region(DOM2IODOM(domain), entry,
  286                     IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE,
  287                     IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma);
  288                 /*
  289                  * Non-failed RMRR entries are owned by context rb
  290                  * tree.  Get rid of the failed entry, but do not stop
  291                  * the loop.  Rest of the parsed RMRR entries are
  292                  * loaded and removed on the context destruction.
  293                  */
  294                 if (error1 == 0 && entry->end != entry->start) {
  295                         IOMMU_LOCK(domain->iodom.iommu);
  296                         domain->refs++; /* XXXKIB prevent free */
  297                         domain->iodom.flags |= IOMMU_DOMAIN_RMRR;
  298                         IOMMU_UNLOCK(domain->iodom.iommu);
  299                 } else {
  300                         if (error1 != 0) {
  301                                 if (dev != NULL)
  302                                         device_printf(dev, "");
  303                                 printf("pci%d:%d:%d ", bus, slot, func);
  304                                 printf(
  305                             "dmar%d failed to map RMRR region (%jx, %jx) %d\n",
  306                                     domain->iodom.iommu->unit, start, end,
  307                                     error1);
  308                                 error = error1;
  309                         }
  310                         TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);
  311                         iommu_gas_free_entry(DOM2IODOM(domain), entry);
  312                 }
  313                 for (i = 0; i < size; i++)
  314                         vm_page_putfake(ma[i]);
  315                 free(ma, M_TEMP);
  316         }
  317         return (error);
  318 }
  319 
  320 /*
  321  * PCI memory address space is shared between memory-mapped devices (MMIO) and
  322  * host memory (which may be remapped by an IOMMU).  Device accesses to an
  323  * address within a memory aperture in a PCIe root port will be treated as
  324  * peer-to-peer and not forwarded to an IOMMU.  To avoid this, reserve the
  325  * address space of the root port's memory apertures in the address space used
  326  * by the IOMMU for remapping.
  327  */
  328 static int
  329 dmar_reserve_pci_regions(struct dmar_domain *domain, device_t dev)
  330 {
  331         struct iommu_domain *iodom;
  332         device_t root;
  333         uint32_t val;
  334         uint64_t base, limit;
  335         int error;
  336 
  337         iodom = DOM2IODOM(domain);
  338 
  339         root = pci_find_pcie_root_port(dev);
  340         if (root == NULL)
  341                 return (0);
  342 
  343         /* Disable downstream memory */
  344         base = PCI_PPBMEMBASE(0, pci_read_config(root, PCIR_MEMBASE_1, 2));
  345         limit = PCI_PPBMEMLIMIT(0, pci_read_config(root, PCIR_MEMLIMIT_1, 2));
  346         error = iommu_gas_reserve_region_extend(iodom, base, limit + 1);
  347         if (bootverbose || error != 0)
  348                 device_printf(dev, "DMAR reserve [%#jx-%#jx] (error %d)\n",
  349                     base, limit + 1, error);
  350         if (error != 0)
  351                 return (error);
  352 
  353         /* Disable downstream prefetchable memory */
  354         val = pci_read_config(root, PCIR_PMBASEL_1, 2);
  355         if (val != 0 || pci_read_config(root, PCIR_PMLIMITL_1, 2) != 0) {
  356                 if ((val & PCIM_BRPM_MASK) == PCIM_BRPM_64) {
  357                         base = PCI_PPBMEMBASE(
  358                             pci_read_config(root, PCIR_PMBASEH_1, 4),
  359                             val);
  360                         limit = PCI_PPBMEMLIMIT(
  361                             pci_read_config(root, PCIR_PMLIMITH_1, 4),
  362                             pci_read_config(root, PCIR_PMLIMITL_1, 2));
  363                 } else {
  364                         base = PCI_PPBMEMBASE(0, val);
  365                         limit = PCI_PPBMEMLIMIT(0,
  366                             pci_read_config(root, PCIR_PMLIMITL_1, 2));
  367                 }
  368                 error = iommu_gas_reserve_region_extend(iodom, base,
  369                     limit + 1);
  370                 if (bootverbose || error != 0)
  371                         device_printf(dev, "DMAR reserve [%#jx-%#jx] "
  372                             "(error %d)\n", base, limit + 1, error);
  373                 if (error != 0)
  374                         return (error);
  375         }
  376 
  377         return (error);
  378 }
  379 
  380 static struct dmar_domain *
  381 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped)
  382 {
  383         struct iommu_domain *iodom;
  384         struct iommu_unit *unit;
  385         struct dmar_domain *domain;
  386         int error, id, mgaw;
  387 
  388         id = alloc_unr(dmar->domids);
  389         if (id == -1)
  390                 return (NULL);
  391         domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO);
  392         iodom = DOM2IODOM(domain);
  393         unit = DMAR2IOMMU(dmar);
  394         domain->domain = id;
  395         LIST_INIT(&domain->contexts);
  396         iommu_domain_init(unit, iodom, &dmar_domain_map_ops);
  397 
  398         domain->dmar = dmar;
  399 
  400         /*
  401          * For now, use the maximal usable physical address of the
  402          * installed memory to calculate the mgaw on id_mapped domain.
  403          * It is useful for the identity mapping, and less so for the
  404          * virtualized bus address space.
  405          */
  406         domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;
  407         mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped);
  408         error = domain_set_agaw(domain, mgaw);
  409         if (error != 0)
  410                 goto fail;
  411         if (!id_mapped)
  412                 /* Use all supported address space for remapping. */
  413                 domain->iodom.end = 1ULL << (domain->agaw - 1);
  414 
  415         iommu_gas_init_domain(DOM2IODOM(domain));
  416 
  417         if (id_mapped) {
  418                 if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
  419                         domain->pgtbl_obj = domain_get_idmap_pgtbl(domain,
  420                             domain->iodom.end);
  421                 }
  422                 domain->iodom.flags |= IOMMU_DOMAIN_IDMAP;
  423         } else {
  424                 error = domain_alloc_pgtbl(domain);
  425                 if (error != 0)
  426                         goto fail;
  427                 /* Disable local apic region access */
  428                 error = iommu_gas_reserve_region(iodom, 0xfee00000,
  429                     0xfeefffff + 1, &iodom->msi_entry);
  430                 if (error != 0)
  431                         goto fail;
  432         }
  433         return (domain);
  434 
  435 fail:
  436         dmar_domain_destroy(domain);
  437         return (NULL);
  438 }
  439 
  440 static struct dmar_ctx *
  441 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid)
  442 {
  443         struct dmar_ctx *ctx;
  444 
  445         ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO);
  446         ctx->context.domain = DOM2IODOM(domain);
  447         ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu),
  448             M_DMAR_CTX, M_WAITOK | M_ZERO);
  449         ctx->context.rid = rid;
  450         ctx->refs = 1;
  451         return (ctx);
  452 }
  453 
  454 static void
  455 dmar_ctx_link(struct dmar_ctx *ctx)
  456 {
  457         struct dmar_domain *domain;
  458 
  459         domain = CTX2DOM(ctx);
  460         IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
  461         KASSERT(domain->refs >= domain->ctx_cnt,
  462             ("dom %p ref underflow %d %d", domain, domain->refs,
  463             domain->ctx_cnt));
  464         domain->refs++;
  465         domain->ctx_cnt++;
  466         LIST_INSERT_HEAD(&domain->contexts, ctx, link);
  467 }
  468 
  469 static void
  470 dmar_ctx_unlink(struct dmar_ctx *ctx)
  471 {
  472         struct dmar_domain *domain;
  473 
  474         domain = CTX2DOM(ctx);
  475         IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
  476         KASSERT(domain->refs > 0,
  477             ("domain %p ctx dtr refs %d", domain, domain->refs));
  478         KASSERT(domain->ctx_cnt >= domain->refs,
  479             ("domain %p ctx dtr refs %d ctx_cnt %d", domain,
  480             domain->refs, domain->ctx_cnt));
  481         domain->refs--;
  482         domain->ctx_cnt--;
  483         LIST_REMOVE(ctx, link);
  484 }
  485 
  486 static void
  487 dmar_domain_destroy(struct dmar_domain *domain)
  488 {
  489         struct iommu_domain *iodom;
  490         struct dmar_unit *dmar;
  491 
  492         iodom = DOM2IODOM(domain);
  493 
  494         KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries),
  495             ("unfinished unloads %p", domain));
  496         KASSERT(LIST_EMPTY(&domain->contexts),
  497             ("destroying dom %p with contexts", domain));
  498         KASSERT(domain->ctx_cnt == 0,
  499             ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));
  500         KASSERT(domain->refs == 0,
  501             ("destroying dom %p with refs %d", domain, domain->refs));
  502         if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
  503                 DMAR_DOMAIN_LOCK(domain);
  504                 iommu_gas_fini_domain(iodom);
  505                 DMAR_DOMAIN_UNLOCK(domain);
  506         }
  507         if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) {
  508                 if (domain->pgtbl_obj != NULL)
  509                         DMAR_DOMAIN_PGLOCK(domain);
  510                 domain_free_pgtbl(domain);
  511         }
  512         iommu_domain_fini(iodom);
  513         dmar = DOM2DMAR(domain);
  514         free_unr(dmar->domids, domain->domain);
  515         free(domain, M_DMAR_DOMAIN);
  516 }
  517 
  518 static struct dmar_ctx *
  519 dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid,
  520     int dev_domain, int dev_busno, const void *dev_path, int dev_path_len,
  521     bool id_mapped, bool rmrr_init)
  522 {
  523         struct dmar_domain *domain, *domain1;
  524         struct dmar_ctx *ctx, *ctx1;
  525         struct iommu_unit *unit __diagused;
  526         dmar_ctx_entry_t *ctxp;
  527         struct sf_buf *sf;
  528         int bus, slot, func, error;
  529         bool enable;
  530 
  531         if (dev != NULL) {
  532                 bus = pci_get_bus(dev);
  533                 slot = pci_get_slot(dev);
  534                 func = pci_get_function(dev);
  535         } else {
  536                 bus = PCI_RID2BUS(rid);
  537                 slot = PCI_RID2SLOT(rid);
  538                 func = PCI_RID2FUNC(rid);
  539         }
  540         enable = false;
  541         TD_PREP_PINNED_ASSERT;
  542         unit = DMAR2IOMMU(dmar);
  543         DMAR_LOCK(dmar);
  544         KASSERT(!iommu_is_buswide_ctx(unit, bus) || (slot == 0 && func == 0),
  545             ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus,
  546             slot, func));
  547         ctx = dmar_find_ctx_locked(dmar, rid);
  548         error = 0;
  549         if (ctx == NULL) {
  550                 /*
  551                  * Perform the allocations which require sleep or have
  552                  * higher chance to succeed if the sleep is allowed.
  553                  */
  554                 DMAR_UNLOCK(dmar);
  555                 dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid));
  556                 domain1 = dmar_domain_alloc(dmar, id_mapped);
  557                 if (domain1 == NULL) {
  558                         TD_PINNED_ASSERT;
  559                         return (NULL);
  560                 }
  561                 if (!id_mapped) {
  562                         error = domain_init_rmrr(domain1, dev, bus,
  563                             slot, func, dev_domain, dev_busno, dev_path,
  564                             dev_path_len);
  565                         if (error == 0)
  566                                 error = dmar_reserve_pci_regions(domain1, dev);
  567                         if (error != 0) {
  568                                 dmar_domain_destroy(domain1);
  569                                 TD_PINNED_ASSERT;
  570                                 return (NULL);
  571                         }
  572                 }
  573                 ctx1 = dmar_ctx_alloc(domain1, rid);
  574                 ctxp = dmar_map_ctx_entry(ctx1, &sf);
  575                 DMAR_LOCK(dmar);
  576 
  577                 /*
  578                  * Recheck the contexts, other thread might have
  579                  * already allocated needed one.
  580                  */
  581                 ctx = dmar_find_ctx_locked(dmar, rid);
  582                 if (ctx == NULL) {
  583                         domain = domain1;
  584                         ctx = ctx1;
  585                         dmar_ctx_link(ctx);
  586                         ctx->context.tag->owner = dev;
  587                         device_tag_init(ctx, dev);
  588 
  589                         /*
  590                          * This is the first activated context for the
  591                          * DMAR unit.  Enable the translation after
  592                          * everything is set up.
  593                          */
  594                         if (LIST_EMPTY(&dmar->domains))
  595                                 enable = true;
  596                         LIST_INSERT_HEAD(&dmar->domains, domain, link);
  597                         ctx_id_entry_init(ctx, ctxp, false, bus);
  598                         if (dev != NULL) {
  599                                 device_printf(dev,
  600                             "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d "
  601                                     "agaw %d %s-mapped\n",
  602                                     dmar->iommu.unit, dmar->segment, bus, slot,
  603                                     func, rid, domain->domain, domain->mgaw,
  604                                     domain->agaw, id_mapped ? "id" : "re");
  605                         }
  606                         dmar_unmap_pgtbl(sf);
  607                 } else {
  608                         dmar_unmap_pgtbl(sf);
  609                         dmar_domain_destroy(domain1);
  610                         /* Nothing needs to be done to destroy ctx1. */
  611                         free(ctx1, M_DMAR_CTX);
  612                         domain = CTX2DOM(ctx);
  613                         ctx->refs++; /* tag referenced us */
  614                 }
  615         } else {
  616                 domain = CTX2DOM(ctx);
  617                 if (ctx->context.tag->owner == NULL)
  618                         ctx->context.tag->owner = dev;
  619                 ctx->refs++; /* tag referenced us */
  620         }
  621 
  622         error = dmar_flush_for_ctx_entry(dmar, enable);
  623         if (error != 0) {
  624                 dmar_free_ctx_locked(dmar, ctx);
  625                 TD_PINNED_ASSERT;
  626                 return (NULL);
  627         }
  628 
  629         /*
  630          * The dmar lock was potentially dropped between check for the
  631          * empty context list and now.  Recheck the state of GCMD_TE
  632          * to avoid unneeded command.
  633          */
  634         if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
  635                 error = dmar_enable_translation(dmar);
  636                 if (error == 0) {
  637                         if (bootverbose) {
  638                                 printf("dmar%d: enabled translation\n",
  639                                     dmar->iommu.unit);
  640                         }
  641                 } else {
  642                         printf("dmar%d: enabling translation failed, "
  643                             "error %d\n", dmar->iommu.unit, error);
  644                         dmar_free_ctx_locked(dmar, ctx);
  645                         TD_PINNED_ASSERT;
  646                         return (NULL);
  647                 }
  648         }
  649         DMAR_UNLOCK(dmar);
  650         TD_PINNED_ASSERT;
  651         return (ctx);
  652 }
  653 
  654 struct dmar_ctx *
  655 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid,
  656     bool id_mapped, bool rmrr_init)
  657 {
  658         int dev_domain, dev_path_len, dev_busno;
  659 
  660         dev_domain = pci_get_domain(dev);
  661         dev_path_len = dmar_dev_depth(dev);
  662         ACPI_DMAR_PCI_PATH dev_path[dev_path_len];
  663         dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len);
  664         return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno,
  665             dev_path, dev_path_len, id_mapped, rmrr_init));
  666 }
  667 
  668 struct dmar_ctx *
  669 dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid,
  670     int dev_domain, int dev_busno,
  671     const void *dev_path, int dev_path_len,
  672     bool id_mapped, bool rmrr_init)
  673 {
  674 
  675         return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno,
  676             dev_path, dev_path_len, id_mapped, rmrr_init));
  677 }
  678 
  679 int
  680 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx)
  681 {
  682         struct dmar_unit *dmar;
  683         struct dmar_domain *old_domain;
  684         dmar_ctx_entry_t *ctxp;
  685         struct sf_buf *sf;
  686         int error;
  687 
  688         dmar = domain->dmar;
  689         old_domain = CTX2DOM(ctx);
  690         if (domain == old_domain)
  691                 return (0);
  692         KASSERT(old_domain->iodom.iommu == domain->iodom.iommu,
  693             ("domain %p %u moving between dmars %u %u", domain,
  694             domain->domain, old_domain->iodom.iommu->unit,
  695             domain->iodom.iommu->unit));
  696         TD_PREP_PINNED_ASSERT;
  697 
  698         ctxp = dmar_map_ctx_entry(ctx, &sf);
  699         DMAR_LOCK(dmar);
  700         dmar_ctx_unlink(ctx);
  701         ctx->context.domain = &domain->iodom;
  702         dmar_ctx_link(ctx);
  703         ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100);
  704         dmar_unmap_pgtbl(sf);
  705         error = dmar_flush_for_ctx_entry(dmar, true);
  706         /* If flush failed, rolling back would not work as well. */
  707         printf("dmar%d rid %x domain %d->%d %s-mapped\n",
  708             dmar->iommu.unit, ctx->context.rid, old_domain->domain,
  709             domain->domain, (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ?
  710             "id" : "re");
  711         dmar_unref_domain_locked(dmar, old_domain);
  712         TD_PINNED_ASSERT;
  713         return (error);
  714 }
  715 
  716 static void
  717 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain)
  718 {
  719 
  720         DMAR_ASSERT_LOCKED(dmar);
  721         KASSERT(domain->refs >= 1,
  722             ("dmar %d domain %p refs %u", dmar->iommu.unit, domain,
  723             domain->refs));
  724         KASSERT(domain->refs > domain->ctx_cnt,
  725             ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain,
  726             domain->refs, domain->ctx_cnt));
  727 
  728         if (domain->refs > 1) {
  729                 domain->refs--;
  730                 DMAR_UNLOCK(dmar);
  731                 return;
  732         }
  733 
  734         KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0,
  735             ("lost ref on RMRR domain %p", domain));
  736 
  737         LIST_REMOVE(domain, link);
  738         DMAR_UNLOCK(dmar);
  739 
  740         taskqueue_drain(dmar->iommu.delayed_taskqueue,
  741             &domain->iodom.unload_task);
  742         dmar_domain_destroy(domain);
  743 }
  744 
  745 void
  746 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
  747 {
  748         struct sf_buf *sf;
  749         dmar_ctx_entry_t *ctxp;
  750         struct dmar_domain *domain;
  751 
  752         DMAR_ASSERT_LOCKED(dmar);
  753         KASSERT(ctx->refs >= 1,
  754             ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
  755 
  756         /*
  757          * If our reference is not last, only the dereference should
  758          * be performed.
  759          */
  760         if (ctx->refs > 1) {
  761                 ctx->refs--;
  762                 DMAR_UNLOCK(dmar);
  763                 return;
  764         }
  765 
  766         KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
  767             ("lost ref on disabled ctx %p", ctx));
  768 
  769         /*
  770          * Otherwise, the context entry must be cleared before the
  771          * page table is destroyed.  The mapping of the context
  772          * entries page could require sleep, unlock the dmar.
  773          */
  774         DMAR_UNLOCK(dmar);
  775         TD_PREP_PINNED_ASSERT;
  776         ctxp = dmar_map_ctx_entry(ctx, &sf);
  777         DMAR_LOCK(dmar);
  778         KASSERT(ctx->refs >= 1,
  779             ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
  780 
  781         /*
  782          * Other thread might have referenced the context, in which
  783          * case again only the dereference should be performed.
  784          */
  785         if (ctx->refs > 1) {
  786                 ctx->refs--;
  787                 DMAR_UNLOCK(dmar);
  788                 dmar_unmap_pgtbl(sf);
  789                 TD_PINNED_ASSERT;
  790                 return;
  791         }
  792 
  793         KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
  794             ("lost ref on disabled ctx %p", ctx));
  795 
  796         /*
  797          * Clear the context pointer and flush the caches.
  798          * XXXKIB: cannot do this if any RMRR entries are still present.
  799          */
  800         dmar_pte_clear(&ctxp->ctx1);
  801         ctxp->ctx2 = 0;
  802         dmar_flush_ctx_to_ram(dmar, ctxp);
  803         dmar_inv_ctx_glob(dmar);
  804         if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
  805                 if (dmar->qi_enabled)
  806                         dmar_qi_invalidate_iotlb_glob_locked(dmar);
  807                 else
  808                         dmar_inv_iotlb_glob(dmar);
  809         }
  810         dmar_unmap_pgtbl(sf);
  811         domain = CTX2DOM(ctx);
  812         dmar_ctx_unlink(ctx);
  813         free(ctx->context.tag, M_DMAR_CTX);
  814         free(ctx, M_DMAR_CTX);
  815         dmar_unref_domain_locked(dmar, domain);
  816         TD_PINNED_ASSERT;
  817 }
  818 
  819 void
  820 dmar_free_ctx(struct dmar_ctx *ctx)
  821 {
  822         struct dmar_unit *dmar;
  823 
  824         dmar = CTX2DMAR(ctx);
  825         DMAR_LOCK(dmar);
  826         dmar_free_ctx_locked(dmar, ctx);
  827 }
  828 
  829 /*
  830  * Returns with the domain locked.
  831  */
  832 struct dmar_ctx *
  833 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid)
  834 {
  835         struct dmar_domain *domain;
  836         struct dmar_ctx *ctx;
  837 
  838         DMAR_ASSERT_LOCKED(dmar);
  839 
  840         LIST_FOREACH(domain, &dmar->domains, link) {
  841                 LIST_FOREACH(ctx, &domain->contexts, link) {
  842                         if (ctx->context.rid == rid)
  843                                 return (ctx);
  844                 }
  845         }
  846         return (NULL);
  847 }
  848 
  849 void
  850 dmar_domain_free_entry(struct iommu_map_entry *entry, bool free)
  851 {
  852         struct iommu_domain *domain;
  853 
  854         domain = entry->domain;
  855         IOMMU_DOMAIN_LOCK(domain);
  856         if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
  857                 iommu_gas_free_region(domain, entry);
  858         else
  859                 iommu_gas_free_space(domain, entry);
  860         IOMMU_DOMAIN_UNLOCK(domain);
  861         if (free)
  862                 iommu_gas_free_entry(domain, entry);
  863         else
  864                 entry->flags = 0;
  865 }
  866 
  867 void
  868 dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free)
  869 {
  870         struct dmar_domain *domain;
  871         struct dmar_unit *unit;
  872 
  873         domain = IODOM2DOM(entry->domain);
  874         unit = DOM2DMAR(domain);
  875         if (unit->qi_enabled) {
  876                 DMAR_LOCK(unit);
  877                 dmar_qi_invalidate_locked(IODOM2DOM(entry->domain),
  878                     entry->start, entry->end - entry->start, &entry->gseq,
  879                     true);
  880                 if (!free)
  881                         entry->flags |= IOMMU_MAP_ENTRY_QI_NF;
  882                 TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
  883                 DMAR_UNLOCK(unit);
  884         } else {
  885                 domain_flush_iotlb_sync(IODOM2DOM(entry->domain),
  886                     entry->start, entry->end - entry->start);
  887                 dmar_domain_free_entry(entry, free);
  888         }
  889 }
  890 
  891 static bool
  892 dmar_domain_unload_emit_wait(struct dmar_domain *domain,
  893     struct iommu_map_entry *entry)
  894 {
  895 
  896         if (TAILQ_NEXT(entry, dmamap_link) == NULL)
  897                 return (true);
  898         return (domain->batch_no++ % dmar_batch_coalesce == 0);
  899 }
  900 
  901 void
  902 dmar_domain_unload(struct dmar_domain *domain,
  903     struct iommu_map_entries_tailq *entries, bool cansleep)
  904 {
  905         struct dmar_unit *unit;
  906         struct iommu_domain *iodom;
  907         struct iommu_map_entry *entry, *entry1;
  908         int error __diagused;
  909 
  910         iodom = DOM2IODOM(domain);
  911         unit = DOM2DMAR(domain);
  912 
  913         TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
  914                 KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
  915                     ("not mapped entry %p %p", domain, entry));
  916                 error = iodom->ops->unmap(iodom, entry->start, entry->end -
  917                     entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
  918                 KASSERT(error == 0, ("unmap %p error %d", domain, error));
  919                 if (!unit->qi_enabled) {
  920                         domain_flush_iotlb_sync(domain, entry->start,
  921                             entry->end - entry->start);
  922                         TAILQ_REMOVE(entries, entry, dmamap_link);
  923                         dmar_domain_free_entry(entry, true);
  924                 }
  925         }
  926         if (TAILQ_EMPTY(entries))
  927                 return;
  928 
  929         KASSERT(unit->qi_enabled, ("loaded entry left"));
  930         DMAR_LOCK(unit);
  931         TAILQ_FOREACH(entry, entries, dmamap_link) {
  932                 dmar_qi_invalidate_locked(domain, entry->start, entry->end -
  933                     entry->start, &entry->gseq,
  934                     dmar_domain_unload_emit_wait(domain, entry));
  935         }
  936         TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link);
  937         DMAR_UNLOCK(unit);
  938 }
  939 
  940 struct iommu_ctx *
  941 iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
  942     bool id_mapped, bool rmrr_init)
  943 {
  944         struct dmar_unit *dmar;
  945         struct dmar_ctx *ret;
  946 
  947         dmar = IOMMU2DMAR(iommu);
  948 
  949         ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init);
  950 
  951         return (CTX2IOCTX(ret));
  952 }
  953 
  954 void
  955 iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context)
  956 {
  957         struct dmar_unit *dmar;
  958         struct dmar_ctx *ctx;
  959 
  960         dmar = IOMMU2DMAR(iommu);
  961         ctx = IOCTX2CTX(context);
  962 
  963         dmar_free_ctx_locked(dmar, ctx);
  964 }
  965 
  966 void
  967 iommu_free_ctx(struct iommu_ctx *context)
  968 {
  969         struct dmar_ctx *ctx;
  970 
  971         ctx = IOCTX2CTX(context);
  972 
  973         dmar_free_ctx(ctx);
  974 }
  975 
  976 void
  977 iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free)
  978 {
  979 
  980         dmar_domain_unload_entry(entry, free);
  981 }
  982 
  983 void
  984 iommu_domain_unload(struct iommu_domain *iodom,
  985     struct iommu_map_entries_tailq *entries, bool cansleep)
  986 {
  987         struct dmar_domain *domain;
  988 
  989         domain = IODOM2DOM(iodom);
  990 
  991         dmar_domain_unload(domain, entries, cansleep);
  992 }

Cache object: c1e3135ac611e807f4dbdb1da1508aa5


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.