vtd.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2011 NetApp, Inc.
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  *
   28  * $FreeBSD$
   29  */
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD$");
   33 
   34 #include <sys/param.h>
   35 #include <sys/kernel.h>
   36 #include <sys/systm.h>
   37 #include <sys/malloc.h>
   38 
   39 #include <vm/vm.h>
   40 #include <vm/pmap.h>
   41 
   42 #include <dev/pci/pcireg.h>
   43 
   44 #include <machine/vmparam.h>
   45 #include <contrib/dev/acpica/include/acpi.h>
   46 
   47 #include "io/iommu.h"
   48 
   49 /*
   50  * Documented in the "Intel Virtualization Technology for Directed I/O",
   51  * Architecture Spec, September 2008.
   52  */
   53 
   54 #define VTD_DRHD_INCLUDE_PCI_ALL(Flags)  (((Flags) >> 0) & 0x1)
   55 
   56 /* Section 10.4 "Register Descriptions" */
   57 struct vtdmap {
   58         volatile uint32_t       version;
   59         volatile uint32_t       res0;
   60         volatile uint64_t       cap;
   61         volatile uint64_t       ext_cap;
   62         volatile uint32_t       gcr;
   63         volatile uint32_t       gsr;
   64         volatile uint64_t       rta;
   65         volatile uint64_t       ccr;
   66 };
   67 
   68 #define VTD_CAP_SAGAW(cap)      (((cap) >> 8) & 0x1F)
   69 #define VTD_CAP_ND(cap)         ((cap) & 0x7)
   70 #define VTD_CAP_CM(cap)         (((cap) >> 7) & 0x1)
   71 #define VTD_CAP_SPS(cap)        (((cap) >> 34) & 0xF)
   72 #define VTD_CAP_RWBF(cap)       (((cap) >> 4) & 0x1)
   73 
   74 #define VTD_ECAP_DI(ecap)       (((ecap) >> 2) & 0x1)
   75 #define VTD_ECAP_COHERENCY(ecap) ((ecap) & 0x1)
   76 #define VTD_ECAP_IRO(ecap)      (((ecap) >> 8) & 0x3FF)
   77 
   78 #define VTD_GCR_WBF             (1 << 27)
   79 #define VTD_GCR_SRTP            (1 << 30)
   80 #define VTD_GCR_TE              (1U << 31)
   81 
   82 #define VTD_GSR_WBFS            (1 << 27)
   83 #define VTD_GSR_RTPS            (1 << 30)
   84 #define VTD_GSR_TES             (1U << 31)
   85 
   86 #define VTD_CCR_ICC             (1UL << 63)     /* invalidate context cache */
   87 #define VTD_CCR_CIRG_GLOBAL     (1UL << 61)     /* global invalidation */
   88 
   89 #define VTD_IIR_IVT             (1UL << 63)     /* invalidation IOTLB */
   90 #define VTD_IIR_IIRG_GLOBAL     (1ULL << 60)    /* global IOTLB invalidation */
   91 #define VTD_IIR_IIRG_DOMAIN     (2ULL << 60)    /* domain IOTLB invalidation */
   92 #define VTD_IIR_IIRG_PAGE       (3ULL << 60)    /* page IOTLB invalidation */
   93 #define VTD_IIR_DRAIN_READS     (1ULL << 49)    /* drain pending DMA reads */
   94 #define VTD_IIR_DRAIN_WRITES    (1ULL << 48)    /* drain pending DMA writes */
   95 #define VTD_IIR_DOMAIN_P        32
   96 
   97 #define VTD_ROOT_PRESENT        0x1
   98 #define VTD_CTX_PRESENT         0x1
   99 #define VTD_CTX_TT_ALL          (1UL << 2)
  100 
  101 #define VTD_PTE_RD              (1UL << 0)
  102 #define VTD_PTE_WR              (1UL << 1)
  103 #define VTD_PTE_SUPERPAGE       (1UL << 7)
  104 #define VTD_PTE_ADDR_M          (0x000FFFFFFFFFF000UL)
  105 
  106 #define VTD_RID2IDX(rid)        (((rid) & 0xff) * 2)
  107 
  108 struct domain {
  109         uint64_t        *ptp;           /* first level page table page */
  110         int             pt_levels;      /* number of page table levels */
  111         int             addrwidth;      /* 'AW' field in context entry */
  112         int             spsmask;        /* supported super page sizes */
  113         u_int           id;             /* domain id */
  114         vm_paddr_t      maxaddr;        /* highest address to be mapped */
  115         SLIST_ENTRY(domain) next;
  116 };
  117 
  118 static SLIST_HEAD(, domain) domhead;
  119 
  120 #define DRHD_MAX_UNITS  8
  121 static ACPI_DMAR_HARDWARE_UNIT  *drhds[DRHD_MAX_UNITS];
  122 static int                      drhd_num;
  123 static struct vtdmap            *vtdmaps[DRHD_MAX_UNITS];
  124 static int                      max_domains;
  125 typedef int                     (*drhd_ident_func_t)(void);
  126 
  127 static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
  128 static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
  129 
  130 static MALLOC_DEFINE(M_VTD, "vtd", "vtd");
  131 
  132 static int
  133 vtd_max_domains(struct vtdmap *vtdmap)
  134 {
  135         int nd;
  136 
  137         nd = VTD_CAP_ND(vtdmap->cap);
  138 
  139         switch (nd) {
  140         case 0:
  141                 return (16);
  142         case 1:
  143                 return (64);
  144         case 2:
  145                 return (256);
  146         case 3:
  147                 return (1024);
  148         case 4:
  149                 return (4 * 1024);
  150         case 5:
  151                 return (16 * 1024);
  152         case 6:
  153                 return (64 * 1024);
  154         default:
  155                 panic("vtd_max_domains: invalid value of nd (0x%0x)", nd);
  156         }
  157 }
  158 
  159 static u_int
  160 domain_id(void)
  161 {
  162         u_int id;
  163         struct domain *dom;
  164 
  165         /* Skip domain id 0 - it is reserved when Caching Mode field is set */
  166         for (id = 1; id < max_domains; id++) {
  167                 SLIST_FOREACH(dom, &domhead, next) {
  168                         if (dom->id == id)
  169                                 break;
  170                 }
  171                 if (dom == NULL)
  172                         break;          /* found it */
  173         }
  174 
  175         if (id >= max_domains)
  176                 panic("domain ids exhausted");
  177 
  178         return (id);
  179 }
  180 
  181 static struct vtdmap *
  182 vtd_device_scope(uint16_t rid)
  183 {
  184         int i, remaining, pathremaining;
  185         char *end, *pathend;
  186         struct vtdmap *vtdmap;
  187         ACPI_DMAR_HARDWARE_UNIT *drhd;
  188         ACPI_DMAR_DEVICE_SCOPE *device_scope;
  189         ACPI_DMAR_PCI_PATH *path;
  190 
  191         for (i = 0; i < drhd_num; i++) {
  192                 drhd = drhds[i];
  193 
  194                 if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) {
  195                         /*
  196                          * From Intel VT-d arch spec, version 3.0:
  197                          * If a DRHD structure with INCLUDE_PCI_ALL flag Set is reported
  198                          * for a Segment, it must be enumerated by BIOS after all other
  199                          * DRHD structures for the same Segment.
  200                          */
  201                         vtdmap = vtdmaps[i];
  202                         return(vtdmap);
  203                 }
  204 
  205                 end = (char *)drhd + drhd->Header.Length;
  206                 remaining = drhd->Header.Length - sizeof(ACPI_DMAR_HARDWARE_UNIT);
  207                 while (remaining > sizeof(ACPI_DMAR_DEVICE_SCOPE)) {
  208                         device_scope = (ACPI_DMAR_DEVICE_SCOPE *)(end - remaining);
  209                         remaining -= device_scope->Length;
  210 
  211                         switch (device_scope->EntryType){
  212                                 /* 0x01 and 0x02 are PCI device entries */
  213                                 case 0x01:
  214                                 case 0x02:
  215                                         break;
  216                                 default:
  217                                         continue;
  218                         }
  219 
  220                         if (PCI_RID2BUS(rid) != device_scope->Bus)
  221                                 continue;
  222 
  223                         pathend = (char *)device_scope + device_scope->Length;
  224                         pathremaining = device_scope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE);
  225                         while (pathremaining >= sizeof(ACPI_DMAR_PCI_PATH)) {
  226                                 path = (ACPI_DMAR_PCI_PATH *)(pathend - pathremaining);
  227                                 pathremaining -= sizeof(ACPI_DMAR_PCI_PATH);
  228 
  229                                 if (PCI_RID2SLOT(rid) != path->Device)
  230                                         continue;
  231                                 if (PCI_RID2FUNC(rid) != path->Function)
  232                                         continue;
  233 
  234                                 vtdmap = vtdmaps[i];
  235                                 return (vtdmap);
  236                         }
  237                 }
  238         }
  239 
  240         /* No matching scope */
  241         return (NULL);
  242 }
  243 
  244 static void
  245 vtd_wbflush(struct vtdmap *vtdmap)
  246 {
  247 
  248         if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0)
  249                 pmap_invalidate_cache();
  250 
  251         if (VTD_CAP_RWBF(vtdmap->cap)) {
  252                 vtdmap->gcr = VTD_GCR_WBF;
  253                 while ((vtdmap->gsr & VTD_GSR_WBFS) != 0)
  254                         ;
  255         }
  256 }
  257 
  258 static void
  259 vtd_ctx_global_invalidate(struct vtdmap *vtdmap)
  260 {
  261 
  262         vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL;
  263         while ((vtdmap->ccr & VTD_CCR_ICC) != 0)
  264                 ;
  265 }
  266 
  267 static void
  268 vtd_iotlb_global_invalidate(struct vtdmap *vtdmap)
  269 {
  270         int offset;
  271         volatile uint64_t *iotlb_reg, val;
  272 
  273         vtd_wbflush(vtdmap);
  274 
  275         offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16;
  276         iotlb_reg = (volatile uint64_t *)((caddr_t)vtdmap + offset + 8);
  277 
  278         *iotlb_reg =  VTD_IIR_IVT | VTD_IIR_IIRG_GLOBAL |
  279                       VTD_IIR_DRAIN_READS | VTD_IIR_DRAIN_WRITES;
  280 
  281         while (1) {
  282                 val = *iotlb_reg;
  283                 if ((val & VTD_IIR_IVT) == 0)
  284                         break;
  285         }
  286 }
  287 
  288 static void
  289 vtd_translation_enable(struct vtdmap *vtdmap)
  290 {
  291 
  292         vtdmap->gcr = VTD_GCR_TE;
  293         while ((vtdmap->gsr & VTD_GSR_TES) == 0)
  294                 ;
  295 }
  296 
  297 static void
  298 vtd_translation_disable(struct vtdmap *vtdmap)
  299 {
  300 
  301         vtdmap->gcr = 0;
  302         while ((vtdmap->gsr & VTD_GSR_TES) != 0)
  303                 ;
  304 }
  305 
  306 static int
  307 vtd_init(void)
  308 {
  309         int i, units, remaining, tmp;
  310         struct vtdmap *vtdmap;
  311         vm_paddr_t ctx_paddr;
  312         char *end, envname[32];
  313         unsigned long mapaddr;
  314         ACPI_STATUS status;
  315         ACPI_TABLE_DMAR *dmar;
  316         ACPI_DMAR_HEADER *hdr;
  317         ACPI_DMAR_HARDWARE_UNIT *drhd;
  318 
  319         /*
  320          * Allow the user to override the ACPI DMAR table by specifying the
  321          * physical address of each remapping unit.
  322          *
  323          * The following example specifies two remapping units at
  324          * physical addresses 0xfed90000 and 0xfeda0000 respectively.
  325          * set vtd.regmap.0.addr=0xfed90000
  326          * set vtd.regmap.1.addr=0xfeda0000
  327          */
  328         for (units = 0; units < DRHD_MAX_UNITS; units++) {
  329                 snprintf(envname, sizeof(envname), "vtd.regmap.%d.addr", units);
  330                 if (getenv_ulong(envname, &mapaddr) == 0)
  331                         break;
  332                 vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(mapaddr);
  333         }
  334 
  335         if (units > 0)
  336                 goto skip_dmar;
  337 
  338         /* Search for DMAR table. */
  339         status = AcpiGetTable(ACPI_SIG_DMAR, 0, (ACPI_TABLE_HEADER **)&dmar);
  340         if (ACPI_FAILURE(status))
  341                 return (ENXIO);
  342 
  343         end = (char *)dmar + dmar->Header.Length;
  344         remaining = dmar->Header.Length - sizeof(ACPI_TABLE_DMAR);
  345         while (remaining > sizeof(ACPI_DMAR_HEADER)) {
  346                 hdr = (ACPI_DMAR_HEADER *)(end - remaining);
  347                 if (hdr->Length > remaining)
  348                         break;
  349                 /*
  350                  * From Intel VT-d arch spec, version 1.3:
  351                  * BIOS implementations must report mapping structures
  352                  * in numerical order, i.e. All remapping structures of
  353                  * type 0 (DRHD) enumerated before remapping structures of
  354                  * type 1 (RMRR) and so forth.
  355                  */
  356                 if (hdr->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT)
  357                         break;
  358 
  359                 drhd = (ACPI_DMAR_HARDWARE_UNIT *)hdr;
  360                 drhds[units] = drhd;
  361                 vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address);
  362                 if (++units >= DRHD_MAX_UNITS)
  363                         break;
  364                 remaining -= hdr->Length;
  365         }
  366 
  367         if (units <= 0)
  368                 return (ENXIO);
  369 
  370 skip_dmar:
  371         drhd_num = units;
  372 
  373         max_domains = 64 * 1024; /* maximum valid value */
  374         for (i = 0; i < drhd_num; i++){
  375                 vtdmap = vtdmaps[i];
  376 
  377                 if (VTD_CAP_CM(vtdmap->cap) != 0)
  378                         panic("vtd_init: invalid caching mode");
  379 
  380                 /* take most compatible (minimum) value */
  381                 if ((tmp = vtd_max_domains(vtdmap)) < max_domains)
  382                         max_domains = tmp;
  383         }
  384 
  385         /*
  386          * Set up the root-table to point to the context-entry tables
  387          */
  388         for (i = 0; i < 256; i++) {
  389                 ctx_paddr = vtophys(ctx_tables[i]);
  390                 if (ctx_paddr & PAGE_MASK)
  391                         panic("ctx table (0x%0lx) not page aligned", ctx_paddr);
  392 
  393                 root_table[i * 2] = ctx_paddr | VTD_ROOT_PRESENT;
  394         }
  395 
  396         return (0);
  397 }
  398 
  399 static void
  400 vtd_cleanup(void)
  401 {
  402 }
  403 
  404 static void
  405 vtd_enable(void)
  406 {
  407         int i;
  408         struct vtdmap *vtdmap;
  409 
  410         for (i = 0; i < drhd_num; i++) {
  411                 vtdmap = vtdmaps[i];
  412                 vtd_wbflush(vtdmap);
  413 
  414                 /* Update the root table address */
  415                 vtdmap->rta = vtophys(root_table);
  416                 vtdmap->gcr = VTD_GCR_SRTP;
  417                 while ((vtdmap->gsr & VTD_GSR_RTPS) == 0)
  418                         ;
  419 
  420                 vtd_ctx_global_invalidate(vtdmap);
  421                 vtd_iotlb_global_invalidate(vtdmap);
  422 
  423                 vtd_translation_enable(vtdmap);
  424         }
  425 }
  426 
  427 static void
  428 vtd_disable(void)
  429 {
  430         int i;
  431         struct vtdmap *vtdmap;
  432 
  433         for (i = 0; i < drhd_num; i++) {
  434                 vtdmap = vtdmaps[i];
  435                 vtd_translation_disable(vtdmap);
  436         }
  437 }
  438 
  439 static void
  440 vtd_add_device(void *arg, uint16_t rid)
  441 {
  442         int idx;
  443         uint64_t *ctxp;
  444         struct domain *dom = arg;
  445         vm_paddr_t pt_paddr;
  446         struct vtdmap *vtdmap;
  447         uint8_t bus;
  448 
  449         KASSERT(dom != NULL, ("domain is NULL"));
  450 
  451         bus = PCI_RID2BUS(rid);
  452         ctxp = ctx_tables[bus];
  453         pt_paddr = vtophys(dom->ptp);
  454         idx = VTD_RID2IDX(rid);
  455 
  456         if (ctxp[idx] & VTD_CTX_PRESENT) {
  457                 panic("vtd_add_device: device %x is already owned by "
  458                       "domain %d", rid,
  459                       (uint16_t)(ctxp[idx + 1] >> 8));
  460         }
  461 
  462         if ((vtdmap = vtd_device_scope(rid)) == NULL)
  463                 panic("vtd_add_device: device %x is not in scope for "
  464                       "any DMA remapping unit", rid);
  465 
  466         /*
  467          * Order is important. The 'present' bit is set only after all fields
  468          * of the context pointer are initialized.
  469          */
  470         ctxp[idx + 1] = dom->addrwidth | (dom->id << 8);
  471 
  472         if (VTD_ECAP_DI(vtdmap->ext_cap))
  473                 ctxp[idx] = VTD_CTX_TT_ALL;
  474         else
  475                 ctxp[idx] = 0;
  476 
  477         ctxp[idx] |= pt_paddr | VTD_CTX_PRESENT;
  478 
  479         /*
  480          * 'Not Present' entries are not cached in either the Context Cache
  481          * or in the IOTLB, so there is no need to invalidate either of them.
  482          */
  483 }
  484 
  485 static void
  486 vtd_remove_device(void *arg, uint16_t rid)
  487 {
  488         int i, idx;
  489         uint64_t *ctxp;
  490         struct vtdmap *vtdmap;
  491         uint8_t bus;
  492 
  493         bus = PCI_RID2BUS(rid);
  494         ctxp = ctx_tables[bus];
  495         idx = VTD_RID2IDX(rid);
  496 
  497         /*
  498          * Order is important. The 'present' bit is must be cleared first.
  499          */
  500         ctxp[idx] = 0;
  501         ctxp[idx + 1] = 0;
  502 
  503         /*
  504          * Invalidate the Context Cache and the IOTLB.
  505          *
  506          * XXX use device-selective invalidation for Context Cache
  507          * XXX use domain-selective invalidation for IOTLB
  508          */
  509         for (i = 0; i < drhd_num; i++) {
  510                 vtdmap = vtdmaps[i];
  511                 vtd_ctx_global_invalidate(vtdmap);
  512                 vtd_iotlb_global_invalidate(vtdmap);
  513         }
  514 }
  515 
  516 #define CREATE_MAPPING  0
  517 #define REMOVE_MAPPING  1
  518 
  519 static uint64_t
  520 vtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len,
  521                    int remove)
  522 {
  523         struct domain *dom;
  524         int i, spshift, ptpshift, ptpindex, nlevels;
  525         uint64_t spsize, *ptp;
  526 
  527         dom = arg;
  528         ptpindex = 0;
  529         ptpshift = 0;
  530 
  531         KASSERT(gpa + len > gpa, ("%s: invalid gpa range %#lx/%#lx", __func__,
  532             gpa, len));
  533         KASSERT(gpa + len <= dom->maxaddr, ("%s: gpa range %#lx/%#lx beyond "
  534             "domain maxaddr %#lx", __func__, gpa, len, dom->maxaddr));
  535 
  536         if (gpa & PAGE_MASK)
  537                 panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa);
  538 
  539         if (hpa & PAGE_MASK)
  540                 panic("vtd_create_mapping: unaligned hpa 0x%0lx", hpa);
  541 
  542         if (len & PAGE_MASK)
  543                 panic("vtd_create_mapping: unaligned len 0x%0lx", len);
  544 
  545         /*
  546          * Compute the size of the mapping that we can accommodate.
  547          *
  548          * This is based on three factors:
  549          * - supported super page size
  550          * - alignment of the region starting at 'gpa' and 'hpa'
  551          * - length of the region 'len'
  552          */
  553         spshift = 48;
  554         for (i = 3; i >= 0; i--) {
  555                 spsize = 1UL << spshift;
  556                 if ((dom->spsmask & (1 << i)) != 0 &&
  557                     (gpa & (spsize - 1)) == 0 &&
  558                     (hpa & (spsize - 1)) == 0 &&
  559                     (len >= spsize)) {
  560                         break;
  561                 }
  562                 spshift -= 9;
  563         }
  564 
  565         ptp = dom->ptp;
  566         nlevels = dom->pt_levels;
  567         while (--nlevels >= 0) {
  568                 ptpshift = 12 + nlevels * 9;
  569                 ptpindex = (gpa >> ptpshift) & 0x1FF;
  570 
  571                 /* We have reached the leaf mapping */
  572                 if (spshift >= ptpshift) {
  573                         break;
  574                 }
  575 
  576                 /*
  577                  * We are working on a non-leaf page table page.
  578                  *
  579                  * Create a downstream page table page if necessary and point
  580                  * to it from the current page table.
  581                  */
  582                 if (ptp[ptpindex] == 0) {
  583                         void *nlp = malloc(PAGE_SIZE, M_VTD, M_WAITOK | M_ZERO);
  584                         ptp[ptpindex] = vtophys(nlp)| VTD_PTE_RD | VTD_PTE_WR;
  585                 }
  586 
  587                 ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & VTD_PTE_ADDR_M);
  588         }
  589 
  590         if ((gpa & ((1UL << ptpshift) - 1)) != 0)
  591                 panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift);
  592 
  593         /*
  594          * Update the 'gpa' -> 'hpa' mapping
  595          */
  596         if (remove) {
  597                 ptp[ptpindex] = 0;
  598         } else {
  599                 ptp[ptpindex] = hpa | VTD_PTE_RD | VTD_PTE_WR;
  600 
  601                 if (nlevels > 0)
  602                         ptp[ptpindex] |= VTD_PTE_SUPERPAGE;
  603         }
  604 
  605         return (1UL << ptpshift);
  606 }
  607 
  608 static uint64_t
  609 vtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
  610 {
  611 
  612         return (vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING));
  613 }
  614 
  615 static uint64_t
  616 vtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
  617 {
  618 
  619         return (vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING));
  620 }
  621 
  622 static void
  623 vtd_invalidate_tlb(void *dom)
  624 {
  625         int i;
  626         struct vtdmap *vtdmap;
  627 
  628         /*
  629          * Invalidate the IOTLB.
  630          * XXX use domain-selective invalidation for IOTLB
  631          */
  632         for (i = 0; i < drhd_num; i++) {
  633                 vtdmap = vtdmaps[i];
  634                 vtd_iotlb_global_invalidate(vtdmap);
  635         }
  636 }
  637 
  638 static void *
  639 vtd_create_domain(vm_paddr_t maxaddr)
  640 {
  641         struct domain *dom;
  642         vm_paddr_t addr;
  643         int tmp, i, gaw, agaw, sagaw, res, pt_levels, addrwidth;
  644         struct vtdmap *vtdmap;
  645 
  646         if (drhd_num <= 0)
  647                 panic("vtd_create_domain: no dma remapping hardware available");
  648 
  649         /*
  650          * Calculate AGAW.
  651          * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec.
  652          */
  653         addr = 0;
  654         for (gaw = 0; addr < maxaddr; gaw++)
  655                 addr = 1ULL << gaw;
  656 
  657         res = (gaw - 12) % 9;
  658         if (res == 0)
  659                 agaw = gaw;
  660         else
  661                 agaw = gaw + 9 - res;
  662 
  663         if (agaw > 64)
  664                 agaw = 64;
  665 
  666         /*
  667          * Select the smallest Supported AGAW and the corresponding number
  668          * of page table levels.
  669          */
  670         pt_levels = 2;
  671         sagaw = 30;
  672         addrwidth = 0;
  673 
  674         tmp = ~0;
  675         for (i = 0; i < drhd_num; i++) {
  676                 vtdmap = vtdmaps[i];
  677                 /* take most compatible value */
  678                 tmp &= VTD_CAP_SAGAW(vtdmap->cap);
  679         }
  680 
  681         for (i = 0; i < 5; i++) {
  682                 if ((tmp & (1 << i)) != 0 && sagaw >= agaw)
  683                         break;
  684                 pt_levels++;
  685                 addrwidth++;
  686                 sagaw += 9;
  687                 if (sagaw > 64)
  688                         sagaw = 64;
  689         }
  690 
  691         if (i >= 5) {
  692                 panic("vtd_create_domain: SAGAW 0x%x does not support AGAW %d",
  693                       tmp, agaw);
  694         }
  695 
  696         dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK);
  697         dom->pt_levels = pt_levels;
  698         dom->addrwidth = addrwidth;
  699         dom->id = domain_id();
  700         dom->maxaddr = maxaddr;
  701         dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK);
  702         if ((uintptr_t)dom->ptp & PAGE_MASK)
  703                 panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp);
  704 
  705 #ifdef notyet
  706         /*
  707          * XXX superpage mappings for the iommu do not work correctly.
  708          *
  709          * By default all physical memory is mapped into the host_domain.
  710          * When a VM is allocated wired memory the pages belonging to it
  711          * are removed from the host_domain and added to the vm's domain.
  712          *
  713          * If the page being removed was mapped using a superpage mapping
  714          * in the host_domain then we need to demote the mapping before
  715          * removing the page.
  716          *
  717          * There is not any code to deal with the demotion at the moment
  718          * so we disable superpage mappings altogether.
  719          */
  720         dom->spsmask = ~0;
  721         for (i = 0; i < drhd_num; i++) {
  722                 vtdmap = vtdmaps[i];
  723                 /* take most compatible value */
  724                 dom->spsmask &= VTD_CAP_SPS(vtdmap->cap);
  725         }
  726 #endif
  727 
  728         SLIST_INSERT_HEAD(&domhead, dom, next);
  729 
  730         return (dom);
  731 }
  732 
  733 static void
  734 vtd_free_ptp(uint64_t *ptp, int level)
  735 {
  736         int i;
  737         uint64_t *nlp;
  738 
  739         if (level > 1) {
  740                 for (i = 0; i < 512; i++) {
  741                         if ((ptp[i] & (VTD_PTE_RD | VTD_PTE_WR)) == 0)
  742                                 continue;
  743                         if ((ptp[i] & VTD_PTE_SUPERPAGE) != 0)
  744                                 continue;
  745                         nlp = (uint64_t *)PHYS_TO_DMAP(ptp[i] & VTD_PTE_ADDR_M);
  746                         vtd_free_ptp(nlp, level - 1);
  747                 }
  748         }
  749 
  750         bzero(ptp, PAGE_SIZE);
  751         free(ptp, M_VTD);
  752 }
  753 
  754 static void
  755 vtd_destroy_domain(void *arg)
  756 {
  757         struct domain *dom;
  758 
  759         dom = arg;
  760 
  761         SLIST_REMOVE(&domhead, dom, domain, next);
  762         vtd_free_ptp(dom->ptp, dom->pt_levels);
  763         free(dom, M_VTD);
  764 }
  765 
  766 const struct iommu_ops iommu_ops_intel = {
  767         .init = vtd_init,
  768         .cleanup = vtd_cleanup,
  769         .enable = vtd_enable,
  770         .disable = vtd_disable,
  771         .create_domain = vtd_create_domain,
  772         .destroy_domain = vtd_destroy_domain,
  773         .create_mapping = vtd_create_mapping,
  774         .remove_mapping = vtd_remove_mapping,
  775         .add_device = vtd_add_device,
  776         .remove_device = vtd_remove_device,
  777         .invalidate_tlb = vtd_invalidate_tlb,
  778 };
Cache object: d45af30270e36d55af963a623b1806fb
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/amd64/vmm/intel/vtd.c

FreeBSD/Linux Kernel Cross Reference
sys/amd64/vmm/intel/vtd.c
