The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/intel_qi.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2013 The FreeBSD Foundation
    5  * All rights reserved.
    6  *
    7  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    8  * under sponsorship from the FreeBSD Foundation.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   29  * SUCH DAMAGE.
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 
   35 #include "opt_acpi.h"
   36 
   37 #include <sys/param.h>
   38 #include <sys/bus.h>
   39 #include <sys/kernel.h>
   40 #include <sys/malloc.h>
   41 #include <sys/memdesc.h>
   42 #include <sys/module.h>
   43 #include <sys/rman.h>
   44 #include <sys/taskqueue.h>
   45 #include <sys/time.h>
   46 #include <sys/tree.h>
   47 #include <sys/vmem.h>
   48 #include <vm/vm.h>
   49 #include <vm/vm_extern.h>
   50 #include <vm/vm_kern.h>
   51 #include <vm/vm_page.h>
   52 #include <vm/vm_map.h>
   53 #include <contrib/dev/acpica/include/acpi.h>
   54 #include <contrib/dev/acpica/include/accommon.h>
   55 #include <dev/acpica/acpivar.h>
   56 #include <dev/pci/pcireg.h>
   57 #include <machine/bus.h>
   58 #include <machine/cpu.h>
   59 #include <x86/include/busdma_impl.h>
   60 #include <dev/iommu/busdma_iommu.h>
   61 #include <x86/iommu/intel_reg.h>
   62 #include <x86/iommu/intel_dmar.h>
   63 
   64 static bool
   65 dmar_qi_seq_processed(const struct dmar_unit *unit,
   66     const struct iommu_qi_genseq *pseq)
   67 {
   68 
   69         return (pseq->gen < unit->inv_waitd_gen ||
   70             (pseq->gen == unit->inv_waitd_gen &&
   71              pseq->seq <= unit->inv_waitd_seq_hw));
   72 }
   73 
   74 static int
   75 dmar_enable_qi(struct dmar_unit *unit)
   76 {
   77         int error;
   78 
   79         DMAR_ASSERT_LOCKED(unit);
   80         unit->hw_gcmd |= DMAR_GCMD_QIE;
   81         dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
   82         DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
   83             != 0));
   84         return (error);
   85 }
   86 
   87 static int
   88 dmar_disable_qi(struct dmar_unit *unit)
   89 {
   90         int error;
   91 
   92         DMAR_ASSERT_LOCKED(unit);
   93         unit->hw_gcmd &= ~DMAR_GCMD_QIE;
   94         dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
   95         DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
   96             == 0));
   97         return (error);
   98 }
   99 
  100 static void
  101 dmar_qi_advance_tail(struct dmar_unit *unit)
  102 {
  103 
  104         DMAR_ASSERT_LOCKED(unit);
  105         dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
  106 }
  107 
  108 static void
  109 dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
  110 {
  111         uint32_t head;
  112         int bytes;
  113 
  114         DMAR_ASSERT_LOCKED(unit);
  115         bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
  116         for (;;) {
  117                 if (bytes <= unit->inv_queue_avail)
  118                         break;
  119                 /* refill */
  120                 head = dmar_read4(unit, DMAR_IQH_REG);
  121                 head &= DMAR_IQH_MASK;
  122                 unit->inv_queue_avail = head - unit->inv_queue_tail -
  123                     DMAR_IQ_DESCR_SZ;
  124                 if (head <= unit->inv_queue_tail)
  125                         unit->inv_queue_avail += unit->inv_queue_size;
  126                 if (bytes <= unit->inv_queue_avail)
  127                         break;
  128 
  129                 /*
  130                  * No space in the queue, do busy wait.  Hardware must
  131                  * make a progress.  But first advance the tail to
  132                  * inform the descriptor streamer about entries we
  133                  * might have already filled, otherwise they could
  134                  * clog the whole queue..
  135                  */
  136                 dmar_qi_advance_tail(unit);
  137                 unit->inv_queue_full++;
  138                 cpu_spinwait();
  139         }
  140         unit->inv_queue_avail -= bytes;
  141 }
  142 
  143 static void
  144 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
  145 {
  146 
  147         DMAR_ASSERT_LOCKED(unit);
  148         *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
  149         unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
  150         KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
  151             ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
  152             (uintmax_t)unit->inv_queue_size));
  153         unit->inv_queue_tail &= unit->inv_queue_size - 1;
  154         *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
  155         unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
  156         KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
  157             ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
  158             (uintmax_t)unit->inv_queue_size));
  159         unit->inv_queue_tail &= unit->inv_queue_size - 1;
  160 }
  161 
  162 static void
  163 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
  164     bool memw, bool fence)
  165 {
  166 
  167         DMAR_ASSERT_LOCKED(unit);
  168         dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
  169             (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
  170             (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
  171             (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
  172             (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
  173             memw ? unit->inv_waitd_seq_hw_phys : 0);
  174 }
  175 
  176 static void
  177 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq,
  178     bool emit_wait)
  179 {
  180         struct iommu_qi_genseq gsec;
  181         uint32_t seq;
  182 
  183         KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
  184         DMAR_ASSERT_LOCKED(unit);
  185         if (unit->inv_waitd_seq == 0xffffffff) {
  186                 gsec.gen = unit->inv_waitd_gen;
  187                 gsec.seq = unit->inv_waitd_seq;
  188                 dmar_qi_ensure(unit, 1);
  189                 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
  190                 dmar_qi_advance_tail(unit);
  191                 while (!dmar_qi_seq_processed(unit, &gsec))
  192                         cpu_spinwait();
  193                 unit->inv_waitd_gen++;
  194                 unit->inv_waitd_seq = 1;
  195         }
  196         seq = unit->inv_waitd_seq++;
  197         pseq->gen = unit->inv_waitd_gen;
  198         pseq->seq = seq;
  199         if (emit_wait) {
  200                 dmar_qi_ensure(unit, 1);
  201                 dmar_qi_emit_wait_descr(unit, seq, true, true, false);
  202         }
  203 }
  204 
  205 static void
  206 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct iommu_qi_genseq *gseq,
  207     bool nowait)
  208 {
  209 
  210         DMAR_ASSERT_LOCKED(unit);
  211         unit->inv_seq_waiters++;
  212         while (!dmar_qi_seq_processed(unit, gseq)) {
  213                 if (cold || nowait) {
  214                         cpu_spinwait();
  215                 } else {
  216                         msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0,
  217                             "dmarse", hz);
  218                 }
  219         }
  220         unit->inv_seq_waiters--;
  221 }
  222 
  223 void
  224 dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base,
  225     iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait)
  226 {
  227         struct dmar_unit *unit;
  228         iommu_gaddr_t isize;
  229         int am;
  230 
  231         unit = domain->dmar;
  232         DMAR_ASSERT_LOCKED(unit);
  233         for (; size > 0; base += isize, size -= isize) {
  234                 am = calc_am(unit, base, size, &isize);
  235                 dmar_qi_ensure(unit, 1);
  236                 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
  237                     DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
  238                     DMAR_IQ_DESCR_IOTLB_DR |
  239                     DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
  240                     base | am);
  241         }
  242         dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
  243         dmar_qi_advance_tail(unit);
  244 }
  245 
  246 void
  247 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
  248 {
  249         struct iommu_qi_genseq gseq;
  250 
  251         DMAR_ASSERT_LOCKED(unit);
  252         dmar_qi_ensure(unit, 2);
  253         dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
  254         dmar_qi_emit_wait_seq(unit, &gseq, true);
  255         dmar_qi_advance_tail(unit);
  256         dmar_qi_wait_for_seq(unit, &gseq, false);
  257 }
  258 
  259 void
  260 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
  261 {
  262         struct iommu_qi_genseq gseq;
  263 
  264         DMAR_ASSERT_LOCKED(unit);
  265         dmar_qi_ensure(unit, 2);
  266         dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
  267             DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
  268         dmar_qi_emit_wait_seq(unit, &gseq, true);
  269         dmar_qi_advance_tail(unit);
  270         dmar_qi_wait_for_seq(unit, &gseq, false);
  271 }
  272 
  273 void
  274 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
  275 {
  276         struct iommu_qi_genseq gseq;
  277 
  278         DMAR_ASSERT_LOCKED(unit);
  279         dmar_qi_ensure(unit, 2);
  280         dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
  281         dmar_qi_emit_wait_seq(unit, &gseq, true);
  282         dmar_qi_advance_tail(unit);
  283         dmar_qi_wait_for_seq(unit, &gseq, false);
  284 }
  285 
  286 void
  287 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
  288 {
  289         struct iommu_qi_genseq gseq;
  290         u_int c, l;
  291 
  292         DMAR_ASSERT_LOCKED(unit);
  293         KASSERT(start < unit->irte_cnt && start < start + cnt &&
  294             start + cnt <= unit->irte_cnt,
  295             ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
  296         for (; cnt > 0; cnt -= c, start += c) {
  297                 l = ffs(start | cnt) - 1;
  298                 c = 1 << l;
  299                 dmar_qi_ensure(unit, 1);
  300                 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
  301                     DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
  302                     DMAR_IQ_DESCR_IEC_IM(l), 0);
  303         }
  304         dmar_qi_ensure(unit, 1);
  305         dmar_qi_emit_wait_seq(unit, &gseq, true);
  306         dmar_qi_advance_tail(unit);
  307 
  308         /*
  309          * The caller of the function, in particular,
  310          * dmar_ir_program_irte(), may be called from the context
  311          * where the sleeping is forbidden (in fact, the
  312          * intr_table_lock mutex may be held, locked from
  313          * intr_shuffle_irqs()).  Wait for the invalidation completion
  314          * using the busy wait.
  315          *
  316          * The impact on the interrupt input setup code is small, the
  317          * expected overhead is comparable with the chipset register
  318          * read.  It is more harmful for the parallel DMA operations,
  319          * since we own the dmar unit lock until whole invalidation
  320          * queue is processed, which includes requests possibly issued
  321          * before our request.
  322          */
  323         dmar_qi_wait_for_seq(unit, &gseq, true);
  324 }
  325 
  326 int
  327 dmar_qi_intr(void *arg)
  328 {
  329         struct dmar_unit *unit;
  330 
  331         unit = arg;
  332         KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled",
  333             unit->iommu.unit));
  334         taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
  335         return (FILTER_HANDLED);
  336 }
  337 
  338 static void
  339 dmar_qi_task(void *arg, int pending __unused)
  340 {
  341         struct dmar_unit *unit;
  342         struct iommu_map_entry *entry;
  343         uint32_t ics;
  344 
  345         unit = arg;
  346 
  347         DMAR_LOCK(unit);
  348         for (;;) {
  349                 entry = TAILQ_FIRST(&unit->tlb_flush_entries);
  350                 if (entry == NULL)
  351                         break;
  352                 if (!dmar_qi_seq_processed(unit, &entry->gseq))
  353                         break;
  354                 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
  355                 DMAR_UNLOCK(unit);
  356                 dmar_domain_free_entry(entry, (entry->flags &
  357                     IOMMU_MAP_ENTRY_QI_NF) == 0);
  358                 DMAR_LOCK(unit);
  359         }
  360         ics = dmar_read4(unit, DMAR_ICS_REG);
  361         if ((ics & DMAR_ICS_IWC) != 0) {
  362                 ics = DMAR_ICS_IWC;
  363                 dmar_write4(unit, DMAR_ICS_REG, ics);
  364         }
  365         if (unit->inv_seq_waiters > 0)
  366                 wakeup(&unit->inv_seq_waiters);
  367         DMAR_UNLOCK(unit);
  368 }
  369 
  370 int
  371 dmar_init_qi(struct dmar_unit *unit)
  372 {
  373         uint64_t iqa;
  374         uint32_t ics;
  375         int qi_sz;
  376 
  377         if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
  378                 return (0);
  379         unit->qi_enabled = 1;
  380         TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
  381         if (!unit->qi_enabled)
  382                 return (0);
  383 
  384         TAILQ_INIT(&unit->tlb_flush_entries);
  385         TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
  386         unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
  387             taskqueue_thread_enqueue, &unit->qi_taskqueue);
  388         taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
  389             "dmar%d qi taskq", unit->iommu.unit);
  390 
  391         unit->inv_waitd_gen = 0;
  392         unit->inv_waitd_seq = 1;
  393 
  394         qi_sz = DMAR_IQA_QS_DEF;
  395         TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
  396         if (qi_sz > DMAR_IQA_QS_MAX)
  397                 qi_sz = DMAR_IQA_QS_MAX;
  398         unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
  399         /* Reserve one descriptor to prevent wraparound. */
  400         unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
  401 
  402         /* The invalidation queue reads by DMARs are always coherent. */
  403         unit->inv_queue = kmem_alloc_contig(unit->inv_queue_size, M_WAITOK |
  404             M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
  405         unit->inv_waitd_seq_hw_phys = pmap_kextract(
  406             (vm_offset_t)&unit->inv_waitd_seq_hw);
  407 
  408         DMAR_LOCK(unit);
  409         dmar_write8(unit, DMAR_IQT_REG, 0);
  410         iqa = pmap_kextract(unit->inv_queue);
  411         iqa |= qi_sz;
  412         dmar_write8(unit, DMAR_IQA_REG, iqa);
  413         dmar_enable_qi(unit);
  414         ics = dmar_read4(unit, DMAR_ICS_REG);
  415         if ((ics & DMAR_ICS_IWC) != 0) {
  416                 ics = DMAR_ICS_IWC;
  417                 dmar_write4(unit, DMAR_ICS_REG, ics);
  418         }
  419         dmar_enable_qi_intr(unit);
  420         DMAR_UNLOCK(unit);
  421 
  422         return (0);
  423 }
  424 
  425 void
  426 dmar_fini_qi(struct dmar_unit *unit)
  427 {
  428         struct iommu_qi_genseq gseq;
  429 
  430         if (!unit->qi_enabled)
  431                 return;
  432         taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
  433         taskqueue_free(unit->qi_taskqueue);
  434         unit->qi_taskqueue = NULL;
  435 
  436         DMAR_LOCK(unit);
  437         /* quisce */
  438         dmar_qi_ensure(unit, 1);
  439         dmar_qi_emit_wait_seq(unit, &gseq, true);
  440         dmar_qi_advance_tail(unit);
  441         dmar_qi_wait_for_seq(unit, &gseq, false);
  442         /* only after the quisce, disable queue */
  443         dmar_disable_qi_intr(unit);
  444         dmar_disable_qi(unit);
  445         KASSERT(unit->inv_seq_waiters == 0,
  446             ("dmar%d: waiters on disabled queue", unit->iommu.unit));
  447         DMAR_UNLOCK(unit);
  448 
  449         kmem_free(unit->inv_queue, unit->inv_queue_size);
  450         unit->inv_queue = 0;
  451         unit->inv_queue_size = 0;
  452         unit->qi_enabled = 0;
  453 }
  454 
  455 void
  456 dmar_enable_qi_intr(struct dmar_unit *unit)
  457 {
  458         uint32_t iectl;
  459 
  460         DMAR_ASSERT_LOCKED(unit);
  461         KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
  462             unit->iommu.unit));
  463         iectl = dmar_read4(unit, DMAR_IECTL_REG);
  464         iectl &= ~DMAR_IECTL_IM;
  465         dmar_write4(unit, DMAR_IECTL_REG, iectl);
  466 }
  467 
  468 void
  469 dmar_disable_qi_intr(struct dmar_unit *unit)
  470 {
  471         uint32_t iectl;
  472 
  473         DMAR_ASSERT_LOCKED(unit);
  474         KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
  475             unit->iommu.unit));
  476         iectl = dmar_read4(unit, DMAR_IECTL_REG);
  477         dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
  478 }

Cache object: 442517981a4a97a91afcbca48fc8f44d


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.