The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/intel_qi.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2013 The FreeBSD Foundation
    3  * All rights reserved.
    4  *
    5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    6  * under sponsorship from the FreeBSD Foundation.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/11.1/sys/x86/iommu/intel_qi.c 320357 2017-06-26 12:30:39Z kib $");
   32 
   33 #include "opt_acpi.h"
   34 
   35 #include <sys/param.h>
   36 #include <sys/bus.h>
   37 #include <sys/kernel.h>
   38 #include <sys/malloc.h>
   39 #include <sys/memdesc.h>
   40 #include <sys/module.h>
   41 #include <sys/rman.h>
   42 #include <sys/taskqueue.h>
   43 #include <sys/time.h>
   44 #include <sys/tree.h>
   45 #include <sys/vmem.h>
   46 #include <machine/bus.h>
   47 #include <contrib/dev/acpica/include/acpi.h>
   48 #include <contrib/dev/acpica/include/accommon.h>
   49 #include <dev/acpica/acpivar.h>
   50 #include <vm/vm.h>
   51 #include <vm/vm_extern.h>
   52 #include <vm/vm_kern.h>
   53 #include <vm/vm_page.h>
   54 #include <vm/vm_map.h>
   55 #include <machine/cpu.h>
   56 #include <x86/include/busdma_impl.h>
   57 #include <x86/iommu/intel_reg.h>
   58 #include <x86/iommu/busdma_dmar.h>
   59 #include <x86/iommu/intel_dmar.h>
   60 
   61 static bool
   62 dmar_qi_seq_processed(const struct dmar_unit *unit,
   63     const struct dmar_qi_genseq *pseq)
   64 {
   65 
   66         return (pseq->gen < unit->inv_waitd_gen ||
   67             (pseq->gen == unit->inv_waitd_gen &&
   68              pseq->seq <= unit->inv_waitd_seq_hw));
   69 }
   70 
   71 static int
   72 dmar_enable_qi(struct dmar_unit *unit)
   73 {
   74         int error;
   75 
   76         DMAR_ASSERT_LOCKED(unit);
   77         unit->hw_gcmd |= DMAR_GCMD_QIE;
   78         dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
   79         DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
   80             != 0));
   81         return (error);
   82 }
   83 
   84 static int
   85 dmar_disable_qi(struct dmar_unit *unit)
   86 {
   87         int error;
   88 
   89         DMAR_ASSERT_LOCKED(unit);
   90         unit->hw_gcmd &= ~DMAR_GCMD_QIE;
   91         dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
   92         DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
   93             == 0));
   94         return (error);
   95 }
   96 
   97 static void
   98 dmar_qi_advance_tail(struct dmar_unit *unit)
   99 {
  100 
  101         DMAR_ASSERT_LOCKED(unit);
  102         dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
  103 }
  104 
  105 static void
  106 dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
  107 {
  108         uint32_t head;
  109         int bytes;
  110 
  111         DMAR_ASSERT_LOCKED(unit);
  112         bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
  113         for (;;) {
  114                 if (bytes <= unit->inv_queue_avail)
  115                         break;
  116                 /* refill */
  117                 head = dmar_read4(unit, DMAR_IQH_REG);
  118                 head &= DMAR_IQH_MASK;
  119                 unit->inv_queue_avail = head - unit->inv_queue_tail -
  120                     DMAR_IQ_DESCR_SZ;
  121                 if (head <= unit->inv_queue_tail)
  122                         unit->inv_queue_avail += unit->inv_queue_size;
  123                 if (bytes <= unit->inv_queue_avail)
  124                         break;
  125 
  126                 /*
  127                  * No space in the queue, do busy wait.  Hardware must
  128                  * make a progress.  But first advance the tail to
  129                  * inform the descriptor streamer about entries we
  130                  * might have already filled, otherwise they could
  131                  * clog the whole queue..
  132                  */
  133                 dmar_qi_advance_tail(unit);
  134                 unit->inv_queue_full++;
  135                 cpu_spinwait();
  136         }
  137         unit->inv_queue_avail -= bytes;
  138 }
  139 
  140 static void
  141 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
  142 {
  143 
  144         DMAR_ASSERT_LOCKED(unit);
  145         *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
  146         unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
  147         KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
  148             ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
  149             (uintmax_t)unit->inv_queue_size));
  150         unit->inv_queue_tail &= unit->inv_queue_size - 1;
  151         *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
  152         unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
  153         KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
  154             ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
  155             (uintmax_t)unit->inv_queue_size));
  156         unit->inv_queue_tail &= unit->inv_queue_size - 1;
  157 }
  158 
  159 static void
  160 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
  161     bool memw, bool fence)
  162 {
  163 
  164         DMAR_ASSERT_LOCKED(unit);
  165         dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
  166             (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
  167             (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
  168             (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
  169             (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
  170             memw ? unit->inv_waitd_seq_hw_phys : 0);
  171 }
  172 
  173 static void
  174 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq,
  175     bool emit_wait)
  176 {
  177         struct dmar_qi_genseq gsec;
  178         uint32_t seq;
  179 
  180         KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
  181         DMAR_ASSERT_LOCKED(unit);
  182         if (unit->inv_waitd_seq == 0xffffffff) {
  183                 gsec.gen = unit->inv_waitd_gen;
  184                 gsec.seq = unit->inv_waitd_seq;
  185                 dmar_qi_ensure(unit, 1);
  186                 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
  187                 dmar_qi_advance_tail(unit);
  188                 while (!dmar_qi_seq_processed(unit, &gsec))
  189                         cpu_spinwait();
  190                 unit->inv_waitd_gen++;
  191                 unit->inv_waitd_seq = 1;
  192         }
  193         seq = unit->inv_waitd_seq++;
  194         pseq->gen = unit->inv_waitd_gen;
  195         pseq->seq = seq;
  196         if (emit_wait) {
  197                 dmar_qi_ensure(unit, 1);
  198                 dmar_qi_emit_wait_descr(unit, seq, true, true, false);
  199         }
  200 }
  201 
  202 static void
  203 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq,
  204     bool nowait)
  205 {
  206 
  207         DMAR_ASSERT_LOCKED(unit);
  208         unit->inv_seq_waiters++;
  209         while (!dmar_qi_seq_processed(unit, gseq)) {
  210                 if (cold || nowait) {
  211                         cpu_spinwait();
  212                 } else {
  213                         msleep(&unit->inv_seq_waiters, &unit->lock, 0,
  214                             "dmarse", hz);
  215                 }
  216         }
  217         unit->inv_seq_waiters--;
  218 }
  219 
  220 void
  221 dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
  222     dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait)
  223 {
  224         struct dmar_unit *unit;
  225         dmar_gaddr_t isize;
  226         int am;
  227 
  228         unit = domain->dmar;
  229         DMAR_ASSERT_LOCKED(unit);
  230         for (; size > 0; base += isize, size -= isize) {
  231                 am = calc_am(unit, base, size, &isize);
  232                 dmar_qi_ensure(unit, 1);
  233                 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
  234                     DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
  235                     DMAR_IQ_DESCR_IOTLB_DR |
  236                     DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
  237                     base | am);
  238         }
  239         dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
  240         dmar_qi_advance_tail(unit);
  241 }
  242 
  243 void
  244 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
  245 {
  246         struct dmar_qi_genseq gseq;
  247 
  248         DMAR_ASSERT_LOCKED(unit);
  249         dmar_qi_ensure(unit, 2);
  250         dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
  251         dmar_qi_emit_wait_seq(unit, &gseq, true);
  252         dmar_qi_advance_tail(unit);
  253         dmar_qi_wait_for_seq(unit, &gseq, false);
  254 }
  255 
  256 void
  257 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
  258 {
  259         struct dmar_qi_genseq gseq;
  260 
  261         DMAR_ASSERT_LOCKED(unit);
  262         dmar_qi_ensure(unit, 2);
  263         dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
  264             DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
  265         dmar_qi_emit_wait_seq(unit, &gseq, true);
  266         dmar_qi_advance_tail(unit);
  267         dmar_qi_wait_for_seq(unit, &gseq, false);
  268 }
  269 
  270 void
  271 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
  272 {
  273         struct dmar_qi_genseq gseq;
  274 
  275         DMAR_ASSERT_LOCKED(unit);
  276         dmar_qi_ensure(unit, 2);
  277         dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
  278         dmar_qi_emit_wait_seq(unit, &gseq, true);
  279         dmar_qi_advance_tail(unit);
  280         dmar_qi_wait_for_seq(unit, &gseq, false);
  281 }
  282 
  283 void
  284 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
  285 {
  286         struct dmar_qi_genseq gseq;
  287         u_int c, l;
  288 
  289         DMAR_ASSERT_LOCKED(unit);
  290         KASSERT(start < unit->irte_cnt && start < start + cnt &&
  291             start + cnt <= unit->irte_cnt,
  292             ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
  293         for (; cnt > 0; cnt -= c, start += c) {
  294                 l = ffs(start | cnt) - 1;
  295                 c = 1 << l;
  296                 dmar_qi_ensure(unit, 1);
  297                 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
  298                     DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
  299                     DMAR_IQ_DESCR_IEC_IM(l), 0);
  300         }
  301         dmar_qi_ensure(unit, 1);
  302         dmar_qi_emit_wait_seq(unit, &gseq, true);
  303         dmar_qi_advance_tail(unit);
  304 
  305         /*
  306          * The caller of the function, in particular,
  307          * dmar_ir_program_irte(), may be called from the context
  308          * where the sleeping is forbidden (in fact, the
  309          * intr_table_lock mutex may be held, locked from
  310          * intr_shuffle_irqs()).  Wait for the invalidation completion
  311          * using the busy wait.
  312          *
  313          * The impact on the interrupt input setup code is small, the
  314          * expected overhead is comparable with the chipset register
  315          * read.  It is more harmful for the parallel DMA operations,
  316          * since we own the dmar unit lock until whole invalidation
  317          * queue is processed, which includes requests possibly issued
  318          * before our request.
  319          */
  320         dmar_qi_wait_for_seq(unit, &gseq, true);
  321 }
  322 
  323 int
  324 dmar_qi_intr(void *arg)
  325 {
  326         struct dmar_unit *unit;
  327 
  328         unit = arg;
  329         KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));
  330         taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
  331         return (FILTER_HANDLED);
  332 }
  333 
  334 static void
  335 dmar_qi_task(void *arg, int pending __unused)
  336 {
  337         struct dmar_unit *unit;
  338         struct dmar_map_entry *entry;
  339         uint32_t ics;
  340 
  341         unit = arg;
  342 
  343         DMAR_LOCK(unit);
  344         for (;;) {
  345                 entry = TAILQ_FIRST(&unit->tlb_flush_entries);
  346                 if (entry == NULL)
  347                         break;
  348                 if (!dmar_qi_seq_processed(unit, &entry->gseq))
  349                         break;
  350                 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
  351                 DMAR_UNLOCK(unit);
  352                 dmar_domain_free_entry(entry, (entry->flags &
  353                     DMAR_MAP_ENTRY_QI_NF) == 0);
  354                 DMAR_LOCK(unit);
  355         }
  356         ics = dmar_read4(unit, DMAR_ICS_REG);
  357         if ((ics & DMAR_ICS_IWC) != 0) {
  358                 ics = DMAR_ICS_IWC;
  359                 dmar_write4(unit, DMAR_ICS_REG, ics);
  360         }
  361         if (unit->inv_seq_waiters > 0)
  362                 wakeup(&unit->inv_seq_waiters);
  363         DMAR_UNLOCK(unit);
  364 }
  365 
  366 int
  367 dmar_init_qi(struct dmar_unit *unit)
  368 {
  369         uint64_t iqa;
  370         uint32_t ics;
  371         int qi_sz;
  372 
  373         if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
  374                 return (0);
  375         unit->qi_enabled = 1;
  376         TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
  377         if (!unit->qi_enabled)
  378                 return (0);
  379 
  380         TAILQ_INIT(&unit->tlb_flush_entries);
  381         TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
  382         unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
  383             taskqueue_thread_enqueue, &unit->qi_taskqueue);
  384         taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
  385             "dmar%d qi taskq", unit->unit);
  386 
  387         unit->inv_waitd_gen = 0;
  388         unit->inv_waitd_seq = 1;
  389 
  390         qi_sz = DMAR_IQA_QS_DEF;
  391         TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
  392         if (qi_sz > DMAR_IQA_QS_MAX)
  393                 qi_sz = DMAR_IQA_QS_MAX;
  394         unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
  395         /* Reserve one descriptor to prevent wraparound. */
  396         unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
  397 
  398         /* The invalidation queue reads by DMARs are always coherent. */
  399         unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size,
  400             M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
  401         unit->inv_waitd_seq_hw_phys = pmap_kextract(
  402             (vm_offset_t)&unit->inv_waitd_seq_hw);
  403 
  404         DMAR_LOCK(unit);
  405         dmar_write8(unit, DMAR_IQT_REG, 0);
  406         iqa = pmap_kextract(unit->inv_queue);
  407         iqa |= qi_sz;
  408         dmar_write8(unit, DMAR_IQA_REG, iqa);
  409         dmar_enable_qi(unit);
  410         ics = dmar_read4(unit, DMAR_ICS_REG);
  411         if ((ics & DMAR_ICS_IWC) != 0) {
  412                 ics = DMAR_ICS_IWC;
  413                 dmar_write4(unit, DMAR_ICS_REG, ics);
  414         }
  415         dmar_enable_qi_intr(unit);
  416         DMAR_UNLOCK(unit);
  417 
  418         return (0);
  419 }
  420 
  421 void
  422 dmar_fini_qi(struct dmar_unit *unit)
  423 {
  424         struct dmar_qi_genseq gseq;
  425 
  426         if (unit->qi_enabled)
  427                 return;
  428         taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
  429         taskqueue_free(unit->qi_taskqueue);
  430         unit->qi_taskqueue = NULL;
  431 
  432         DMAR_LOCK(unit);
  433         /* quisce */
  434         dmar_qi_ensure(unit, 1);
  435         dmar_qi_emit_wait_seq(unit, &gseq, true);
  436         dmar_qi_advance_tail(unit);
  437         dmar_qi_wait_for_seq(unit, &gseq, false);
  438         /* only after the quisce, disable queue */
  439         dmar_disable_qi_intr(unit);
  440         dmar_disable_qi(unit);
  441         KASSERT(unit->inv_seq_waiters == 0,
  442             ("dmar%d: waiters on disabled queue", unit->unit));
  443         DMAR_UNLOCK(unit);
  444 
  445         kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size);
  446         unit->inv_queue = 0;
  447         unit->inv_queue_size = 0;
  448         unit->qi_enabled = 0;
  449 }
  450 
  451 void
  452 dmar_enable_qi_intr(struct dmar_unit *unit)
  453 {
  454         uint32_t iectl;
  455 
  456         DMAR_ASSERT_LOCKED(unit);
  457         KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
  458         iectl = dmar_read4(unit, DMAR_IECTL_REG);
  459         iectl &= ~DMAR_IECTL_IM;
  460         dmar_write4(unit, DMAR_IECTL_REG, iectl);
  461 }
  462 
  463 void
  464 dmar_disable_qi_intr(struct dmar_unit *unit)
  465 {
  466         uint32_t iectl;
  467 
  468         DMAR_ASSERT_LOCKED(unit);
  469         KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
  470         iectl = dmar_read4(unit, DMAR_IECTL_REG);
  471         dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
  472 }

Cache object: 7b310bcd104e138019b9439fdd785f95


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.