The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/intel_qi.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2013 The FreeBSD Foundation
    3  * All rights reserved.
    4  *
    5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    6  * under sponsorship from the FreeBSD Foundation.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/10.4/sys/x86/iommu/intel_qi.c 284019 2015-06-05 08:23:33Z kib $");
   32 
   33 #include "opt_acpi.h"
   34 
   35 #include <sys/param.h>
   36 #include <sys/bus.h>
   37 #include <sys/kernel.h>
   38 #include <sys/malloc.h>
   39 #include <sys/memdesc.h>
   40 #include <sys/module.h>
   41 #include <sys/rman.h>
   42 #include <sys/taskqueue.h>
   43 #include <sys/tree.h>
   44 #include <machine/bus.h>
   45 #include <contrib/dev/acpica/include/acpi.h>
   46 #include <contrib/dev/acpica/include/accommon.h>
   47 #include <dev/acpica/acpivar.h>
   48 #include <vm/vm.h>
   49 #include <vm/vm_extern.h>
   50 #include <vm/vm_kern.h>
   51 #include <vm/vm_page.h>
   52 #include <vm/vm_map.h>
   53 #include <machine/cpu.h>
   54 #include <x86/include/busdma_impl.h>
   55 #include <x86/iommu/intel_reg.h>
   56 #include <x86/iommu/busdma_dmar.h>
   57 #include <x86/iommu/intel_dmar.h>
   58 
   59 static bool
   60 dmar_qi_seq_processed(const struct dmar_unit *unit,
   61     const struct dmar_qi_genseq *pseq)
   62 {
   63 
   64         return (pseq->gen < unit->inv_waitd_gen ||
   65             (pseq->gen == unit->inv_waitd_gen &&
   66              pseq->seq <= unit->inv_waitd_seq_hw));
   67 }
   68 
   69 static int
   70 dmar_enable_qi(struct dmar_unit *unit)
   71 {
   72 
   73         DMAR_ASSERT_LOCKED(unit);
   74         unit->hw_gcmd |= DMAR_GCMD_QIE;
   75         dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
   76         /* XXXKIB should have a timeout */
   77         while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) == 0)
   78                 cpu_spinwait();
   79         return (0);
   80 }
   81 
   82 static int
   83 dmar_disable_qi(struct dmar_unit *unit)
   84 {
   85 
   86         DMAR_ASSERT_LOCKED(unit);
   87         unit->hw_gcmd &= ~DMAR_GCMD_QIE;
   88         dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
   89         /* XXXKIB should have a timeout */
   90         while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) != 0)
   91                 cpu_spinwait();
   92         return (0);
   93 }
   94 
   95 static void
   96 dmar_qi_advance_tail(struct dmar_unit *unit)
   97 {
   98 
   99         DMAR_ASSERT_LOCKED(unit);
  100         dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
  101 }
  102 
  103 static void
  104 dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
  105 {
  106         uint32_t head;
  107         int bytes;
  108 
  109         DMAR_ASSERT_LOCKED(unit);
  110         bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
  111         for (;;) {
  112                 if (bytes <= unit->inv_queue_avail)
  113                         break;
  114                 /* refill */
  115                 head = dmar_read4(unit, DMAR_IQH_REG);
  116                 head &= DMAR_IQH_MASK;
  117                 unit->inv_queue_avail = head - unit->inv_queue_tail -
  118                     DMAR_IQ_DESCR_SZ;
  119                 if (head <= unit->inv_queue_tail)
  120                         unit->inv_queue_avail += unit->inv_queue_size;
  121                 if (bytes <= unit->inv_queue_avail)
  122                         break;
  123 
  124                 /*
  125                  * No space in the queue, do busy wait.  Hardware must
  126                  * make a progress.  But first advance the tail to
  127                  * inform the descriptor streamer about entries we
  128                  * might have already filled, otherwise they could
  129                  * clog the whole queue..
  130                  */
  131                 dmar_qi_advance_tail(unit);
  132                 unit->inv_queue_full++;
  133                 cpu_spinwait();
  134         }
  135         unit->inv_queue_avail -= bytes;
  136 }
  137 
  138 static void
  139 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
  140 {
  141 
  142         DMAR_ASSERT_LOCKED(unit);
  143         *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
  144         unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
  145         KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
  146             ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
  147             (uintmax_t)unit->inv_queue_size));
  148         unit->inv_queue_tail &= unit->inv_queue_size - 1;
  149         *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
  150         unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
  151         KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
  152             ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
  153             (uintmax_t)unit->inv_queue_size));
  154         unit->inv_queue_tail &= unit->inv_queue_size - 1;
  155 }
  156 
  157 static void
  158 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
  159     bool memw, bool fence)
  160 {
  161 
  162         DMAR_ASSERT_LOCKED(unit);
  163         dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
  164             (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
  165             (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
  166             (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
  167             (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
  168             memw ? unit->inv_waitd_seq_hw_phys : 0);
  169 }
  170 
  171 static void
  172 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
  173 {
  174         struct dmar_qi_genseq gsec;
  175         uint32_t seq;
  176 
  177         KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
  178         DMAR_ASSERT_LOCKED(unit);
  179         if (unit->inv_waitd_seq == 0xffffffff) {
  180                 gsec.gen = unit->inv_waitd_gen;
  181                 gsec.seq = unit->inv_waitd_seq;
  182                 dmar_qi_ensure(unit, 1);
  183                 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
  184                 dmar_qi_advance_tail(unit);
  185                 while (!dmar_qi_seq_processed(unit, &gsec))
  186                         cpu_spinwait();
  187                 unit->inv_waitd_gen++;
  188                 unit->inv_waitd_seq = 1;
  189         }
  190         seq = unit->inv_waitd_seq++;
  191         pseq->gen = unit->inv_waitd_gen;
  192         pseq->seq = seq;
  193         dmar_qi_emit_wait_descr(unit, seq, true, true, false);
  194 }
  195 
  196 static void
  197 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq)
  198 {
  199 
  200         DMAR_ASSERT_LOCKED(unit);
  201         unit->inv_seq_waiters++;
  202         while (!dmar_qi_seq_processed(unit, gseq)) {
  203                 if (cold) {
  204                         cpu_spinwait();
  205                 } else {
  206                         msleep(&unit->inv_seq_waiters, &unit->lock, 0,
  207                             "dmarse", hz);
  208                 }
  209         }
  210         unit->inv_seq_waiters--;
  211 }
  212 
  213 void
  214 dmar_qi_invalidate_locked(struct dmar_ctx *ctx, dmar_gaddr_t base,
  215     dmar_gaddr_t size, struct dmar_qi_genseq *pseq)
  216 {
  217         struct dmar_unit *unit;
  218         dmar_gaddr_t isize;
  219         int am;
  220 
  221         unit = ctx->dmar;
  222         DMAR_ASSERT_LOCKED(unit);
  223         for (; size > 0; base += isize, size -= isize) {
  224                 am = calc_am(unit, base, size, &isize);
  225                 dmar_qi_ensure(unit, 1);
  226                 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
  227                     DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
  228                     DMAR_IQ_DESCR_IOTLB_DR |
  229                     DMAR_IQ_DESCR_IOTLB_DID(ctx->domain),
  230                     base | am);
  231         }
  232         if (pseq != NULL) {
  233                 dmar_qi_ensure(unit, 1);
  234                 dmar_qi_emit_wait_seq(unit, pseq);
  235         }
  236         dmar_qi_advance_tail(unit);
  237 }
  238 
  239 void
  240 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
  241 {
  242         struct dmar_qi_genseq gseq;
  243 
  244         DMAR_ASSERT_LOCKED(unit);
  245         dmar_qi_ensure(unit, 2);
  246         dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
  247         dmar_qi_emit_wait_seq(unit, &gseq);
  248         dmar_qi_advance_tail(unit);
  249         dmar_qi_wait_for_seq(unit, &gseq);
  250 }
  251 
  252 void
  253 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
  254 {
  255         struct dmar_qi_genseq gseq;
  256 
  257         DMAR_ASSERT_LOCKED(unit);
  258         dmar_qi_ensure(unit, 2);
  259         dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
  260             DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
  261         dmar_qi_emit_wait_seq(unit, &gseq);
  262         dmar_qi_advance_tail(unit);
  263         dmar_qi_wait_for_seq(unit, &gseq);
  264 }
  265 
  266 int
  267 dmar_qi_intr(void *arg)
  268 {
  269         struct dmar_unit *unit;
  270 
  271         unit = arg;
  272         KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));
  273         taskqueue_enqueue_fast(unit->qi_taskqueue, &unit->qi_task);
  274         return (FILTER_HANDLED);
  275 }
  276 
  277 static void
  278 dmar_qi_task(void *arg, int pending __unused)
  279 {
  280         struct dmar_unit *unit;
  281         struct dmar_map_entry *entry;
  282         uint32_t ics;
  283 
  284         unit = arg;
  285 
  286         DMAR_LOCK(unit);
  287         for (;;) {
  288                 entry = TAILQ_FIRST(&unit->tlb_flush_entries);
  289                 if (entry == NULL)
  290                         break;
  291                 if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) ||
  292                     !dmar_qi_seq_processed(unit, &entry->gseq))
  293                         break;
  294                 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
  295                 DMAR_UNLOCK(unit);
  296                 dmar_ctx_free_entry(entry, (entry->flags &
  297                     DMAR_MAP_ENTRY_QI_NF) == 0);
  298                 DMAR_LOCK(unit);
  299         }
  300         ics = dmar_read4(unit, DMAR_ICS_REG);
  301         if ((ics & DMAR_ICS_IWC) != 0) {
  302                 ics = DMAR_ICS_IWC;
  303                 dmar_write4(unit, DMAR_ICS_REG, ics);
  304         }
  305         if (unit->inv_seq_waiters > 0)
  306                 wakeup(&unit->inv_seq_waiters);
  307         DMAR_UNLOCK(unit);
  308 }
  309 
  310 int
  311 dmar_init_qi(struct dmar_unit *unit)
  312 {
  313         uint64_t iqa;
  314         uint32_t ics;
  315         int qi_sz;
  316 
  317         if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
  318                 return (0);
  319         unit->qi_enabled = 1;
  320         TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
  321         if (!unit->qi_enabled)
  322                 return (0);
  323 
  324         TAILQ_INIT(&unit->tlb_flush_entries);
  325         TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
  326         unit->qi_taskqueue = taskqueue_create_fast("dmar", M_WAITOK,
  327             taskqueue_thread_enqueue, &unit->qi_taskqueue);
  328         taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
  329             "dmar%d qi taskq", unit->unit);
  330 
  331         unit->inv_waitd_gen = 0;
  332         unit->inv_waitd_seq = 1;
  333 
  334         qi_sz = DMAR_IQA_QS_DEF;
  335         TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
  336         if (qi_sz > DMAR_IQA_QS_MAX)
  337                 qi_sz = DMAR_IQA_QS_MAX;
  338         unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
  339         /* Reserve one descriptor to prevent wraparound. */
  340         unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
  341 
  342         /* The invalidation queue reads by DMARs are always coherent. */
  343         unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size,
  344             M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
  345         unit->inv_waitd_seq_hw_phys = pmap_kextract(
  346             (vm_offset_t)&unit->inv_waitd_seq_hw);
  347 
  348         DMAR_LOCK(unit);
  349         dmar_write8(unit, DMAR_IQT_REG, 0);
  350         iqa = pmap_kextract(unit->inv_queue);
  351         iqa |= qi_sz;
  352         dmar_write8(unit, DMAR_IQA_REG, iqa);
  353         dmar_enable_qi(unit);
  354         ics = dmar_read4(unit, DMAR_ICS_REG);
  355         if ((ics & DMAR_ICS_IWC) != 0) {
  356                 ics = DMAR_ICS_IWC;
  357                 dmar_write4(unit, DMAR_ICS_REG, ics);
  358         }
  359         dmar_enable_qi_intr(unit);
  360         DMAR_UNLOCK(unit);
  361 
  362         return (0);
  363 }
  364 
  365 void
  366 dmar_fini_qi(struct dmar_unit *unit)
  367 {
  368         struct dmar_qi_genseq gseq;
  369 
  370         if (unit->qi_enabled)
  371                 return;
  372         taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
  373         taskqueue_free(unit->qi_taskqueue);
  374         unit->qi_taskqueue = NULL;
  375 
  376         DMAR_LOCK(unit);
  377         /* quisce */
  378         dmar_qi_ensure(unit, 1);
  379         dmar_qi_emit_wait_seq(unit, &gseq);
  380         dmar_qi_advance_tail(unit);
  381         dmar_qi_wait_for_seq(unit, &gseq);
  382         /* only after the quisce, disable queue */
  383         dmar_disable_qi_intr(unit);
  384         dmar_disable_qi(unit);
  385         KASSERT(unit->inv_seq_waiters == 0,
  386             ("dmar%d: waiters on disabled queue", unit->unit));
  387         DMAR_UNLOCK(unit);
  388 
  389         kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size);
  390         unit->inv_queue = 0;
  391         unit->inv_queue_size = 0;
  392         unit->qi_enabled = 0;
  393 }
  394 
  395 void
  396 dmar_enable_qi_intr(struct dmar_unit *unit)
  397 {
  398         uint32_t iectl;
  399 
  400         DMAR_ASSERT_LOCKED(unit);
  401         KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
  402         iectl = dmar_read4(unit, DMAR_IECTL_REG);
  403         iectl &= ~DMAR_IECTL_IM;
  404         dmar_write4(unit, DMAR_IECTL_REG, iectl);
  405 }
  406 
  407 void
  408 dmar_disable_qi_intr(struct dmar_unit *unit)
  409 {
  410         uint32_t iectl;
  411 
  412         DMAR_ASSERT_LOCKED(unit);
  413         KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
  414         iectl = dmar_read4(unit, DMAR_IECTL_REG);
  415         dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
  416 }

Cache object: dc9647498fd8df1723b743b50aeea103


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.