The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/intel_fault.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2013 The FreeBSD Foundation
    5  * All rights reserved.
    6  *
    7  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    8  * under sponsorship from the FreeBSD Foundation.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   29  * SUCH DAMAGE.
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 
   35 #include "opt_acpi.h"
   36 
   37 #include <sys/param.h>
   38 #include <sys/bus.h>
   39 #include <sys/kernel.h>
   40 #include <sys/malloc.h>
   41 #include <sys/memdesc.h>
   42 #include <sys/module.h>
   43 #include <sys/rman.h>
   44 #include <sys/taskqueue.h>
   45 #include <sys/tree.h>
   46 #include <sys/vmem.h>
   47 #include <machine/bus.h>
   48 #include <contrib/dev/acpica/include/acpi.h>
   49 #include <contrib/dev/acpica/include/accommon.h>
   50 #include <dev/acpica/acpivar.h>
   51 #include <dev/pci/pcireg.h>
   52 #include <dev/pci/pcivar.h>
   53 #include <vm/vm.h>
   54 #include <vm/vm_extern.h>
   55 #include <vm/vm_kern.h>
   56 #include <vm/vm_page.h>
   57 #include <vm/vm_map.h>
   58 #include <x86/include/busdma_impl.h>
   59 #include <x86/iommu/intel_reg.h>
   60 #include <x86/iommu/busdma_dmar.h>
   61 #include <x86/iommu/intel_dmar.h>
   62 
   63 /*
   64  * Fault interrupt handling for DMARs.  If advanced fault logging is
   65  * not implemented by hardware, the code emulates it.  Fast interrupt
   66  * handler flushes the fault registers into circular buffer at
   67  * unit->fault_log, and schedules a task.
   68  *
   69  * The fast handler is used since faults usually come in bursts, and
   70  * number of fault log registers is limited, e.g. down to one for 5400
   71  * MCH.  We are trying to reduce the latency for clearing the fault
   72  * register file.  The task is usually long-running, since printf() is
   73  * slow, but this is not problematic because bursts are rare.
   74  *
   75  * For the same reason, each translation unit task is executed in its
   76  * own thread.
   77  *
   78  * XXXKIB It seems there is no hardware available which implements
   79  * advanced fault logging, so the code to handle AFL is not written.
   80  */
   81 
   82 static int
   83 dmar_fault_next(struct dmar_unit *unit, int faultp)
   84 {
   85 
   86         faultp += 2;
   87         if (faultp == unit->fault_log_size)
   88                 faultp = 0;
   89         return (faultp);
   90 }
   91 
   92 static void
   93 dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts)
   94 {
   95         uint32_t clear;
   96 
   97         clear = 0;
   98         if ((fsts & DMAR_FSTS_ITE) != 0) {
   99                 printf("DMAR%d: Invalidation timed out\n", unit->unit);
  100                 clear |= DMAR_FSTS_ITE;
  101         }
  102         if ((fsts & DMAR_FSTS_ICE) != 0) {
  103                 printf("DMAR%d: Invalidation completion error\n",
  104                     unit->unit);
  105                 clear |= DMAR_FSTS_ICE;
  106         }
  107         if ((fsts & DMAR_FSTS_IQE) != 0) {
  108                 printf("DMAR%d: Invalidation queue error\n",
  109                     unit->unit);
  110                 clear |= DMAR_FSTS_IQE;
  111         }
  112         if ((fsts & DMAR_FSTS_APF) != 0) {
  113                 printf("DMAR%d: Advanced pending fault\n", unit->unit);
  114                 clear |= DMAR_FSTS_APF;
  115         }
  116         if ((fsts & DMAR_FSTS_AFO) != 0) {
  117                 printf("DMAR%d: Advanced fault overflow\n", unit->unit);
  118                 clear |= DMAR_FSTS_AFO;
  119         }
  120         if (clear != 0)
  121                 dmar_write4(unit, DMAR_FSTS_REG, clear);
  122 }
  123 
  124 int
  125 dmar_fault_intr(void *arg)
  126 {
  127         struct dmar_unit *unit;
  128         uint64_t fault_rec[2];
  129         uint32_t fsts;
  130         int fri, frir, faultp;
  131         bool enqueue;
  132 
  133         unit = arg;
  134         enqueue = false;
  135         fsts = dmar_read4(unit, DMAR_FSTS_REG);
  136         dmar_fault_intr_clear(unit, fsts);
  137 
  138         if ((fsts & DMAR_FSTS_PPF) == 0)
  139                 goto done;
  140 
  141         fri = DMAR_FSTS_FRI(fsts);
  142         for (;;) {
  143                 frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16;
  144                 fault_rec[1] = dmar_read8(unit, frir + 8);
  145                 if ((fault_rec[1] & DMAR_FRCD2_F) == 0)
  146                         break;
  147                 fault_rec[0] = dmar_read8(unit, frir);
  148                 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
  149                 DMAR_FAULT_LOCK(unit);
  150                 faultp = unit->fault_log_head;
  151                 if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) {
  152                         /* XXXKIB log overflow */
  153                 } else {
  154                         unit->fault_log[faultp] = fault_rec[0];
  155                         unit->fault_log[faultp + 1] = fault_rec[1];
  156                         unit->fault_log_head = dmar_fault_next(unit, faultp);
  157                         enqueue = true;
  158                 }
  159                 DMAR_FAULT_UNLOCK(unit);
  160                 fri += 1;
  161                 if (fri >= DMAR_CAP_NFR(unit->hw_cap))
  162                         fri = 0;
  163         }
  164 
  165 done:
  166         /*
  167          * On SandyBridge, due to errata BJ124, IvyBridge errata
  168          * BV100, and Haswell errata HSD40, "Spurious Intel VT-d
  169          * Interrupts May Occur When the PFO Bit is Set".  Handle the
  170          * cases by clearing overflow bit even if no fault is
  171          * reported.
  172          *
  173          * On IvyBridge, errata BV30 states that clearing clear
  174          * DMAR_FRCD2_F bit in the fault register causes spurious
  175          * interrupt.  Do nothing.
  176          *
  177          */
  178         if ((fsts & DMAR_FSTS_PFO) != 0) {
  179                 printf("DMAR%d: Fault Overflow\n", unit->unit);
  180                 dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO);
  181         }
  182 
  183         if (enqueue) {
  184                 taskqueue_enqueue(unit->fault_taskqueue,
  185                     &unit->fault_task);
  186         }
  187         return (FILTER_HANDLED);
  188 }
  189 
  190 static void
  191 dmar_fault_task(void *arg, int pending __unused)
  192 {
  193         struct dmar_unit *unit;
  194         struct dmar_ctx *ctx;
  195         uint64_t fault_rec[2];
  196         int sid, bus, slot, func, faultp;
  197 
  198         unit = arg;
  199         DMAR_FAULT_LOCK(unit);
  200         for (;;) {
  201                 faultp = unit->fault_log_tail;
  202                 if (faultp == unit->fault_log_head)
  203                         break;
  204 
  205                 fault_rec[0] = unit->fault_log[faultp];
  206                 fault_rec[1] = unit->fault_log[faultp + 1];
  207                 unit->fault_log_tail = dmar_fault_next(unit, faultp);
  208                 DMAR_FAULT_UNLOCK(unit);
  209 
  210                 sid = DMAR_FRCD2_SID(fault_rec[1]);
  211                 printf("DMAR%d: ", unit->unit);
  212                 DMAR_LOCK(unit);
  213                 ctx = dmar_find_ctx_locked(unit, sid);
  214                 if (ctx == NULL) {
  215                         printf("<unknown dev>:");
  216 
  217                         /*
  218                          * Note that the slot and function will not be correct
  219                          * if ARI is in use, but without a ctx entry we have
  220                          * no way of knowing whether ARI is in use or not.
  221                          */
  222                         bus = PCI_RID2BUS(sid);
  223                         slot = PCI_RID2SLOT(sid);
  224                         func = PCI_RID2FUNC(sid);
  225                 } else {
  226                         ctx->flags |= DMAR_CTX_FAULTED;
  227                         ctx->last_fault_rec[0] = fault_rec[0];
  228                         ctx->last_fault_rec[1] = fault_rec[1];
  229                         device_print_prettyname(ctx->ctx_tag.owner);
  230                         bus = pci_get_bus(ctx->ctx_tag.owner);
  231                         slot = pci_get_slot(ctx->ctx_tag.owner);
  232                         func = pci_get_function(ctx->ctx_tag.owner);
  233                 }
  234                 DMAR_UNLOCK(unit);
  235                 printf(
  236                     "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x "
  237                     "addr %jx\n",
  238                     bus, slot, func, sid, DMAR_FRCD2_T(fault_rec[1]),
  239                     DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]),
  240                     (uintmax_t)fault_rec[0]);
  241                 DMAR_FAULT_LOCK(unit);
  242         }
  243         DMAR_FAULT_UNLOCK(unit);
  244 }
  245 
  246 static void
  247 dmar_clear_faults(struct dmar_unit *unit)
  248 {
  249         uint32_t frec, frir, fsts;
  250         int i;
  251 
  252         for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) {
  253                 frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16;
  254                 frec = dmar_read4(unit, frir + 12);
  255                 if ((frec & DMAR_FRCD2_F32) == 0)
  256                         continue;
  257                 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
  258         }
  259         fsts = dmar_read4(unit, DMAR_FSTS_REG);
  260         dmar_write4(unit, DMAR_FSTS_REG, fsts);
  261 }
  262 
  263 int
  264 dmar_init_fault_log(struct dmar_unit *unit)
  265 {
  266 
  267         mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN);
  268         unit->fault_log_size = 256; /* 128 fault log entries */
  269         TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size);
  270         if (unit->fault_log_size % 2 != 0)
  271                 panic("hw.dmar_fault_log_size must be even");
  272         unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size,
  273             M_DEVBUF, M_WAITOK | M_ZERO);
  274 
  275         TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit);
  276         unit->fault_taskqueue = taskqueue_create_fast("dmarff", M_WAITOK,
  277             taskqueue_thread_enqueue, &unit->fault_taskqueue);
  278         taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV,
  279             "dmar%d fault taskq", unit->unit);
  280 
  281         DMAR_LOCK(unit);
  282         dmar_disable_fault_intr(unit);
  283         dmar_clear_faults(unit);
  284         dmar_enable_fault_intr(unit);
  285         DMAR_UNLOCK(unit);
  286 
  287         return (0);
  288 }
  289 
  290 void
  291 dmar_fini_fault_log(struct dmar_unit *unit)
  292 {
  293 
  294         DMAR_LOCK(unit);
  295         dmar_disable_fault_intr(unit);
  296         DMAR_UNLOCK(unit);
  297 
  298         if (unit->fault_taskqueue == NULL)
  299                 return;
  300 
  301         taskqueue_drain(unit->fault_taskqueue, &unit->fault_task);
  302         taskqueue_free(unit->fault_taskqueue);
  303         unit->fault_taskqueue = NULL;
  304         mtx_destroy(&unit->fault_lock);
  305 
  306         free(unit->fault_log, M_DEVBUF);
  307         unit->fault_log = NULL;
  308         unit->fault_log_head = unit->fault_log_tail = 0;
  309 }
  310 
  311 void
  312 dmar_enable_fault_intr(struct dmar_unit *unit)
  313 {
  314         uint32_t fectl;
  315 
  316         DMAR_ASSERT_LOCKED(unit);
  317         fectl = dmar_read4(unit, DMAR_FECTL_REG);
  318         fectl &= ~DMAR_FECTL_IM;
  319         dmar_write4(unit, DMAR_FECTL_REG, fectl);
  320 }
  321 
  322 void
  323 dmar_disable_fault_intr(struct dmar_unit *unit)
  324 {
  325         uint32_t fectl;
  326 
  327         DMAR_ASSERT_LOCKED(unit);
  328         fectl = dmar_read4(unit, DMAR_FECTL_REG);
  329         dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM);
  330 }

Cache object: 5e2bc476c8e7adbe6fe128ee267ee730


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.