xen_apic.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD$");
   29 
   30 #include <sys/param.h>
   31 #include <sys/bus.h>
   32 #include <sys/kernel.h>
   33 #include <sys/malloc.h>
   34 #include <sys/proc.h>
   35 #include <sys/smp.h>
   36 #include <sys/systm.h>
   37 
   38 #include <vm/vm.h>
   39 #include <vm/pmap.h>
   40 
   41 #include <machine/cpufunc.h>
   42 #include <machine/cpu.h>
   43 #include <machine/intr_machdep.h>
   44 #include <machine/md_var.h>
   45 #include <machine/smp.h>
   46 
   47 #include <x86/apicreg.h>
   48 #include <x86/apicvar.h>
   49 
   50 #include <xen/xen-os.h>
   51 #include <xen/features.h>
   52 #include <xen/gnttab.h>
   53 #include <xen/hypervisor.h>
   54 #include <xen/hvm.h>
   55 #include <xen/xen_intr.h>
   56 
   57 #include <contrib/xen/arch-x86/cpuid.h>
   58 #include <contrib/xen/vcpu.h>
   59 
   60 /*--------------------------- Forward Declarations ---------------------------*/
   61 static driver_filter_t xen_smp_rendezvous_action;
   62 #ifdef __amd64__
   63 static driver_filter_t xen_invlop;
   64 #else
   65 static driver_filter_t xen_invltlb;
   66 static driver_filter_t xen_invlpg;
   67 static driver_filter_t xen_invlrng;
   68 static driver_filter_t xen_invlcache;
   69 #endif
   70 static driver_filter_t xen_ipi_bitmap_handler;
   71 static driver_filter_t xen_cpustop_handler;
   72 static driver_filter_t xen_cpususpend_handler;
   73 static driver_filter_t xen_ipi_swi_handler;
   74 
   75 /*---------------------------------- Macros ----------------------------------*/
   76 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
   77 
   78 /*--------------------------------- Xen IPIs ---------------------------------*/
   79 struct xen_ipi_handler
   80 {
   81         driver_filter_t *filter;
   82         const char      *description;
   83 };
   84 
   85 static struct xen_ipi_handler xen_ipis[] = 
   86 {
   87         [IPI_TO_IDX(IPI_RENDEZVOUS)]    = { xen_smp_rendezvous_action,  "r"   },
   88 #ifdef __amd64__
   89         [IPI_TO_IDX(IPI_INVLOP)]        = { xen_invlop,                 "itlb"},
   90 #else
   91         [IPI_TO_IDX(IPI_INVLTLB)]       = { xen_invltlb,                "itlb"},
   92         [IPI_TO_IDX(IPI_INVLPG)]        = { xen_invlpg,                 "ipg" },
   93         [IPI_TO_IDX(IPI_INVLRNG)]       = { xen_invlrng,                "irg" },
   94         [IPI_TO_IDX(IPI_INVLCACHE)]     = { xen_invlcache,              "ic"  },
   95 #endif
   96         [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler,     "b"   },
   97         [IPI_TO_IDX(IPI_STOP)]          = { xen_cpustop_handler,        "st"  },
   98         [IPI_TO_IDX(IPI_SUSPEND)]       = { xen_cpususpend_handler,     "sp"  },
   99         [IPI_TO_IDX(IPI_SWI)]           = { xen_ipi_swi_handler,        "sw"  },
  100 };
  101 
  102 /*
  103  * Save previous (native) handler as a fallback. Xen < 4.7 doesn't support
  104  * VCPUOP_send_nmi for HVM guests, and thus we need a fallback in that case:
  105  *
  106  * https://lists.freebsd.org/archives/freebsd-xen/2022-January/000032.html
  107  */
  108 void (*native_ipi_vectored)(u_int, int);
  109 
  110 /*------------------------------- Per-CPU Data -------------------------------*/
  111 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
  112 
  113 /*------------------------------- Xen PV APIC --------------------------------*/
  114 
  115 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field)
  116 static int
  117 send_nmi(int dest)
  118 {
  119         unsigned int cpu;
  120         int rc = 0;
  121 
  122         /*
  123          * NMIs are not routed over event channels, and instead delivered as on
  124          * native using the exception vector (#2). Triggering them can be done
  125          * using the local APIC, or an hypercall as a shortcut like it's done
  126          * below.
  127          */
  128         switch(dest) {
  129         case APIC_IPI_DEST_SELF:
  130                 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL);
  131                 break;
  132         case APIC_IPI_DEST_ALL:
  133                 CPU_FOREACH(cpu) {
  134                         rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
  135                             PCPU_ID_GET(cpu, vcpu_id), NULL);
  136                         if (rc != 0)
  137                                 break;
  138                 }
  139                 break;
  140         case APIC_IPI_DEST_OTHERS:
  141                 CPU_FOREACH(cpu) {
  142                         if (cpu != PCPU_GET(cpuid)) {
  143                                 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
  144                                     PCPU_ID_GET(cpu, vcpu_id), NULL);
  145                                 if (rc != 0)
  146                                         break;
  147                         }
  148                 }
  149                 break;
  150         default:
  151                 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
  152                     PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL);
  153                 break;
  154         }
  155 
  156         return rc;
  157 }
  158 #undef PCPU_ID_GET
  159 
  160 static void
  161 xen_pv_lapic_ipi_vectored(u_int vector, int dest)
  162 {
  163         xen_intr_handle_t *ipi_handle;
  164         int ipi_idx, to_cpu, self;
  165         static bool pvnmi = true;
  166 
  167         if (vector >= IPI_NMI_FIRST) {
  168                 if (pvnmi) {
  169                         int rc = send_nmi(dest);
  170 
  171                         if (rc != 0) {
  172                                 printf(
  173     "Sending NMI using hypercall failed (%d) switching to APIC\n", rc);
  174                                 pvnmi = false;
  175                                 native_ipi_vectored(vector, dest);
  176                         }
  177                 } else
  178                         native_ipi_vectored(vector, dest);
  179 
  180                 return;
  181         }
  182 
  183         ipi_idx = IPI_TO_IDX(vector);
  184         if (ipi_idx >= nitems(xen_ipis))
  185                 panic("IPI out of range");
  186 
  187         switch(dest) {
  188         case APIC_IPI_DEST_SELF:
  189                 ipi_handle = DPCPU_GET(ipi_handle);
  190                 xen_intr_signal(ipi_handle[ipi_idx]);
  191                 break;
  192         case APIC_IPI_DEST_ALL:
  193                 CPU_FOREACH(to_cpu) {
  194                         ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
  195                         xen_intr_signal(ipi_handle[ipi_idx]);
  196                 }
  197                 break;
  198         case APIC_IPI_DEST_OTHERS:
  199                 self = PCPU_GET(cpuid);
  200                 CPU_FOREACH(to_cpu) {
  201                         if (to_cpu != self) {
  202                                 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
  203                                 xen_intr_signal(ipi_handle[ipi_idx]);
  204                         }
  205                 }
  206                 break;
  207         default:
  208                 to_cpu = apic_cpuid(dest);
  209                 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
  210                 xen_intr_signal(ipi_handle[ipi_idx]);
  211                 break;
  212         }
  213 }
  214 
  215 /*---------------------------- XEN PV IPI Handlers ---------------------------*/
  216 /*
  217  * These are C clones of the ASM functions found in apic_vector.
  218  */
  219 static int
  220 xen_ipi_bitmap_handler(void *arg)
  221 {
  222         struct trapframe *frame;
  223 
  224         frame = arg;
  225         ipi_bitmap_handler(*frame);
  226         return (FILTER_HANDLED);
  227 }
  228 
  229 static int
  230 xen_smp_rendezvous_action(void *arg)
  231 {
  232 #ifdef COUNT_IPIS
  233         (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
  234 #endif /* COUNT_IPIS */
  235 
  236         smp_rendezvous_action();
  237         return (FILTER_HANDLED);
  238 }
  239 
  240 #ifdef __amd64__
  241 static int
  242 xen_invlop(void *arg)
  243 {
  244 
  245         invlop_handler();
  246         return (FILTER_HANDLED);
  247 }
  248 
  249 #else /* __i386__ */
  250 
  251 static int
  252 xen_invltlb(void *arg)
  253 {
  254 
  255         invltlb_handler();
  256         return (FILTER_HANDLED);
  257 }
  258 
  259 static int
  260 xen_invlpg(void *arg)
  261 {
  262 
  263         invlpg_handler();
  264         return (FILTER_HANDLED);
  265 }
  266 
  267 static int
  268 xen_invlrng(void *arg)
  269 {
  270 
  271         invlrng_handler();
  272         return (FILTER_HANDLED);
  273 }
  274 
  275 static int
  276 xen_invlcache(void *arg)
  277 {
  278 
  279         invlcache_handler();
  280         return (FILTER_HANDLED);
  281 }
  282 #endif /* __amd64__ */
  283 
  284 static int
  285 xen_cpustop_handler(void *arg)
  286 {
  287 
  288         cpustop_handler();
  289         return (FILTER_HANDLED);
  290 }
  291 
  292 static int
  293 xen_cpususpend_handler(void *arg)
  294 {
  295 
  296         cpususpend_handler();
  297         return (FILTER_HANDLED);
  298 }
  299 
  300 static int
  301 xen_ipi_swi_handler(void *arg)
  302 {
  303         struct trapframe *frame = arg;
  304 
  305         ipi_swi_handler(*frame);
  306         return (FILTER_HANDLED);
  307 }
  308 
  309 /*----------------------------- XEN PV IPI setup -----------------------------*/
  310 /*
  311  * Those functions are provided outside of the Xen PV APIC implementation
  312  * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC,
  313  * because on PVHVM there's an emulated LAPIC provided by Xen.
  314  */
  315 static void
  316 xen_cpu_ipi_init(int cpu)
  317 {
  318         xen_intr_handle_t *ipi_handle;
  319         const struct xen_ipi_handler *ipi;
  320         int idx, rc;
  321 
  322         ipi_handle = DPCPU_ID_GET(cpu, ipi_handle);
  323 
  324         for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) {
  325                 if (ipi->filter == NULL) {
  326                         ipi_handle[idx] = NULL;
  327                         continue;
  328                 }
  329 
  330                 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter,
  331                     INTR_TYPE_TTY, &ipi_handle[idx]);
  332                 if (rc != 0)
  333                         panic("Unable to allocate a XEN IPI port");
  334                 xen_intr_describe(ipi_handle[idx], "%s", ipi->description);
  335         }
  336 }
  337 
  338 static void
  339 xen_setup_cpus(void)
  340 {
  341         uint32_t regs[4];
  342         int i;
  343 
  344         if (!xen_vector_callback_enabled)
  345                 return;
  346 
  347         /*
  348          * Check whether the APIC virtualization is hardware assisted, as
  349          * that's faster than using event channels because it avoids the VM
  350          * exit.
  351          */
  352         KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf"));
  353         cpuid_count(xen_cpuid_base + 4, 0, regs);
  354         if ((x2apic_mode && (regs[0] & XEN_HVM_CPUID_X2APIC_VIRT)) ||
  355             (!x2apic_mode && (regs[0] & XEN_HVM_CPUID_APIC_ACCESS_VIRT)))
  356                 return;
  357 
  358         CPU_FOREACH(i)
  359                 xen_cpu_ipi_init(i);
  360 
  361         /* Set the xen pv ipi ops to replace the native ones */
  362         ipi_vectored = xen_pv_lapic_ipi_vectored;
  363         native_ipi_vectored = ipi_vectored;
  364 }
  365 
  366 /* Switch to using PV IPIs as soon as the vcpu_id is set. */
  367 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL);
Cache object: 2dfd0329177e89899c1d7104d33b16b9
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/x86/xen/xen_apic.c

FreeBSD/Linux Kernel Cross Reference
sys/x86/xen/xen_apic.c
