The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/x86/xen/xen_apic.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD$");
   29 
   30 #include <sys/param.h>
   31 #include <sys/bus.h>
   32 #include <sys/kernel.h>
   33 #include <sys/malloc.h>
   34 #include <sys/proc.h>
   35 #include <sys/smp.h>
   36 #include <sys/systm.h>
   37 
   38 #include <vm/vm.h>
   39 #include <vm/pmap.h>
   40 
   41 #include <machine/cpufunc.h>
   42 #include <machine/cpu.h>
   43 #include <machine/intr_machdep.h>
   44 #include <machine/md_var.h>
   45 #include <machine/smp.h>
   46 
   47 #include <x86/apicreg.h>
   48 #include <x86/apicvar.h>
   49 
   50 #include <xen/xen-os.h>
   51 #include <xen/features.h>
   52 #include <xen/gnttab.h>
   53 #include <xen/hypervisor.h>
   54 #include <xen/hvm.h>
   55 #include <xen/xen_intr.h>
   56 
   57 #include <xen/interface/vcpu.h>
   58 
   59 /*--------------------------------- Macros -----------------------------------*/
   60 
   61 #define XEN_APIC_UNSUPPORTED \
   62         panic("%s: not available in Xen PV port.", __func__)
   63 
   64 
   65 /*--------------------------- Forward Declarations ---------------------------*/
   66 #ifdef SMP
   67 static driver_filter_t xen_smp_rendezvous_action;
   68 static driver_filter_t xen_invltlb;
   69 static driver_filter_t xen_invlpg;
   70 static driver_filter_t xen_invlrng;
   71 static driver_filter_t xen_invlcache;
   72 static driver_filter_t xen_ipi_bitmap_handler;
   73 static driver_filter_t xen_cpustop_handler;
   74 static driver_filter_t xen_cpususpend_handler;
   75 #endif
   76 
   77 /*---------------------------------- Macros ----------------------------------*/
   78 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
   79 
   80 /*--------------------------------- Xen IPIs ---------------------------------*/
   81 #ifdef SMP
   82 struct xen_ipi_handler
   83 {
   84         driver_filter_t *filter;
   85         const char      *description;
   86 };
   87 
   88 static struct xen_ipi_handler xen_ipis[] = 
   89 {
   90         [IPI_TO_IDX(IPI_RENDEZVOUS)]    = { xen_smp_rendezvous_action,  "r"   },
   91         [IPI_TO_IDX(IPI_INVLTLB)]       = { xen_invltlb,                "itlb"},
   92         [IPI_TO_IDX(IPI_INVLPG)]        = { xen_invlpg,                 "ipg" },
   93         [IPI_TO_IDX(IPI_INVLRNG)]       = { xen_invlrng,                "irg" },
   94         [IPI_TO_IDX(IPI_INVLCACHE)]     = { xen_invlcache,              "ic"  },
   95         [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler,     "b"   },
   96         [IPI_TO_IDX(IPI_STOP)]          = { xen_cpustop_handler,        "st"  },
   97         [IPI_TO_IDX(IPI_SUSPEND)]       = { xen_cpususpend_handler,     "sp"  },
   98 };
   99 #endif
  100 
  101 /*------------------------------- Per-CPU Data -------------------------------*/
  102 #ifdef SMP
  103 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
  104 #endif
  105 
  106 /*------------------------------- Xen PV APIC --------------------------------*/
  107 
  108 static void
  109 xen_pv_lapic_create(u_int apic_id, int boot_cpu)
  110 {
  111 #ifdef SMP
  112         cpu_add(apic_id, boot_cpu);
  113 #endif
  114 }
  115 
  116 static void
  117 xen_pv_lapic_init(vm_paddr_t addr)
  118 {
  119 
  120 }
  121 
  122 static void
  123 xen_pv_lapic_setup(int boot)
  124 {
  125 
  126 }
  127 
  128 static void
  129 xen_pv_lapic_dump(const char *str)
  130 {
  131 
  132         printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str);
  133 }
  134 
  135 static void
  136 xen_pv_lapic_disable(void)
  137 {
  138 
  139 }
  140 
  141 static bool
  142 xen_pv_lapic_is_x2apic(void)
  143 {
  144 
  145         return (false);
  146 }
  147 
  148 static void
  149 xen_pv_lapic_eoi(void)
  150 {
  151 
  152         XEN_APIC_UNSUPPORTED;
  153 }
  154 
  155 static int
  156 xen_pv_lapic_id(void)
  157 {
  158 
  159         return (PCPU_GET(apic_id));
  160 }
  161 
  162 static int
  163 xen_pv_lapic_intr_pending(u_int vector)
  164 {
  165 
  166         XEN_APIC_UNSUPPORTED;
  167         return (0);
  168 }
  169 
  170 static u_int
  171 xen_pv_apic_cpuid(u_int apic_id)
  172 {
  173 #ifdef SMP
  174         return (apic_cpuids[apic_id]);
  175 #else
  176         return (0);
  177 #endif
  178 }
  179 
  180 static u_int
  181 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq)
  182 {
  183 
  184         XEN_APIC_UNSUPPORTED;
  185         return (0);
  186 }
  187 
  188 static u_int
  189 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
  190 {
  191 
  192         XEN_APIC_UNSUPPORTED;
  193         return (0);
  194 }
  195 
  196 static void
  197 xen_pv_apic_disable_vector(u_int apic_id, u_int vector)
  198 {
  199 
  200         XEN_APIC_UNSUPPORTED;
  201 }
  202 
  203 static void
  204 xen_pv_apic_enable_vector(u_int apic_id, u_int vector)
  205 {
  206 
  207         XEN_APIC_UNSUPPORTED;
  208 }
  209 
  210 static void
  211 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
  212 {
  213 
  214         XEN_APIC_UNSUPPORTED;
  215 }
  216 
  217 static void
  218 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
  219 {
  220 
  221         XEN_APIC_UNSUPPORTED;
  222 }
  223 
  224 static int
  225 xen_pv_lapic_enable_pmc(void)
  226 {
  227 
  228         XEN_APIC_UNSUPPORTED;
  229         return (0);
  230 }
  231 
  232 static void
  233 xen_pv_lapic_disable_pmc(void)
  234 {
  235 
  236         XEN_APIC_UNSUPPORTED;
  237 }
  238 
  239 static void
  240 xen_pv_lapic_reenable_pmc(void)
  241 {
  242 
  243         XEN_APIC_UNSUPPORTED;
  244 }
  245 
  246 static void
  247 xen_pv_lapic_enable_cmc(void)
  248 {
  249 
  250 }
  251 
  252 #ifdef SMP
  253 static void
  254 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest)
  255 {
  256 
  257         XEN_APIC_UNSUPPORTED;
  258 }
  259 
  260 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field)
  261 static void
  262 send_nmi(int dest)
  263 {
  264         unsigned int cpu;
  265 
  266         /*
  267          * NMIs are not routed over event channels, and instead delivered as on
  268          * native using the exception vector (#2). Triggering them can be done
  269          * using the local APIC, or an hypercall as a shortcut like it's done
  270          * below.
  271          */
  272         switch(dest) {
  273         case APIC_IPI_DEST_SELF:
  274                 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL);
  275                 break;
  276         case APIC_IPI_DEST_ALL:
  277                 CPU_FOREACH(cpu)
  278                         HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
  279                             PCPU_ID_GET(cpu, vcpu_id), NULL);
  280                 break;
  281         case APIC_IPI_DEST_OTHERS:
  282                 CPU_FOREACH(cpu)
  283                         if (cpu != PCPU_GET(cpuid))
  284                                 HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
  285                                     PCPU_ID_GET(cpu, vcpu_id), NULL);
  286                 break;
  287         default:
  288                 HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
  289                     PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL);
  290                 break;
  291         }
  292 }
  293 #undef PCPU_ID_GET
  294 
  295 static void
  296 xen_pv_lapic_ipi_vectored(u_int vector, int dest)
  297 {
  298         xen_intr_handle_t *ipi_handle;
  299         int ipi_idx, to_cpu, self;
  300 
  301         if (vector >= IPI_NMI_FIRST) {
  302                 send_nmi(dest);
  303                 return;
  304         }
  305 
  306         ipi_idx = IPI_TO_IDX(vector);
  307         if (ipi_idx >= nitems(xen_ipis))
  308                 panic("IPI out of range");
  309 
  310         switch(dest) {
  311         case APIC_IPI_DEST_SELF:
  312                 ipi_handle = DPCPU_GET(ipi_handle);
  313                 xen_intr_signal(ipi_handle[ipi_idx]);
  314                 break;
  315         case APIC_IPI_DEST_ALL:
  316                 CPU_FOREACH(to_cpu) {
  317                         ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
  318                         xen_intr_signal(ipi_handle[ipi_idx]);
  319                 }
  320                 break;
  321         case APIC_IPI_DEST_OTHERS:
  322                 self = PCPU_GET(cpuid);
  323                 CPU_FOREACH(to_cpu) {
  324                         if (to_cpu != self) {
  325                                 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
  326                                 xen_intr_signal(ipi_handle[ipi_idx]);
  327                         }
  328                 }
  329                 break;
  330         default:
  331                 to_cpu = apic_cpuid(dest);
  332                 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
  333                 xen_intr_signal(ipi_handle[ipi_idx]);
  334                 break;
  335         }
  336 }
  337 
  338 static int
  339 xen_pv_lapic_ipi_wait(int delay)
  340 {
  341 
  342         XEN_APIC_UNSUPPORTED;
  343         return (0);
  344 }
  345 #endif  /* SMP */
  346 
  347 static int
  348 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc)
  349 {
  350 
  351         XEN_APIC_UNSUPPORTED;
  352         return (-1);
  353 }
  354 
  355 static void
  356 xen_pv_lapic_ipi_free(int vector)
  357 {
  358 
  359         XEN_APIC_UNSUPPORTED;
  360 }
  361 
  362 static int
  363 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked)
  364 {
  365 
  366         XEN_APIC_UNSUPPORTED;
  367         return (0);
  368 }
  369 
  370 static int
  371 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode)
  372 {
  373 
  374         XEN_APIC_UNSUPPORTED;
  375         return (0);
  376 }
  377 
  378 static int
  379 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol)
  380 {
  381 
  382         XEN_APIC_UNSUPPORTED;
  383         return (0);
  384 }
  385 
  386 static int
  387 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
  388     enum intr_trigger trigger)
  389 {
  390 
  391         XEN_APIC_UNSUPPORTED;
  392         return (0);
  393 }
  394 
  395 /* Xen apic_ops implementation */
  396 struct apic_ops xen_apic_ops = {
  397         .create                 = xen_pv_lapic_create,
  398         .init                   = xen_pv_lapic_init,
  399         .xapic_mode             = xen_pv_lapic_disable,
  400         .is_x2apic              = xen_pv_lapic_is_x2apic,
  401         .setup                  = xen_pv_lapic_setup,
  402         .dump                   = xen_pv_lapic_dump,
  403         .disable                = xen_pv_lapic_disable,
  404         .eoi                    = xen_pv_lapic_eoi,
  405         .id                     = xen_pv_lapic_id,
  406         .intr_pending           = xen_pv_lapic_intr_pending,
  407         .set_logical_id         = xen_pv_lapic_set_logical_id,
  408         .cpuid                  = xen_pv_apic_cpuid,
  409         .alloc_vector           = xen_pv_apic_alloc_vector,
  410         .alloc_vectors          = xen_pv_apic_alloc_vectors,
  411         .enable_vector          = xen_pv_apic_enable_vector,
  412         .disable_vector         = xen_pv_apic_disable_vector,
  413         .free_vector            = xen_pv_apic_free_vector,
  414         .enable_pmc             = xen_pv_lapic_enable_pmc,
  415         .disable_pmc            = xen_pv_lapic_disable_pmc,
  416         .reenable_pmc           = xen_pv_lapic_reenable_pmc,
  417         .enable_cmc             = xen_pv_lapic_enable_cmc,
  418 #ifdef SMP
  419         .ipi_raw                = xen_pv_lapic_ipi_raw,
  420         .ipi_vectored           = xen_pv_lapic_ipi_vectored,
  421         .ipi_wait               = xen_pv_lapic_ipi_wait,
  422 #endif
  423         .ipi_alloc              = xen_pv_lapic_ipi_alloc,
  424         .ipi_free               = xen_pv_lapic_ipi_free,
  425         .set_lvt_mask           = xen_pv_lapic_set_lvt_mask,
  426         .set_lvt_mode           = xen_pv_lapic_set_lvt_mode,
  427         .set_lvt_polarity       = xen_pv_lapic_set_lvt_polarity,
  428         .set_lvt_triggermode    = xen_pv_lapic_set_lvt_triggermode,
  429 };
  430 
  431 #ifdef SMP
  432 /*---------------------------- XEN PV IPI Handlers ---------------------------*/
  433 /*
  434  * These are C clones of the ASM functions found in apic_vector.
  435  */
  436 static int
  437 xen_ipi_bitmap_handler(void *arg)
  438 {
  439         struct trapframe *frame;
  440 
  441         frame = arg;
  442         ipi_bitmap_handler(*frame);
  443         return (FILTER_HANDLED);
  444 }
  445 
  446 static int
  447 xen_smp_rendezvous_action(void *arg)
  448 {
  449 #ifdef COUNT_IPIS
  450         (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
  451 #endif /* COUNT_IPIS */
  452 
  453         smp_rendezvous_action();
  454         return (FILTER_HANDLED);
  455 }
  456 
  457 static int
  458 xen_invltlb(void *arg)
  459 {
  460 
  461         invltlb_handler();
  462         return (FILTER_HANDLED);
  463 }
  464 
  465 #ifdef __amd64__
  466 static int
  467 xen_invltlb_invpcid(void *arg)
  468 {
  469 
  470         invltlb_invpcid_handler();
  471         return (FILTER_HANDLED);
  472 }
  473 
  474 static int
  475 xen_invltlb_pcid(void *arg)
  476 {
  477 
  478         invltlb_pcid_handler();
  479         return (FILTER_HANDLED);
  480 }
  481 
  482 static int
  483 xen_invltlb_invpcid_pti(void *arg)
  484 {
  485 
  486         invltlb_invpcid_pti_handler();
  487         return (FILTER_HANDLED);
  488 }
  489 
  490 static int
  491 xen_invlpg_invpcid_handler(void *arg)
  492 {
  493 
  494         invlpg_invpcid_handler();
  495         return (FILTER_HANDLED);
  496 }
  497 
  498 static int
  499 xen_invlpg_pcid_handler(void *arg)
  500 {
  501 
  502         invlpg_pcid_handler();
  503         return (FILTER_HANDLED);
  504 }
  505 
  506 static int
  507 xen_invlrng_invpcid_handler(void *arg)
  508 {
  509 
  510         invlrng_invpcid_handler();
  511         return (FILTER_HANDLED);
  512 }
  513 
  514 static int
  515 xen_invlrng_pcid_handler(void *arg)
  516 {
  517 
  518         invlrng_pcid_handler();
  519         return (FILTER_HANDLED);
  520 }
  521 #endif
  522 
  523 static int
  524 xen_invlpg(void *arg)
  525 {
  526 
  527         invlpg_handler();
  528         return (FILTER_HANDLED);
  529 }
  530 
  531 static int
  532 xen_invlrng(void *arg)
  533 {
  534 
  535         invlrng_handler();
  536         return (FILTER_HANDLED);
  537 }
  538 
  539 static int
  540 xen_invlcache(void *arg)
  541 {
  542 
  543         invlcache_handler();
  544         return (FILTER_HANDLED);
  545 }
  546 
  547 static int
  548 xen_cpustop_handler(void *arg)
  549 {
  550 
  551         cpustop_handler();
  552         return (FILTER_HANDLED);
  553 }
  554 
  555 static int
  556 xen_cpususpend_handler(void *arg)
  557 {
  558 
  559         cpususpend_handler();
  560         return (FILTER_HANDLED);
  561 }
  562 
  563 /*----------------------------- XEN PV IPI setup -----------------------------*/
  564 /*
  565  * Those functions are provided outside of the Xen PV APIC implementation
  566  * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC,
  567  * because on PVHVM there's an emulated LAPIC provided by Xen.
  568  */
  569 static void
  570 xen_cpu_ipi_init(int cpu)
  571 {
  572         xen_intr_handle_t *ipi_handle;
  573         const struct xen_ipi_handler *ipi;
  574         int idx, rc;
  575 
  576         ipi_handle = DPCPU_ID_GET(cpu, ipi_handle);
  577 
  578         for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) {
  579 
  580                 if (ipi->filter == NULL) {
  581                         ipi_handle[idx] = NULL;
  582                         continue;
  583                 }
  584 
  585                 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter,
  586                     INTR_TYPE_TTY, &ipi_handle[idx]);
  587                 if (rc != 0)
  588                         panic("Unable to allocate a XEN IPI port");
  589                 xen_intr_describe(ipi_handle[idx], "%s", ipi->description);
  590         }
  591 }
  592 
  593 static void
  594 xen_setup_cpus(void)
  595 {
  596         int i;
  597 
  598         if (!xen_vector_callback_enabled)
  599                 return;
  600 
  601 #ifdef __amd64__
  602         if (pmap_pcid_enabled) {
  603                 if (pti)
  604                         xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter =
  605                             invpcid_works ? xen_invltlb_invpcid_pti :
  606                             xen_invltlb_pcid;
  607                 else
  608                         xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter =
  609                             invpcid_works ? xen_invltlb_invpcid :
  610                             xen_invltlb_pcid;
  611                 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = invpcid_works ?
  612                     xen_invlpg_invpcid_handler : xen_invlpg_pcid_handler;
  613                 xen_ipis[IPI_TO_IDX(IPI_INVLRNG)].filter = invpcid_works ?
  614                     xen_invlrng_invpcid_handler : xen_invlrng_pcid_handler;
  615         }
  616 #endif
  617         CPU_FOREACH(i)
  618                 xen_cpu_ipi_init(i);
  619 
  620         /* Set the xen pv ipi ops to replace the native ones */
  621         if (xen_hvm_domain())
  622                 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored;
  623 }
  624 
  625 /* We need to setup IPIs before APs are started */
  626 SYSINIT(xen_setup_cpus, SI_SUB_SMP-1, SI_ORDER_FIRST, xen_setup_cpus, NULL);
  627 #endif /* SMP */

Cache object: befe1f56181e4ec4e4c0115e765c3a8a


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.