xics.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright 2011 Nathan Whitehorn
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
   20  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   21  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
   22  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   23  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   25  * SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD$");
   30 
   31 #include "opt_platform.h"
   32 
   33 #include <sys/param.h>
   34 #include <sys/systm.h>
   35 #include <sys/module.h>
   36 #include <sys/bus.h>
   37 #include <sys/conf.h>
   38 #include <sys/kernel.h>
   39 #include <sys/lock.h>
   40 #include <sys/malloc.h>
   41 #include <sys/mutex.h>
   42 #include <sys/smp.h>
   43 
   44 #include <vm/vm.h>
   45 #include <vm/pmap.h>
   46 
   47 #include <machine/bus.h>
   48 #include <machine/intr_machdep.h>
   49 #include <machine/md_var.h>
   50 #include <machine/rtas.h>
   51 
   52 #include <dev/ofw/ofw_bus.h>
   53 #include <dev/ofw/ofw_bus_subr.h>
   54 
   55 #ifdef POWERNV
   56 #include <powerpc/powernv/opal.h>
   57 #endif
   58 
   59 #include "phyp-hvcall.h"
   60 #include "pic_if.h"
   61 
   62 #define XICP_PRIORITY   5       /* Random non-zero number */
   63 #define XICP_IPI        2
   64 #define MAX_XICP_IRQS   (1<<24) /* 24-bit XIRR field */
   65 
   66 static int      xicp_probe(device_t);
   67 static int      xicp_attach(device_t);
   68 static int      xics_probe(device_t);
   69 static int      xics_attach(device_t);
   70 
   71 static void     xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv);
   72 static void     xicp_dispatch(device_t, struct trapframe *);
   73 static void     xicp_enable(device_t, u_int, u_int, void **priv);
   74 static void     xicp_eoi(device_t, u_int, void *priv);
   75 static void     xicp_ipi(device_t, u_int);
   76 static void     xicp_mask(device_t, u_int, void *priv);
   77 static void     xicp_unmask(device_t, u_int, void *priv);
   78 
   79 #ifdef POWERNV
   80 extern void (*powernv_smp_ap_extra_init)(void);
   81 static void     xicp_smp_cpu_startup(void);
   82 #endif
   83 
   84 static device_method_t  xicp_methods[] = {
   85         /* Device interface */
   86         DEVMETHOD(device_probe,         xicp_probe),
   87         DEVMETHOD(device_attach,        xicp_attach),
   88 
   89         /* PIC interface */
   90         DEVMETHOD(pic_bind,             xicp_bind),
   91         DEVMETHOD(pic_dispatch,         xicp_dispatch),
   92         DEVMETHOD(pic_enable,           xicp_enable),
   93         DEVMETHOD(pic_eoi,              xicp_eoi),
   94         DEVMETHOD(pic_ipi,              xicp_ipi),
   95         DEVMETHOD(pic_mask,             xicp_mask),
   96         DEVMETHOD(pic_unmask,           xicp_unmask),
   97 
   98         DEVMETHOD_END
   99 };
  100 
  101 static device_method_t  xics_methods[] = {
  102         /* Device interface */
  103         DEVMETHOD(device_probe,         xics_probe),
  104         DEVMETHOD(device_attach,        xics_attach),
  105 
  106         DEVMETHOD_END
  107 };
  108 
  109 struct xicp_intvec {
  110         int irq;
  111         int vector;
  112         int cpu;
  113 };
  114 
  115 struct xicp_softc {
  116         struct mtx sc_mtx;
  117         struct resource *mem[MAXCPU];
  118 
  119         int cpu_range[2];
  120 
  121         int ibm_int_on;
  122         int ibm_int_off;
  123         int ibm_get_xive;
  124         int ibm_set_xive;
  125 
  126         /* XXX: inefficient -- hash table? tree? */
  127         struct xicp_intvec intvecs[256];
  128         int nintvecs;
  129         int ipi_vec;
  130         bool xics_emu;
  131 };
  132 
  133 static driver_t xicp_driver = {
  134         "xicp",
  135         xicp_methods,
  136         sizeof(struct xicp_softc)
  137 };
  138 
  139 static driver_t xics_driver = {
  140         "xics",
  141         xics_methods,
  142         0
  143 };
  144 
  145 #ifdef POWERNV
  146 /* We can only pass physical addresses into OPAL.  Kernel stacks are in the KVA,
  147  * not in the direct map, so we need to somehow extract the physical address.
  148  * However, pmap_kextract() takes locks, which is forbidden in a critical region
  149  * (which PIC_DISPATCH() operates in).  The kernel is mapped into the Direct
  150  * Map (0xc000....), and the CPU implicitly drops the top two bits when doing
  151  * real address by nature that the bus width is smaller than 64-bits.  Placing
  152  * cpu_xirr into the DMAP lets us take advantage of this and avoids the
  153  * pmap_kextract() that would otherwise be needed if using the stack variable.
  154  */
  155 static uint32_t cpu_xirr[MAXCPU];
  156 #endif
  157 
  158 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, 0, 0, BUS_PASS_INTERRUPT - 1);
  159 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, 0, 0, BUS_PASS_INTERRUPT);
  160 
  161 #ifdef POWERNV
  162 static struct resource *
  163 xicp_mem_for_cpu(int cpu)
  164 {
  165         devclass_t dc;
  166         device_t dev;
  167         struct xicp_softc *sc;
  168         int i;
  169 
  170         dc = devclass_find(xicp_driver.name);
  171         for (i = 0; (dev = devclass_get_device(dc, i)) != NULL; i++){
  172                 sc = device_get_softc(dev);
  173                 if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1])
  174                         return (sc->mem[cpu - sc->cpu_range[0]]);
  175         }
  176 
  177         return (NULL);
  178 }
  179 #endif
  180 
  181 static int
  182 xicp_probe(device_t dev)
  183 {
  184 
  185         if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") &&
  186             !ofw_bus_is_compatible(dev, "ibm,opal-intc"))
  187                 return (ENXIO);
  188 
  189         device_set_desc(dev, "External Interrupt Presentation Controller");
  190         return (BUS_PROBE_GENERIC);
  191 }
  192 
  193 static int
  194 xics_probe(device_t dev)
  195 {
  196 
  197         if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") &&
  198             !ofw_bus_is_compatible(dev, "IBM,opal-xics"))
  199                 return (ENXIO);
  200 
  201         device_set_desc(dev, "External Interrupt Source Controller");
  202         return (BUS_PROBE_GENERIC);
  203 }
  204 
  205 static int
  206 xicp_attach(device_t dev)
  207 {
  208         struct xicp_softc *sc = device_get_softc(dev);
  209         phandle_t phandle = ofw_bus_get_node(dev);
  210 
  211         if (rtas_exists()) {
  212                 sc->ibm_int_on = rtas_token_lookup("ibm,int-on");
  213                 sc->ibm_int_off = rtas_token_lookup("ibm,int-off");
  214                 sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive");
  215                 sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive");
  216 #ifdef POWERNV
  217         } else if (opal_check() == 0) {
  218                 /* No init needed */
  219 #endif
  220         } else {
  221                 device_printf(dev, "Cannot attach without RTAS or OPAL\n");
  222                 return (ENXIO);
  223         }
  224 
  225         if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) {
  226                 OF_getencprop(phandle, "ibm,interrupt-server-ranges",
  227                     sc->cpu_range, sizeof(sc->cpu_range));
  228                 sc->cpu_range[1] += sc->cpu_range[0];
  229                 device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0],
  230                     sc->cpu_range[1]-1);
  231 #ifdef POWERNV
  232         } else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) {
  233                         /*
  234                          * For now run POWER9 XIVE interrupt controller in XICS
  235                          * compatibility mode.
  236                          */
  237                         sc->xics_emu = true;
  238                         opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EMU);
  239 #endif
  240         } else {
  241                 sc->cpu_range[0] = 0;
  242                 sc->cpu_range[1] = mp_ncpus;
  243         }
  244 
  245 #ifdef POWERNV
  246         if (mfmsr() & PSL_HV) {
  247                 int i;
  248 
  249                 if (sc->xics_emu) {
  250                         opal_call(OPAL_INT_SET_CPPR, 0xff);
  251                         for (i = 0; i < mp_ncpus; i++) {
  252                                 opal_call(OPAL_INT_SET_MFRR,
  253                                     pcpu_find(i)->pc_hwref, 0xff);
  254                         }
  255                 } else {
  256                         for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) {
  257                                 sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  258                                     &i, RF_ACTIVE);
  259                                 if (sc->mem[i] == NULL) {
  260                                         device_printf(dev, "Could not alloc mem "
  261                                             "resource %d\n", i);
  262                                         return (ENXIO);
  263                                 }
  264 
  265                                 /* Unmask interrupts on all cores */
  266                                 bus_write_1(sc->mem[i], 4, 0xff);
  267                                 bus_write_1(sc->mem[i], 12, 0xff);
  268                         }
  269                 }
  270         }
  271 #endif
  272 
  273         mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF);
  274         sc->nintvecs = 0;
  275 
  276         powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS,
  277             1 /* Number of IPIs */, FALSE);
  278         root_pic = dev;
  279 
  280 #ifdef POWERNV
  281         if (sc->xics_emu)
  282                 powernv_smp_ap_extra_init = xicp_smp_cpu_startup;
  283 #endif
  284 
  285         return (0);
  286 }
  287 
  288 static int
  289 xics_attach(device_t dev)
  290 {
  291         phandle_t phandle = ofw_bus_get_node(dev);
  292 
  293         /* The XICP (root PIC) will handle all our interrupts */
  294         powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
  295             MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE);
  296 
  297         return (0);
  298 }
  299 
  300 static __inline struct xicp_intvec *
  301 xicp_setup_priv(struct xicp_softc *sc, u_int irq, void **priv)
  302 {
  303         if (*priv == NULL) {
  304                 KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs),
  305                         ("Too many XICP interrupts"));
  306                 mtx_lock(&sc->sc_mtx);
  307                 *priv = &sc->intvecs[sc->nintvecs++];
  308                 mtx_unlock(&sc->sc_mtx);
  309         }
  310 
  311         return (*priv);
  312 }
  313 
  314 /*
  315  * PIC I/F methods.
  316  */
  317 
  318 static void
  319 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
  320 {
  321         struct xicp_softc *sc = device_get_softc(dev);
  322         struct xicp_intvec *iv;
  323         cell_t status, cpu;
  324         int ncpus, i, error = -1;
  325 
  326         /* Ignore IPIs */
  327         if (irq == MAX_XICP_IRQS)
  328                 return;
  329 
  330         iv = xicp_setup_priv(sc, irq, priv);
  331 
  332         /*
  333          * This doesn't appear to actually support affinity groups, so pick a
  334          * random CPU.
  335          */
  336         ncpus = 0;
  337         CPU_FOREACH(cpu)
  338                 if (CPU_ISSET(cpu, &cpumask)) ncpus++;
  339 
  340         i = mftb() % ncpus;
  341         ncpus = 0;
  342         CPU_FOREACH(cpu) {
  343                 if (!CPU_ISSET(cpu, &cpumask))
  344                         continue;
  345                 if (ncpus == i)
  346                         break;
  347                 ncpus++;
  348         }
  349 
  350         cpu = pcpu_find(cpu)->pc_hwref;
  351         iv->cpu = cpu;
  352 
  353         if (rtas_exists())
  354                 error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
  355                     XICP_PRIORITY, &status);
  356 #ifdef POWERNV
  357         else
  358                 error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
  359 #endif
  360 
  361         if (error < 0)
  362                 panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
  363 }
  364 
  365 static void
  366 xicp_dispatch(device_t dev, struct trapframe *tf)
  367 {
  368         struct xicp_softc *sc;
  369         struct resource *regs = NULL;
  370         uint64_t xirr, junk;
  371         int i;
  372 
  373         sc = device_get_softc(dev);
  374 #ifdef POWERNV
  375         if ((mfmsr() & PSL_HV) && !sc->xics_emu) {
  376                 regs = xicp_mem_for_cpu(PCPU_GET(hwref));
  377                 KASSERT(regs != NULL,
  378                     ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref)));
  379         }
  380 #endif
  381 
  382         for (;;) {
  383                 /* Return value in R4, use the PFT call */
  384                 if (regs) {
  385                         xirr = bus_read_4(regs, 4);
  386 #ifdef POWERNV
  387                 } else if (sc->xics_emu) {
  388                         opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)],
  389                             false);
  390                         xirr = cpu_xirr[PCPU_GET(cpuid)];
  391 #endif
  392                 } else {
  393                         /* Return value in R4, use the PFT call */
  394                         phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk);
  395                 }
  396                 xirr &= 0x00ffffff;
  397 
  398                 if (xirr == 0) /* No more pending interrupts? */
  399                         break;
  400 
  401                 if (xirr == XICP_IPI) {         /* Magic number for IPIs */
  402                         xirr = MAX_XICP_IRQS;   /* Map to FreeBSD magic */
  403 
  404                         /* Clear IPI */
  405                         if (regs)
  406                                 bus_write_1(regs, 12, 0xff);
  407 #ifdef POWERNV
  408                         else if (sc->xics_emu)
  409                                 opal_call(OPAL_INT_SET_MFRR,
  410                                     PCPU_GET(hwref), 0xff);
  411 #endif
  412                         else
  413                                 phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)),
  414                                     0xff);
  415                         i = sc->ipi_vec;
  416                 } else {
  417                         /* XXX: super inefficient */
  418                         for (i = 0; i < sc->nintvecs; i++) {
  419                                 if (sc->intvecs[i].irq == xirr)
  420                                         break;
  421                         }
  422                         KASSERT(i < sc->nintvecs, ("Unmapped XIRR"));
  423                 }
  424 
  425                 powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
  426         }
  427 }
  428 
  429 static void
  430 xicp_enable(device_t dev, u_int irq, u_int vector, void **priv)
  431 {
  432         struct xicp_softc *sc;
  433         struct xicp_intvec *intr;
  434         cell_t status, cpu;
  435 
  436         sc = device_get_softc(dev);
  437 
  438         /* Bind to this CPU to start: distrib. ID is last entry in gserver# */
  439         cpu = PCPU_GET(hwref);
  440 
  441         intr = xicp_setup_priv(sc, irq, priv);
  442 
  443         intr->irq = irq;
  444         intr->vector = vector;
  445         intr->cpu = cpu;
  446         mb();
  447 
  448         /* IPIs are also enabled.  Stash off the vector index */
  449         if (irq == MAX_XICP_IRQS) {
  450                 sc->ipi_vec = intr - sc->intvecs;
  451                 return;
  452         }
  453 
  454         if (rtas_exists()) {
  455                 rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
  456                     XICP_PRIORITY, &status);
  457                 xicp_unmask(dev, irq, intr);
  458 #ifdef POWERNV
  459         } else {
  460                 status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
  461                 /* Unmask implicit for OPAL */
  462 
  463                 if (status != 0)
  464                         panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
  465                             cpu, status);
  466 #endif
  467         }
  468 }
  469 
  470 static void
  471 xicp_eoi(device_t dev, u_int irq, void *priv)
  472 {
  473 #ifdef POWERNV
  474         struct xicp_softc *sc;
  475 #endif
  476         uint64_t xirr;
  477 
  478         if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */
  479                 irq = XICP_IPI;
  480         xirr = irq | (0xff << 24);
  481 
  482 #ifdef POWERNV
  483         if (mfmsr() & PSL_HV) {
  484                 sc = device_get_softc(dev);
  485                 if (sc->xics_emu)
  486                         opal_call(OPAL_INT_EOI, xirr);
  487                 else
  488                         bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr);
  489         } else
  490 #endif
  491                 phyp_hcall(H_EOI, xirr);
  492 }
  493 
  494 static void
  495 xicp_ipi(device_t dev, u_int cpu)
  496 {
  497 
  498 #ifdef POWERNV
  499         struct xicp_softc *sc;
  500         cpu = pcpu_find(cpu)->pc_hwref;
  501 
  502         if (mfmsr() & PSL_HV) {
  503                 sc = device_get_softc(dev);
  504                 if (sc->xics_emu) {
  505                         int64_t rv;
  506                         rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY);
  507                         if (rv != 0)
  508                             device_printf(dev, "IPI SET_MFRR result: %ld\n", rv);
  509                 } else
  510                         bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY);
  511         } else
  512 #endif
  513                 phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY);
  514 }
  515 
  516 static void
  517 xicp_mask(device_t dev, u_int irq, void *priv)
  518 {
  519         struct xicp_softc *sc = device_get_softc(dev);
  520         cell_t status;
  521 
  522         if (irq == MAX_XICP_IRQS)
  523                 return;
  524 
  525         if (rtas_exists()) {
  526                 rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status);
  527 #ifdef POWERNV
  528         } else {
  529                 struct xicp_intvec *ivec = priv;
  530 
  531                 KASSERT(ivec != NULL, ("Masking unconfigured interrupt"));
  532                 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, 0xff);
  533 #endif
  534         }
  535 }
  536 
  537 static void
  538 xicp_unmask(device_t dev, u_int irq, void *priv)
  539 {
  540         struct xicp_softc *sc = device_get_softc(dev);
  541         cell_t status;
  542 
  543         if (irq == MAX_XICP_IRQS)
  544                 return;
  545 
  546         if (rtas_exists()) {
  547                 rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status);
  548 #ifdef POWERNV
  549         } else {
  550                 struct xicp_intvec *ivec = priv;
  551 
  552                 KASSERT(ivec != NULL, ("Unmasking unconfigured interrupt"));
  553                 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, XICP_PRIORITY);
  554 #endif
  555         }
  556 }
  557 
  558 #ifdef POWERNV
  559 /* This is only used on POWER9 systems with the XIVE's XICS emulation. */
  560 static void
  561 xicp_smp_cpu_startup(void)
  562 {
  563         struct xicp_softc *sc;
  564 
  565         if (mfmsr() & PSL_HV) {
  566                 sc = device_get_softc(root_pic);
  567 
  568                 if (sc->xics_emu)
  569                         opal_call(OPAL_INT_SET_CPPR, 0xff);
  570         }
  571 }
  572 #endif
Cache object: 03b1c48aa3a941af36dddb98b24bac31
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/powerpc/pseries/xics.c

FreeBSD/Linux Kernel Cross Reference
sys/powerpc/pseries/xics.c
