The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/sparc64/pci/fire.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-3-Clause
    3  *
    4  * Copyright (c) 1999, 2000 Matthew R. Green
    5  * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org>
    6  * Copyright (c) 2009 by Marius Strobl <marius@FreeBSD.org>
    7  * All rights reserved.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  * 3. The name of the author may not be used to endorse or promote products
   18  *    derived from this software without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
   25  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   26  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
   27  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   28  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp
   33  *      from: FreeBSD: psycho.c 183152 2008-09-18 19:45:22Z marius
   34  */
   35 
   36 #include <sys/cdefs.h>
   37 __FBSDID("$FreeBSD$");
   38 
   39 /*
   40  * Driver for `Fire' JBus to PCI Express and `Oberon' Uranus to PCI Express
   41  * bridges
   42  */
   43 
   44 #include "opt_fire.h"
   45 #include "opt_ofw_pci.h"
   46 
   47 #include <sys/param.h>
   48 #include <sys/systm.h>
   49 #include <sys/bus.h>
   50 #include <sys/interrupt.h>
   51 #include <sys/kernel.h>
   52 #include <sys/lock.h>
   53 #include <sys/malloc.h>
   54 #include <sys/module.h>
   55 #include <sys/mutex.h>
   56 #include <sys/pciio.h>
   57 #include <sys/pcpu.h>
   58 #include <sys/rman.h>
   59 #include <sys/smp.h>
   60 #include <sys/sysctl.h>
   61 #include <sys/timetc.h>
   62 
   63 #include <dev/ofw/ofw_bus.h>
   64 #include <dev/ofw/openfirm.h>
   65 
   66 #include <vm/vm.h>
   67 #include <vm/pmap.h>
   68 
   69 #include <machine/bus.h>
   70 #include <machine/bus_common.h>
   71 #include <machine/bus_private.h>
   72 #include <machine/iommureg.h>
   73 #include <machine/iommuvar.h>
   74 #include <machine/resource.h>
   75 
   76 #include <dev/pci/pcireg.h>
   77 #include <dev/pci/pcivar.h>
   78 #include <dev/pci/pcib_private.h>
   79 
   80 #include <sparc64/pci/ofw_pci.h>
   81 #include <sparc64/pci/firereg.h>
   82 #include <sparc64/pci/firevar.h>
   83 
   84 #include "pcib_if.h"
   85 
   86 struct fire_msiqarg;
   87 
   88 static const struct fire_desc *fire_get_desc(device_t dev);
   89 static void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
   90     bus_dmasync_op_t op);
   91 static int fire_get_intrmap(struct fire_softc *sc, u_int ino,
   92     bus_addr_t *intrmapptr, bus_addr_t *intrclrptr);
   93 static void fire_intr_assign(void *arg);
   94 static void fire_intr_clear(void *arg);
   95 static void fire_intr_disable(void *arg);
   96 static void fire_intr_enable(void *arg);
   97 static int fire_intr_register(struct fire_softc *sc, u_int ino);
   98 static inline void fire_msiq_common(struct intr_vector *iv,
   99     struct fire_msiqarg *fmqa);
  100 static void fire_msiq_filter(void *cookie);
  101 static void fire_msiq_handler(void *cookie);
  102 static void fire_set_intr(struct fire_softc *sc, u_int index, u_int ino,
  103     driver_filter_t handler, void *arg);
  104 static timecounter_get_t fire_get_timecount;
  105 
  106 /* Interrupt handlers */
  107 static driver_filter_t fire_dmc_pec;
  108 static driver_filter_t fire_pcie;
  109 static driver_filter_t fire_xcb;
  110 
  111 /*
  112  * Methods
  113  */
  114 static pcib_alloc_msi_t fire_alloc_msi;
  115 static pcib_alloc_msix_t fire_alloc_msix;
  116 static bus_alloc_resource_t fire_alloc_resource;
  117 static device_attach_t fire_attach;
  118 static pcib_map_msi_t fire_map_msi;
  119 static pcib_maxslots_t fire_maxslots;
  120 static device_probe_t fire_probe;
  121 static pcib_read_config_t fire_read_config;
  122 static pcib_release_msi_t fire_release_msi;
  123 static pcib_release_msix_t fire_release_msix;
  124 static pcib_route_interrupt_t fire_route_interrupt;
  125 static bus_setup_intr_t fire_setup_intr;
  126 static bus_teardown_intr_t fire_teardown_intr;
  127 static pcib_write_config_t fire_write_config;
  128 
  129 static device_method_t fire_methods[] = {
  130         /* Device interface */
  131         DEVMETHOD(device_probe,         fire_probe),
  132         DEVMETHOD(device_attach,        fire_attach),
  133         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
  134         DEVMETHOD(device_suspend,       bus_generic_suspend),
  135         DEVMETHOD(device_resume,        bus_generic_resume),
  136 
  137         /* Bus interface */
  138         DEVMETHOD(bus_read_ivar,        ofw_pci_read_ivar),
  139         DEVMETHOD(bus_setup_intr,       fire_setup_intr),
  140         DEVMETHOD(bus_teardown_intr,    fire_teardown_intr),
  141         DEVMETHOD(bus_alloc_resource,   fire_alloc_resource),
  142         DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource),
  143         DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
  144         DEVMETHOD(bus_adjust_resource,  ofw_pci_adjust_resource),
  145         DEVMETHOD(bus_release_resource, bus_generic_release_resource),
  146         DEVMETHOD(bus_get_dma_tag,      ofw_pci_get_dma_tag),
  147 
  148         /* pcib interface */
  149         DEVMETHOD(pcib_maxslots,        fire_maxslots),
  150         DEVMETHOD(pcib_read_config,     fire_read_config),
  151         DEVMETHOD(pcib_write_config,    fire_write_config),
  152         DEVMETHOD(pcib_route_interrupt, fire_route_interrupt),
  153         DEVMETHOD(pcib_alloc_msi,       fire_alloc_msi),
  154         DEVMETHOD(pcib_release_msi,     fire_release_msi),
  155         DEVMETHOD(pcib_alloc_msix,      fire_alloc_msix),
  156         DEVMETHOD(pcib_release_msix,    fire_release_msix),
  157         DEVMETHOD(pcib_map_msi,         fire_map_msi),
  158         DEVMETHOD(pcib_request_feature, pcib_request_feature_allow),
  159 
  160         /* ofw_bus interface */
  161         DEVMETHOD(ofw_bus_get_node,     ofw_pci_get_node),
  162 
  163         DEVMETHOD_END
  164 };
  165 
  166 static devclass_t fire_devclass;
  167 
  168 DEFINE_CLASS_0(pcib, fire_driver, fire_methods, sizeof(struct fire_softc));
  169 EARLY_DRIVER_MODULE(fire, nexus, fire_driver, fire_devclass, 0, 0,
  170     BUS_PASS_BUS);
  171 MODULE_DEPEND(fire, nexus, 1, 1, 1);
  172 
  173 static const struct intr_controller fire_ic = {
  174         fire_intr_enable,
  175         fire_intr_disable,
  176         fire_intr_assign,
  177         fire_intr_clear
  178 };
  179 
  180 struct fire_icarg {
  181         struct fire_softc       *fica_sc;
  182         bus_addr_t              fica_map;
  183         bus_addr_t              fica_clr;
  184 };
  185 
  186 static const struct intr_controller fire_msiqc_filter = {
  187         fire_intr_enable,
  188         fire_intr_disable,
  189         fire_intr_assign,
  190         NULL
  191 };
  192 
  193 struct fire_msiqarg {
  194         struct fire_icarg       fmqa_fica;
  195         struct mtx              fmqa_mtx;
  196         struct fo_msiq_record   *fmqa_base;
  197         uint64_t                fmqa_head;
  198         uint64_t                fmqa_tail;
  199         uint32_t                fmqa_msiq;
  200         uint32_t                fmqa_msi;
  201 };
  202 
  203 #define FIRE_PERF_CNT_QLTY      100
  204 
  205 #define FIRE_SPC_BARRIER(spc, sc, offs, len, flags)                     \
  206         bus_barrier((sc)->sc_mem_res[(spc)], (offs), (len), (flags))
  207 #define FIRE_SPC_READ_8(spc, sc, offs)                                  \
  208         bus_read_8((sc)->sc_mem_res[(spc)], (offs))
  209 #define FIRE_SPC_WRITE_8(spc, sc, offs, v)                              \
  210         bus_write_8((sc)->sc_mem_res[(spc)], (offs), (v))
  211 
  212 #ifndef FIRE_DEBUG
  213 #define FIRE_SPC_SET(spc, sc, offs, reg, v)                             \
  214         FIRE_SPC_WRITE_8((spc), (sc), (offs), (v))
  215 #else
  216 #define FIRE_SPC_SET(spc, sc, offs, reg, v) do {                        \
  217         device_printf((sc)->sc_dev, reg " 0x%016llx -> 0x%016llx\n",    \
  218             (unsigned long long)FIRE_SPC_READ_8((spc), (sc), (offs)),   \
  219             (unsigned long long)(v));                                   \
  220         FIRE_SPC_WRITE_8((spc), (sc), (offs), (v));                     \
  221         } while (0)
  222 #endif
  223 
  224 #define FIRE_PCI_BARRIER(sc, offs, len, flags)                          \
  225         FIRE_SPC_BARRIER(FIRE_PCI, (sc), (offs), len, flags)
  226 #define FIRE_PCI_READ_8(sc, offs)                                       \
  227         FIRE_SPC_READ_8(FIRE_PCI, (sc), (offs))
  228 #define FIRE_PCI_WRITE_8(sc, offs, v)                                   \
  229         FIRE_SPC_WRITE_8(FIRE_PCI, (sc), (offs), (v))
  230 #define FIRE_CTRL_BARRIER(sc, offs, len, flags)                         \
  231         FIRE_SPC_BARRIER(FIRE_CTRL, (sc), (offs), len, flags)
  232 #define FIRE_CTRL_READ_8(sc, offs)                                      \
  233         FIRE_SPC_READ_8(FIRE_CTRL, (sc), (offs))
  234 #define FIRE_CTRL_WRITE_8(sc, offs, v)                                  \
  235         FIRE_SPC_WRITE_8(FIRE_CTRL, (sc), (offs), (v))
  236 
  237 #define FIRE_PCI_SET(sc, offs, v)                                       \
  238         FIRE_SPC_SET(FIRE_PCI, (sc), (offs), # offs, (v))
  239 #define FIRE_CTRL_SET(sc, offs, v)                                      \
  240         FIRE_SPC_SET(FIRE_CTRL, (sc), (offs), # offs, (v))
  241 
  242 struct fire_desc {
  243         const char      *fd_string;
  244         int             fd_mode;
  245         const char      *fd_name;
  246 };
  247 
  248 static const struct fire_desc fire_compats[] = {
  249         { "pciex108e,80f0",     FIRE_MODE_FIRE,         "Fire" },
  250 #if 0
  251         { "pciex108e,80f8",     FIRE_MODE_OBERON,       "Oberon" },
  252 #endif
  253         { NULL,                 0,                      NULL }
  254 };
  255 
  256 static const struct fire_desc *
  257 fire_get_desc(device_t dev)
  258 {
  259         const struct fire_desc *desc;
  260         const char *compat;
  261 
  262         compat = ofw_bus_get_compat(dev);
  263         if (compat == NULL)
  264                 return (NULL);
  265         for (desc = fire_compats; desc->fd_string != NULL; desc++)
  266                 if (strcmp(desc->fd_string, compat) == 0)
  267                         return (desc);
  268         return (NULL);
  269 }
  270 
  271 static int
  272 fire_probe(device_t dev)
  273 {
  274         const char *dtype;
  275 
  276         dtype = ofw_bus_get_type(dev);
  277         if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCIE) == 0 &&
  278             fire_get_desc(dev) != NULL) {
  279                 device_set_desc(dev, "Sun Host-PCIe bridge");
  280                 return (BUS_PROBE_GENERIC);
  281         }
  282         return (ENXIO);
  283 }
  284 
  285 static int
  286 fire_attach(device_t dev)
  287 {
  288         struct fire_softc *sc;
  289         const struct fire_desc *desc;
  290         struct ofw_pci_msi_ranges msi_ranges;
  291         struct ofw_pci_msi_addr_ranges msi_addr_ranges;
  292         struct ofw_pci_msi_eq_to_devino msi_eq_to_devino;
  293         struct fire_msiqarg *fmqa;
  294         struct timecounter *tc;
  295         bus_dma_tag_t dmat;
  296         uint64_t ino_bitmap, val;
  297         phandle_t node;
  298         uint32_t prop, prop_array[2];
  299         int i, j, mode;
  300         u_int lw;
  301         uint16_t mps;
  302 
  303         sc = device_get_softc(dev);
  304         node = ofw_bus_get_node(dev);
  305         desc = fire_get_desc(dev);
  306         mode = desc->fd_mode;
  307 
  308         sc->sc_dev = dev;
  309         sc->sc_mode = mode;
  310         sc->sc_flags = 0;
  311 
  312         mtx_init(&sc->sc_msi_mtx, "msi_mtx", NULL, MTX_DEF);
  313         mtx_init(&sc->sc_pcib_mtx, "pcib_mtx", NULL, MTX_SPIN);
  314 
  315         /*
  316          * Fire and Oberon have two register banks:
  317          * (0) per-PBM PCI Express configuration and status registers
  318          * (1) (shared) Fire/Oberon controller configuration and status
  319          *     registers
  320          */
  321         for (i = 0; i < FIRE_NREG; i++) {
  322                 j = i;
  323                 sc->sc_mem_res[i] = bus_alloc_resource_any(dev,
  324                     SYS_RES_MEMORY, &j, RF_ACTIVE);
  325                 if (sc->sc_mem_res[i] == NULL)
  326                         panic("%s: could not allocate register bank %d",
  327                             __func__, i);
  328         }
  329 
  330         if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1)
  331                 panic("%s: could not determine IGN", __func__);
  332         if (OF_getprop(node, "module-revision#", &prop, sizeof(prop)) == -1)
  333                 panic("%s: could not determine module-revision", __func__);
  334 
  335         device_printf(dev, "%s, module-revision %d, IGN %#x\n",
  336             desc->fd_name, prop, sc->sc_ign);
  337 
  338         /*
  339          * Hunt through all the interrupt mapping regs and register
  340          * the interrupt controller for our interrupt vectors.  We do
  341          * this early in order to be able to catch stray interrupts.
  342          */
  343         i = OF_getprop(node, "ino-bitmap", (void *)prop_array,
  344             sizeof(prop_array));
  345         if (i == -1)
  346                 panic("%s: could not get ino-bitmap", __func__);
  347         ino_bitmap = ((uint64_t)prop_array[1] << 32) | prop_array[0];
  348         for (i = 0; i <= FO_MAX_INO; i++) {
  349                 if ((ino_bitmap & (1ULL << i)) == 0)
  350                         continue;
  351                 j = fire_intr_register(sc, i);
  352                 if (j != 0)
  353                         device_printf(dev, "could not register interrupt "
  354                             "controller for INO %d (%d)\n", i, j);
  355         }
  356 
  357         /* JBC/UBC module initialization */
  358         FIRE_CTRL_SET(sc, FO_XBC_ERR_LOG_EN, ~0ULL);
  359         FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
  360         /* not enabled by OpenSolaris */
  361         FIRE_CTRL_SET(sc, FO_XBC_INT_EN, ~0ULL);
  362         if (sc->sc_mode == FIRE_MODE_FIRE) {
  363                 FIRE_CTRL_SET(sc, FIRE_JBUS_PAR_CTRL,
  364                     FIRE_JBUS_PAR_CTRL_P_EN);
  365                 FIRE_CTRL_SET(sc, FIRE_JBC_FATAL_RST_EN,
  366                     ((1ULL << FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_SHFT) &
  367                     FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_MASK) |
  368                     FIRE_JBC_FATAL_RST_EN_MB_PEA_P_INT |
  369                     FIRE_JBC_FATAL_RST_EN_CPE_P_INT |
  370                     FIRE_JBC_FATAL_RST_EN_APE_P_INT |
  371                     FIRE_JBC_FATAL_RST_EN_PIO_CPE_INT |
  372                     FIRE_JBC_FATAL_RST_EN_JTCEEW_P_INT |
  373                     FIRE_JBC_FATAL_RST_EN_JTCEEI_P_INT |
  374                     FIRE_JBC_FATAL_RST_EN_JTCEER_P_INT);
  375                 FIRE_CTRL_SET(sc, FIRE_JBC_CORE_BLOCK_INT_EN, ~0ULL);
  376         }
  377 
  378         /* TLU initialization */
  379         FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_STAT_CLR,
  380             FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK);
  381         /* not enabled by OpenSolaris */
  382         FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_INT_EN,
  383             FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK);
  384         FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_STAT_CLR,
  385             FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK);
  386         /* not enabled by OpenSolaris */
  387         FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_INT_EN,
  388             FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK);
  389         FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_STAT_CLR,
  390             FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK);
  391         /* not enabled by OpenSolaris */
  392         FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_INT_EN,
  393             FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK);
  394         val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) |
  395             ((FO_PCI_TLU_CTRL_L0S_TIM_DFLT << FO_PCI_TLU_CTRL_L0S_TIM_SHFT) &
  396             FO_PCI_TLU_CTRL_L0S_TIM_MASK) |
  397             ((FO_PCI_TLU_CTRL_CFG_DFLT << FO_PCI_TLU_CTRL_CFG_SHFT) &
  398             FO_PCI_TLU_CTRL_CFG_MASK);
  399         if (sc->sc_mode == FIRE_MODE_OBERON)
  400                 val &= ~FO_PCI_TLU_CTRL_NWPR_EN;
  401         val |= FO_PCI_TLU_CTRL_CFG_REMAIN_DETECT_QUIET;
  402         FIRE_PCI_SET(sc, FO_PCI_TLU_CTRL, val);
  403         FIRE_PCI_SET(sc, FO_PCI_TLU_DEV_CTRL, 0);
  404         FIRE_PCI_SET(sc, FO_PCI_TLU_LNK_CTRL, FO_PCI_TLU_LNK_CTRL_CLK);
  405 
  406         /* DLU/LPU initialization */
  407         if (sc->sc_mode == FIRE_MODE_OBERON)
  408                 FIRE_PCI_SET(sc, FO_PCI_LPU_INT_MASK, 0);
  409         else
  410                 FIRE_PCI_SET(sc, FO_PCI_LPU_RST, 0);
  411         FIRE_PCI_SET(sc, FO_PCI_LPU_LNK_LYR_CFG,
  412             FO_PCI_LPU_LNK_LYR_CFG_VC0_EN);
  413         FIRE_PCI_SET(sc, FO_PCI_LPU_FLW_CTRL_UPDT_CTRL,
  414             FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_NP_EN |
  415             FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_P_EN);
  416         if (sc->sc_mode == FIRE_MODE_OBERON)
  417                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS,
  418                     (OBERON_PCI_LPU_TXLNK_RPLY_TMR_THRS_DFLT <<
  419                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) &
  420                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK);
  421         else {
  422                 switch ((FIRE_PCI_READ_8(sc, FO_PCI_TLU_LNK_STAT) &
  423                     FO_PCI_TLU_LNK_STAT_WDTH_MASK) >>
  424                     FO_PCI_TLU_LNK_STAT_WDTH_SHFT) {
  425                 case 1:
  426                         lw = 0;
  427                         break;
  428                 case 4:
  429                         lw = 1;
  430                         break;
  431                 case 8:
  432                         lw = 2;
  433                         break;
  434                 case 16:
  435                         lw = 3;
  436                         break;
  437                 default:
  438                         lw = 0;
  439                 }
  440                 mps = (FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) &
  441                     FO_PCI_TLU_CTRL_CFG_MPS_MASK) >>
  442                     FO_PCI_TLU_CTRL_CFG_MPS_SHFT;
  443                 i = sizeof(fire_freq_nak_tmr_thrs) /
  444                     sizeof(*fire_freq_nak_tmr_thrs);
  445                 if (mps >= i)
  446                         mps = i - 1;
  447                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS,
  448                     (fire_freq_nak_tmr_thrs[mps][lw] <<
  449                     FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_SHFT) &
  450                     FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_MASK);
  451                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS,
  452                     (fire_rply_tmr_thrs[mps][lw] <<
  453                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) &
  454                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK);
  455                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RTR_FIFO_PTR,
  456                     ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_DFLT <<
  457                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_SHFT) &
  458                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_MASK) |
  459                     ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_DFLT <<
  460                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_SHFT) &
  461                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_MASK));
  462                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG2,
  463                     (FO_PCI_LPU_LTSSM_CFG2_12_TO_DFLT <<
  464                     FO_PCI_LPU_LTSSM_CFG2_12_TO_SHFT) &
  465                     FO_PCI_LPU_LTSSM_CFG2_12_TO_MASK);
  466                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG3,
  467                     (FO_PCI_LPU_LTSSM_CFG3_2_TO_DFLT <<
  468                     FO_PCI_LPU_LTSSM_CFG3_2_TO_SHFT) &
  469                     FO_PCI_LPU_LTSSM_CFG3_2_TO_MASK);
  470                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG4,
  471                     ((FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_DFLT <<
  472                     FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_SHFT) &
  473                     FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_MASK) |
  474                     ((FO_PCI_LPU_LTSSM_CFG4_N_FTS_DFLT <<
  475                     FO_PCI_LPU_LTSSM_CFG4_N_FTS_SHFT) &
  476                     FO_PCI_LPU_LTSSM_CFG4_N_FTS_MASK));
  477                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG5, 0);
  478         }
  479 
  480         /* ILU initialization */
  481         FIRE_PCI_SET(sc, FO_PCI_ILU_ERR_STAT_CLR, ~0ULL);
  482         /* not enabled by OpenSolaris */
  483         FIRE_PCI_SET(sc, FO_PCI_ILU_INT_EN, ~0ULL);
  484 
  485         /* IMU initialization */
  486         FIRE_PCI_SET(sc, FO_PCI_IMU_ERR_STAT_CLR, ~0ULL);
  487         FIRE_PCI_SET(sc, FO_PCI_IMU_INT_EN,
  488             FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_EN) &
  489             ~(FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_S |
  490             FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_S |
  491             FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_S |
  492             FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P |
  493             FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P |
  494             FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P));
  495 
  496         /* MMU initialization */
  497         FIRE_PCI_SET(sc, FO_PCI_MMU_ERR_STAT_CLR,
  498             FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK);
  499         /* not enabled by OpenSolaris */
  500         FIRE_PCI_SET(sc, FO_PCI_MMU_INT_EN,
  501             FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK);
  502 
  503         /* DMC initialization */
  504         FIRE_PCI_SET(sc, FO_PCI_DMC_CORE_BLOCK_INT_EN, ~0ULL);
  505         FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTA, 0);
  506         FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTB, 0);
  507 
  508         /* PEC initialization */
  509         FIRE_PCI_SET(sc, FO_PCI_PEC_CORE_BLOCK_INT_EN, ~0ULL);
  510 
  511         /* Establish handlers for interesting interrupts. */
  512         if ((ino_bitmap & (1ULL << FO_DMC_PEC_INO)) != 0)
  513                 fire_set_intr(sc, 1, FO_DMC_PEC_INO, fire_dmc_pec, sc);
  514         if ((ino_bitmap & (1ULL << FO_XCB_INO)) != 0)
  515                 fire_set_intr(sc, 0, FO_XCB_INO, fire_xcb, sc);
  516 
  517         /* MSI/MSI-X support */
  518         if (OF_getprop(node, "#msi", &sc->sc_msi_count,
  519             sizeof(sc->sc_msi_count)) == -1)
  520                 panic("%s: could not determine MSI count", __func__);
  521         if (OF_getprop(node, "msi-ranges", &msi_ranges,
  522             sizeof(msi_ranges)) == -1)
  523                 sc->sc_msi_first = 0;
  524         else
  525                 sc->sc_msi_first = msi_ranges.first;
  526         if (OF_getprop(node, "msi-data-mask", &sc->sc_msi_data_mask,
  527             sizeof(sc->sc_msi_data_mask)) == -1)
  528                 panic("%s: could not determine MSI data mask", __func__);
  529         if (OF_getprop(node, "msix-data-width", &sc->sc_msix_data_width,
  530             sizeof(sc->sc_msix_data_width)) > 0)
  531                 sc->sc_flags |= FIRE_MSIX;
  532         if (OF_getprop(node, "msi-address-ranges", &msi_addr_ranges,
  533             sizeof(msi_addr_ranges)) == -1)
  534                 panic("%s: could not determine MSI address ranges", __func__);
  535         sc->sc_msi_addr32 = OFW_PCI_MSI_ADDR_RANGE_32(&msi_addr_ranges);
  536         sc->sc_msi_addr64 = OFW_PCI_MSI_ADDR_RANGE_64(&msi_addr_ranges);
  537         if (OF_getprop(node, "#msi-eqs", &sc->sc_msiq_count,
  538             sizeof(sc->sc_msiq_count)) == -1)
  539                 panic("%s: could not determine MSI event queue count",
  540                     __func__);
  541         if (OF_getprop(node, "msi-eq-size", &sc->sc_msiq_size,
  542             sizeof(sc->sc_msiq_size)) == -1)
  543                 panic("%s: could not determine MSI event queue size",
  544                     __func__);
  545         if (OF_getprop(node, "msi-eq-to-devino", &msi_eq_to_devino,
  546             sizeof(msi_eq_to_devino)) == -1 &&
  547             OF_getprop(node, "msi-eq-devino", &msi_eq_to_devino,
  548             sizeof(msi_eq_to_devino)) == -1) {
  549                 sc->sc_msiq_first = 0;
  550                 sc->sc_msiq_ino_first = FO_EQ_FIRST_INO;
  551         } else {
  552                 sc->sc_msiq_first = msi_eq_to_devino.eq_first;
  553                 sc->sc_msiq_ino_first = msi_eq_to_devino.devino_first;
  554         }
  555         if (sc->sc_msiq_ino_first < FO_EQ_FIRST_INO ||
  556             sc->sc_msiq_ino_first + sc->sc_msiq_count - 1 > FO_EQ_LAST_INO)
  557                 panic("%s: event queues exceed INO range", __func__);
  558         sc->sc_msi_bitmap = malloc(roundup2(sc->sc_msi_count, NBBY) / NBBY,
  559             M_DEVBUF, M_NOWAIT | M_ZERO);
  560         if (sc->sc_msi_bitmap == NULL)
  561                 panic("%s: could not malloc MSI bitmap", __func__);
  562         sc->sc_msi_msiq_table = malloc(sc->sc_msi_count *
  563             sizeof(*sc->sc_msi_msiq_table), M_DEVBUF, M_NOWAIT | M_ZERO);
  564         if (sc->sc_msi_msiq_table == NULL)
  565                 panic("%s: could not malloc MSI-MSI event queue table",
  566                     __func__);
  567         sc->sc_msiq_bitmap = malloc(roundup2(sc->sc_msiq_count, NBBY) / NBBY,
  568             M_DEVBUF, M_NOWAIT | M_ZERO);
  569         if (sc->sc_msiq_bitmap == NULL)
  570                 panic("%s: could not malloc MSI event queue bitmap", __func__);
  571         j = FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * sc->sc_msiq_count;
  572         sc->sc_msiq = contigmalloc(j, M_DEVBUF, M_NOWAIT, 0, ~0UL,
  573             FO_EQ_ALIGNMENT, 0);
  574         if (sc->sc_msiq == NULL)
  575                 panic("%s: could not contigmalloc MSI event queue", __func__);
  576         memset(sc->sc_msiq, 0, j);
  577         FIRE_PCI_SET(sc, FO_PCI_EQ_BASE_ADDR, FO_PCI_EQ_BASE_ADDR_BYPASS |
  578             (pmap_kextract((vm_offset_t)sc->sc_msiq) &
  579             FO_PCI_EQ_BASE_ADDR_MASK));
  580         for (i = 0; i < sc->sc_msi_count; i++) {
  581                 j = (i + sc->sc_msi_first) << 3;
  582                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + j,
  583                     FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + j) &
  584                     ~FO_PCI_MSI_MAP_V);
  585         }
  586         for (i = 0; i < sc->sc_msiq_count; i++) {
  587                 j = i + sc->sc_msiq_ino_first;
  588                 if ((ino_bitmap & (1ULL << j)) == 0) {
  589                         mtx_lock(&sc->sc_msi_mtx);
  590                         setbit(sc->sc_msiq_bitmap, i);
  591                         mtx_unlock(&sc->sc_msi_mtx);
  592                 }
  593                 fmqa = intr_vectors[INTMAP_VEC(sc->sc_ign, j)].iv_icarg;
  594                 mtx_init(&fmqa->fmqa_mtx, "msiq_mtx", NULL, MTX_SPIN);
  595                 fmqa->fmqa_base =
  596                     (struct fo_msiq_record *)((caddr_t)sc->sc_msiq +
  597                     (FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * i));
  598                 j = i + sc->sc_msiq_first;
  599                 fmqa->fmqa_msiq = j;
  600                 j <<= 3;
  601                 fmqa->fmqa_head = FO_PCI_EQ_HD_BASE + j;
  602                 fmqa->fmqa_tail = FO_PCI_EQ_TL_BASE + j;
  603                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + j,
  604                     FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I |
  605                     FO_PCI_EQ_CTRL_CLR_DIS);
  606                 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_tail,
  607                     (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK);
  608                 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head,
  609                     (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK);
  610         }
  611         FIRE_PCI_SET(sc, FO_PCI_MSI_32_BIT_ADDR, sc->sc_msi_addr32 &
  612             FO_PCI_MSI_32_BIT_ADDR_MASK);
  613         FIRE_PCI_SET(sc, FO_PCI_MSI_64_BIT_ADDR, sc->sc_msi_addr64 &
  614             FO_PCI_MSI_64_BIT_ADDR_MASK);
  615 
  616         /*
  617          * Establish a handler for interesting PCIe messages and disable
  618          * unintersting ones.
  619          */
  620         mtx_lock(&sc->sc_msi_mtx);
  621         for (i = 0; i < sc->sc_msiq_count; i++) {
  622                 if (isclr(sc->sc_msiq_bitmap, i) != 0) {
  623                         j = i;
  624                         break;
  625                 }
  626         }
  627         if (i == sc->sc_msiq_count) {
  628                 mtx_unlock(&sc->sc_msi_mtx);
  629                 panic("%s: no spare event queue for PCIe messages", __func__);
  630         }
  631         setbit(sc->sc_msiq_bitmap, j);
  632         mtx_unlock(&sc->sc_msi_mtx);
  633         i = INTMAP_VEC(sc->sc_ign, j + sc->sc_msiq_ino_first);
  634         if (bus_set_resource(dev, SYS_RES_IRQ, 2, i, 1) != 0)
  635                 panic("%s: failed to add interrupt for PCIe messages",
  636                     __func__);
  637         fire_set_intr(sc, 2, INTINO(i), fire_pcie, intr_vectors[i].iv_icarg);
  638         j += sc->sc_msiq_first;
  639         /*
  640          * "Please note that setting the EQNUM field to a value larger than
  641          * 35 will yield unpredictable results."
  642          */
  643         if (j > 35)
  644                 panic("%s: invalid queue for PCIe messages (%d)",
  645                     __func__, j);
  646         FIRE_PCI_SET(sc, FO_PCI_ERR_COR, FO_PCI_ERR_PME_V |
  647             ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
  648         FIRE_PCI_SET(sc, FO_PCI_ERR_NONFATAL, FO_PCI_ERR_PME_V |
  649             ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
  650         FIRE_PCI_SET(sc, FO_PCI_ERR_FATAL, FO_PCI_ERR_PME_V |
  651             ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
  652         FIRE_PCI_SET(sc, FO_PCI_PM_PME, 0);
  653         FIRE_PCI_SET(sc, FO_PCI_PME_TO_ACK, 0);
  654         FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (j << 3),
  655             FO_PCI_EQ_CTRL_SET_EN);
  656 
  657 #define TC_COUNTER_MAX_MASK     0xffffffff
  658 
  659         /*
  660          * Setup JBC/UBC performance counter 0 in bus cycle counting
  661          * mode as timecounter.
  662          */
  663         if (device_get_unit(dev) == 0) {
  664                 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT0, 0);
  665                 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT1, 0);
  666                 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT_SEL,
  667                     (FO_XBC_PRF_CNT_NONE << FO_XBC_PRF_CNT_CNT1_SHFT) |
  668                     (FO_XBC_PRF_CNT_XB_CLK << FO_XBC_PRF_CNT_CNT0_SHFT));
  669                 tc = malloc(sizeof(*tc), M_DEVBUF, M_NOWAIT | M_ZERO);
  670                 if (tc == NULL)
  671                         panic("%s: could not malloc timecounter", __func__);
  672                 tc->tc_get_timecount = fire_get_timecount;
  673                 tc->tc_counter_mask = TC_COUNTER_MAX_MASK;
  674                 if (OF_getprop(OF_peer(0), "clock-frequency", &prop,
  675                     sizeof(prop)) == -1)
  676                         panic("%s: could not determine clock frequency",
  677                             __func__);
  678                 tc->tc_frequency = prop;
  679                 tc->tc_name = strdup(device_get_nameunit(dev), M_DEVBUF);
  680                 tc->tc_priv = sc;
  681                 /*
  682                  * Due to initial problems with the JBus-driven performance
  683                  * counters not advancing which might be firmware dependent
  684                  * ensure that it actually works.
  685                  */
  686                 if (fire_get_timecount(tc) - fire_get_timecount(tc) != 0)
  687                         tc->tc_quality = FIRE_PERF_CNT_QLTY;
  688                 else
  689                         tc->tc_quality = -FIRE_PERF_CNT_QLTY;
  690                 tc_init(tc);
  691         }
  692 
  693         /*
  694          * Set up the IOMMU.  Both Fire and Oberon have one per PBM, but
  695          * neither has a streaming buffer.
  696          */
  697         memcpy(&sc->sc_dma_methods, &iommu_dma_methods,
  698             sizeof(sc->sc_dma_methods));
  699         sc->sc_is.is_flags = IOMMU_FIRE | IOMMU_PRESERVE_PROM;
  700         if (sc->sc_mode == FIRE_MODE_OBERON) {
  701                 sc->sc_is.is_flags |= IOMMU_FLUSH_CACHE;
  702                 sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(OBERON_IOMMU_BITS);
  703         } else {
  704                 sc->sc_dma_methods.dm_dmamap_sync = fire_dmamap_sync;
  705                 sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(FIRE_IOMMU_BITS);
  706         }
  707         sc->sc_is.is_sb[0] = sc->sc_is.is_sb[1] = 0;
  708         /* Punch in our copies. */
  709         sc->sc_is.is_bustag = rman_get_bustag(sc->sc_mem_res[FIRE_PCI]);
  710         sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[FIRE_PCI]);
  711         sc->sc_is.is_iommu = FO_PCI_MMU;
  712         val = FIRE_PCI_READ_8(sc, FO_PCI_MMU + IMR_CTL);
  713         iommu_init(device_get_nameunit(dev), &sc->sc_is, 7, -1, 0);
  714 #ifdef FIRE_DEBUG
  715         device_printf(dev, "FO_PCI_MMU + IMR_CTL 0x%016llx -> 0x%016llx\n",
  716             (long long unsigned)val, (long long unsigned)sc->sc_is.is_cr);
  717 #endif
  718         /* Create our DMA tag. */
  719         if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0x100000000,
  720             sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr,
  721             0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0)
  722                 panic("%s: could not create PCI DMA tag", __func__);
  723         dmat->dt_cookie = &sc->sc_is;
  724         dmat->dt_mt = &sc->sc_dma_methods;
  725 
  726         if (ofw_pci_attach_common(dev, dmat, FO_IO_SIZE, FO_MEM_SIZE) != 0)
  727                 panic("%s: ofw_pci_attach_common() failed", __func__);
  728 
  729 #define FIRE_SYSCTL_ADD_UINT(name, arg, desc)                           \
  730         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),                     \
  731             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,     \
  732             (name), CTLFLAG_RD, (arg), 0, (desc))
  733 
  734         FIRE_SYSCTL_ADD_UINT("ilu_err", &sc->sc_stats_ilu_err,
  735             "ILU unknown errors");
  736         FIRE_SYSCTL_ADD_UINT("jbc_ce_async", &sc->sc_stats_jbc_ce_async,
  737             "JBC correctable errors");
  738         FIRE_SYSCTL_ADD_UINT("jbc_unsol_int", &sc->sc_stats_jbc_unsol_int,
  739             "JBC unsolicited interrupt ACK/NACK errors");
  740         FIRE_SYSCTL_ADD_UINT("jbc_unsol_rd", &sc->sc_stats_jbc_unsol_rd,
  741             "JBC unsolicited read response errors");
  742         FIRE_SYSCTL_ADD_UINT("mmu_err", &sc->sc_stats_mmu_err, "MMU errors");
  743         FIRE_SYSCTL_ADD_UINT("tlu_ce", &sc->sc_stats_tlu_ce,
  744             "DLU/TLU correctable errors");
  745         FIRE_SYSCTL_ADD_UINT("tlu_oe_non_fatal",
  746             &sc->sc_stats_tlu_oe_non_fatal,
  747             "DLU/TLU other event non-fatal errors summary");
  748         FIRE_SYSCTL_ADD_UINT("tlu_oe_rx_err", &sc->sc_stats_tlu_oe_rx_err,
  749             "DLU/TLU receive other event errors");
  750         FIRE_SYSCTL_ADD_UINT("tlu_oe_tx_err", &sc->sc_stats_tlu_oe_tx_err,
  751             "DLU/TLU transmit other event errors");
  752         FIRE_SYSCTL_ADD_UINT("ubc_dmardue", &sc->sc_stats_ubc_dmardue,
  753             "UBC DMARDUE erros");
  754 
  755 #undef FIRE_SYSCTL_ADD_UINT
  756 
  757         device_add_child(dev, "pci", -1);
  758         return (bus_generic_attach(dev));
  759 }
  760 
  761 static void
  762 fire_set_intr(struct fire_softc *sc, u_int index, u_int ino,
  763     driver_filter_t handler, void *arg)
  764 {
  765         u_long vec;
  766         int rid;
  767 
  768         rid = index;
  769         sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev,
  770             SYS_RES_IRQ, &rid, RF_ACTIVE);
  771         if (sc->sc_irq_res[index] == NULL ||
  772             INTINO(vec = rman_get_start(sc->sc_irq_res[index])) != ino ||
  773             INTIGN(vec) != sc->sc_ign ||
  774             intr_vectors[vec].iv_ic != &fire_ic ||
  775             bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index],
  776             INTR_TYPE_MISC | INTR_BRIDGE, handler, NULL, arg,
  777             &sc->sc_ihand[index]) != 0)
  778                 panic("%s: failed to set up interrupt %d", __func__, index);
  779 }
  780 
  781 static int
  782 fire_intr_register(struct fire_softc *sc, u_int ino)
  783 {
  784         struct fire_icarg *fica;
  785         bus_addr_t intrclr, intrmap;
  786         int error;
  787 
  788         if (fire_get_intrmap(sc, ino, &intrmap, &intrclr) == 0)
  789                 return (ENXIO);
  790         fica = malloc((ino >= FO_EQ_FIRST_INO && ino <= FO_EQ_LAST_INO) ?
  791             sizeof(struct fire_msiqarg) : sizeof(struct fire_icarg), M_DEVBUF,
  792             M_NOWAIT | M_ZERO);
  793         if (fica == NULL)
  794                 return (ENOMEM);
  795         fica->fica_sc = sc;
  796         fica->fica_map = intrmap;
  797         fica->fica_clr = intrclr;
  798         error = (intr_controller_register(INTMAP_VEC(sc->sc_ign, ino),
  799             &fire_ic, fica));
  800         if (error != 0)
  801                 free(fica, M_DEVBUF);
  802         return (error);
  803 }
  804 
  805 static int
  806 fire_get_intrmap(struct fire_softc *sc, u_int ino, bus_addr_t *intrmapptr,
  807     bus_addr_t *intrclrptr)
  808 {
  809 
  810         if (ino > FO_MAX_INO) {
  811                 device_printf(sc->sc_dev, "out of range INO %d requested\n",
  812                     ino);
  813                 return (0);
  814         }
  815 
  816         ino <<= 3;
  817         if (intrmapptr != NULL)
  818                 *intrmapptr = FO_PCI_INT_MAP_BASE + ino;
  819         if (intrclrptr != NULL)
  820                 *intrclrptr = FO_PCI_INT_CLR_BASE + ino;
  821         return (1);
  822 }
  823 
  824 /*
  825  * Interrupt handlers
  826  */
  827 static int
  828 fire_dmc_pec(void *arg)
  829 {
  830         struct fire_softc *sc;
  831         device_t dev;
  832         uint64_t cestat, dmcstat, ilustat, imustat, mcstat, mmustat, mmutfar;
  833         uint64_t mmutfsr, oestat, pecstat, uestat, val;
  834         u_int fatal, oenfatal;
  835 
  836         fatal = 0;
  837         sc = arg;
  838         dev = sc->sc_dev;
  839         mtx_lock_spin(&sc->sc_pcib_mtx);
  840         mcstat = FIRE_PCI_READ_8(sc, FO_PCI_MULTI_CORE_ERR_STAT);
  841         if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_DMC) != 0) {
  842                 dmcstat = FIRE_PCI_READ_8(sc, FO_PCI_DMC_CORE_BLOCK_ERR_STAT);
  843                 if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_IMU) != 0) {
  844                         imustat = FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_STAT);
  845                         device_printf(dev, "IMU error %#llx\n",
  846                             (unsigned long long)imustat);
  847                         if ((imustat &
  848                             FO_PCI_IMU_ERR_INT_EQ_NOT_EN_P) != 0) {
  849                                 fatal = 1;
  850                                 val = FIRE_PCI_READ_8(sc,
  851                                     FO_PCI_IMU_SCS_ERR_LOG);
  852                                 device_printf(dev, "SCS error log %#llx\n",
  853                                     (unsigned long long)val);
  854                         }
  855                         if ((imustat & FO_PCI_IMU_ERR_INT_EQ_OVER_P) != 0) {
  856                                 fatal = 1;
  857                                 val = FIRE_PCI_READ_8(sc,
  858                                     FO_PCI_IMU_EQS_ERR_LOG);
  859                                 device_printf(dev, "EQS error log %#llx\n",
  860                                     (unsigned long long)val);
  861                         }
  862                         if ((imustat & (FO_PCI_IMU_ERR_INT_MSI_MAL_ERR_P |
  863                             FO_PCI_IMU_ERR_INT_MSI_PAR_ERR_P |
  864                             FO_PCI_IMU_ERR_INT_PMEACK_MES_NOT_EN_P |
  865                             FO_PCI_IMU_ERR_INT_PMPME_MES_NOT_EN_P |
  866                             FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P |
  867                             FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P |
  868                             FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P |
  869                             FO_PCI_IMU_ERR_INT_MSI_NOT_EN_P)) != 0) {
  870                                 fatal = 1;
  871                                 val = FIRE_PCI_READ_8(sc,
  872                                     FO_PCI_IMU_RDS_ERR_LOG);
  873                                 device_printf(dev, "RDS error log %#llx\n",
  874                                     (unsigned long long)val);
  875                         }
  876                 }
  877                 if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_MMU) != 0) {
  878                         fatal = 1;
  879                         mmustat = FIRE_PCI_READ_8(sc, FO_PCI_MMU_INT_STAT);
  880                         mmutfar = FIRE_PCI_READ_8(sc,
  881                             FO_PCI_MMU_TRANS_FAULT_ADDR);
  882                         mmutfsr = FIRE_PCI_READ_8(sc,
  883                             FO_PCI_MMU_TRANS_FAULT_STAT);
  884                         if ((mmustat & (FO_PCI_MMU_ERR_INT_TBW_DPE_P |
  885                             FO_PCI_MMU_ERR_INT_TBW_ERR_P |
  886                             FO_PCI_MMU_ERR_INT_TBW_UDE_P |
  887                             FO_PCI_MMU_ERR_INT_TBW_DME_P |
  888                             FO_PCI_MMU_ERR_INT_TTC_CAE_P |
  889                             FIRE_PCI_MMU_ERR_INT_TTC_DPE_P |
  890                             OBERON_PCI_MMU_ERR_INT_TTC_DUE_P |
  891                             FO_PCI_MMU_ERR_INT_TRN_ERR_P)) != 0)
  892                                 fatal = 1;
  893                         else {
  894                                 sc->sc_stats_mmu_err++;
  895                                 FIRE_PCI_WRITE_8(sc, FO_PCI_MMU_ERR_STAT_CLR,
  896                                     mmustat);
  897                         }
  898                         device_printf(dev,
  899                             "MMU error %#llx: TFAR %#llx TFSR %#llx\n",
  900                             (unsigned long long)mmustat,
  901                             (unsigned long long)mmutfar,
  902                             (unsigned long long)mmutfsr);
  903                 }
  904         }
  905         if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_PEC) != 0) {
  906                 pecstat = FIRE_PCI_READ_8(sc, FO_PCI_PEC_CORE_BLOCK_INT_STAT);
  907                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_UERR) != 0) {
  908                         fatal = 1;
  909                         uestat = FIRE_PCI_READ_8(sc,
  910                             FO_PCI_TLU_UERR_INT_STAT);
  911                         device_printf(dev,
  912                             "DLU/TLU uncorrectable error %#llx\n",
  913                             (unsigned long long)uestat);
  914                         if ((uestat & (FO_PCI_TLU_UERR_INT_UR_P |
  915                             OBERON_PCI_TLU_UERR_INT_POIS_P |
  916                             FO_PCI_TLU_UERR_INT_MFP_P |
  917                             FO_PCI_TLU_UERR_INT_ROF_P |
  918                             FO_PCI_TLU_UERR_INT_UC_P |
  919                             FIRE_PCI_TLU_UERR_INT_PP_P |
  920                             OBERON_PCI_TLU_UERR_INT_POIS_P)) != 0) {
  921                                 val = FIRE_PCI_READ_8(sc,
  922                                     FO_PCI_TLU_RX_UERR_HDR1_LOG);
  923                                 device_printf(dev,
  924                                     "receive header log %#llx\n",
  925                                     (unsigned long long)val);
  926                                 val = FIRE_PCI_READ_8(sc,
  927                                     FO_PCI_TLU_RX_UERR_HDR2_LOG);
  928                                 device_printf(dev,
  929                                     "receive header log 2 %#llx\n",
  930                                     (unsigned long long)val);
  931                         }
  932                         if ((uestat & FO_PCI_TLU_UERR_INT_CTO_P) != 0) {
  933                                 val = FIRE_PCI_READ_8(sc,
  934                                     FO_PCI_TLU_TX_UERR_HDR1_LOG);
  935                                 device_printf(dev,
  936                                     "transmit header log %#llx\n",
  937                                     (unsigned long long)val);
  938                                 val = FIRE_PCI_READ_8(sc,
  939                                     FO_PCI_TLU_TX_UERR_HDR2_LOG);
  940                                 device_printf(dev,
  941                                     "transmit header log 2 %#llx\n",
  942                                     (unsigned long long)val);
  943                         }
  944                         if ((uestat & FO_PCI_TLU_UERR_INT_DLP_P) != 0) {
  945                                 val = FIRE_PCI_READ_8(sc,
  946                                     FO_PCI_LPU_LNK_LYR_INT_STAT);
  947                                 device_printf(dev,
  948                                     "link layer interrupt and status %#llx\n",
  949                                     (unsigned long long)val);
  950                         }
  951                         if ((uestat & FO_PCI_TLU_UERR_INT_TE_P) != 0) {
  952                                 val = FIRE_PCI_READ_8(sc,
  953                                     FO_PCI_LPU_PHY_LYR_INT_STAT);
  954                                 device_printf(dev,
  955                                     "phy layer interrupt and status %#llx\n",
  956                                     (unsigned long long)val);
  957                         }
  958                 }
  959                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_CERR) != 0) {
  960                         sc->sc_stats_tlu_ce++;
  961                         cestat = FIRE_PCI_READ_8(sc,
  962                             FO_PCI_TLU_CERR_INT_STAT);
  963                         device_printf(dev,
  964                             "DLU/TLU correctable error %#llx\n",
  965                             (unsigned long long)cestat);
  966                         val = FIRE_PCI_READ_8(sc,
  967                             FO_PCI_LPU_LNK_LYR_INT_STAT);
  968                         device_printf(dev,
  969                             "link layer interrupt and status %#llx\n",
  970                             (unsigned long long)val);
  971                         if ((cestat & FO_PCI_TLU_CERR_INT_RE_P) != 0) {
  972                                 FIRE_PCI_WRITE_8(sc,
  973                                     FO_PCI_LPU_LNK_LYR_INT_STAT, val);
  974                                 val = FIRE_PCI_READ_8(sc,
  975                                     FO_PCI_LPU_PHY_LYR_INT_STAT);
  976                                 device_printf(dev,
  977                                     "phy layer interrupt and status %#llx\n",
  978                                     (unsigned long long)val);
  979                         }
  980                         FIRE_PCI_WRITE_8(sc, FO_PCI_TLU_CERR_STAT_CLR,
  981                             cestat);
  982                 }
  983                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_OEVENT) != 0) {
  984                         oenfatal = 0;
  985                         oestat = FIRE_PCI_READ_8(sc,
  986                             FO_PCI_TLU_OEVENT_INT_STAT);
  987                         device_printf(dev, "DLU/TLU other event %#llx\n",
  988                             (unsigned long long)oestat);
  989                         if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
  990                             FO_PCI_TLU_OEVENT_MRC_P |
  991                             FO_PCI_TLU_OEVENT_WUC_P |
  992                             FO_PCI_TLU_OEVENT_RUC_P |
  993                             FO_PCI_TLU_OEVENT_CRS_P)) != 0) {
  994                                 val = FIRE_PCI_READ_8(sc,
  995                                     FO_PCI_TLU_RX_OEVENT_HDR1_LOG);
  996                                 device_printf(dev,
  997                                     "receive header log %#llx\n",
  998                                     (unsigned long long)val);
  999                                 val = FIRE_PCI_READ_8(sc,
 1000                                     FO_PCI_TLU_RX_OEVENT_HDR2_LOG);
 1001                                 device_printf(dev,
 1002                                     "receive header log 2 %#llx\n",
 1003                                     (unsigned long long)val);
 1004                                 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
 1005                                     FO_PCI_TLU_OEVENT_MRC_P |
 1006                                     FO_PCI_TLU_OEVENT_WUC_P |
 1007                                     FO_PCI_TLU_OEVENT_RUC_P)) != 0)
 1008                                         fatal = 1;
 1009                                 else {
 1010                                         sc->sc_stats_tlu_oe_rx_err++;
 1011                                         oenfatal = 1;
 1012                                 }
 1013                         }
 1014                         if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
 1015                             FO_PCI_TLU_OEVENT_CTO_P |
 1016                             FO_PCI_TLU_OEVENT_WUC_P |
 1017                             FO_PCI_TLU_OEVENT_RUC_P)) != 0) {
 1018                                 val = FIRE_PCI_READ_8(sc,
 1019                                     FO_PCI_TLU_TX_OEVENT_HDR1_LOG);
 1020                                 device_printf(dev,
 1021                                     "transmit header log %#llx\n",
 1022                                     (unsigned long long)val);
 1023                                 val = FIRE_PCI_READ_8(sc,
 1024                                     FO_PCI_TLU_TX_OEVENT_HDR2_LOG);
 1025                                 device_printf(dev,
 1026                                     "transmit header log 2 %#llx\n",
 1027                                     (unsigned long long)val);
 1028                                 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
 1029                                     FO_PCI_TLU_OEVENT_CTO_P |
 1030                                     FO_PCI_TLU_OEVENT_WUC_P |
 1031                                     FO_PCI_TLU_OEVENT_RUC_P)) != 0)
 1032                                         fatal = 1;
 1033                                 else {
 1034                                         sc->sc_stats_tlu_oe_tx_err++;
 1035                                         oenfatal = 1;
 1036                                 }
 1037                         }
 1038                         if ((oestat & (FO_PCI_TLU_OEVENT_ERO_P |
 1039                             FO_PCI_TLU_OEVENT_EMP_P |
 1040                             FO_PCI_TLU_OEVENT_EPE_P |
 1041                             FIRE_PCI_TLU_OEVENT_ERP_P |
 1042                             OBERON_PCI_TLU_OEVENT_ERBU_P |
 1043                             FIRE_PCI_TLU_OEVENT_EIP_P |
 1044                             OBERON_PCI_TLU_OEVENT_EIUE_P)) != 0) {
 1045                                 fatal = 1;
 1046                                 val = FIRE_PCI_READ_8(sc,
 1047                                     FO_PCI_LPU_LNK_LYR_INT_STAT);
 1048                                 device_printf(dev,
 1049                                     "link layer interrupt and status %#llx\n",
 1050                                     (unsigned long long)val);
 1051                         }
 1052                         if ((oestat & (FO_PCI_TLU_OEVENT_IIP_P |
 1053                             FO_PCI_TLU_OEVENT_EDP_P |
 1054                             FIRE_PCI_TLU_OEVENT_EHP_P |
 1055                             OBERON_PCI_TLU_OEVENT_TLUEITMO_S |
 1056                             FO_PCI_TLU_OEVENT_ERU_P)) != 0)
 1057                                 fatal = 1;
 1058                         if ((oestat & (FO_PCI_TLU_OEVENT_NFP_P |
 1059                             FO_PCI_TLU_OEVENT_LWC_P |
 1060                             FO_PCI_TLU_OEVENT_LIN_P |
 1061                             FO_PCI_TLU_OEVENT_LRS_P |
 1062                             FO_PCI_TLU_OEVENT_LDN_P |
 1063                             FO_PCI_TLU_OEVENT_LUP_P)) != 0)
 1064                                 oenfatal = 1;
 1065                         if (oenfatal != 0) {
 1066                                 sc->sc_stats_tlu_oe_non_fatal++;
 1067                                 FIRE_PCI_WRITE_8(sc,
 1068                                     FO_PCI_TLU_OEVENT_STAT_CLR, oestat);
 1069                                 if ((oestat & FO_PCI_TLU_OEVENT_LIN_P) != 0)
 1070                                         FIRE_PCI_WRITE_8(sc,
 1071                                             FO_PCI_LPU_LNK_LYR_INT_STAT,
 1072                                             FIRE_PCI_READ_8(sc,
 1073                                             FO_PCI_LPU_LNK_LYR_INT_STAT));
 1074                         }
 1075                 }
 1076                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_ILU) != 0) {
 1077                         ilustat = FIRE_PCI_READ_8(sc, FO_PCI_ILU_INT_STAT);
 1078                         device_printf(dev, "ILU error %#llx\n",
 1079                             (unsigned long long)ilustat);
 1080                         if ((ilustat & (FIRE_PCI_ILU_ERR_INT_IHB_PE_P |
 1081                             FIRE_PCI_ILU_ERR_INT_IHB_PE_P)) != 0)
 1082                             fatal = 1;
 1083                         else {
 1084                                 sc->sc_stats_ilu_err++;
 1085                                 FIRE_PCI_WRITE_8(sc, FO_PCI_ILU_INT_STAT,
 1086                                     ilustat);
 1087                         }
 1088                 }
 1089         }
 1090         mtx_unlock_spin(&sc->sc_pcib_mtx);
 1091         if (fatal != 0)
 1092                 panic("%s: fatal DMC/PEC error",
 1093                     device_get_nameunit(sc->sc_dev));
 1094         return (FILTER_HANDLED);
 1095 }
 1096 
 1097 static int
 1098 fire_xcb(void *arg)
 1099 {
 1100         struct fire_softc *sc;
 1101         device_t dev;
 1102         uint64_t errstat, intstat, val;
 1103         u_int fatal;
 1104 
 1105         fatal = 0;
 1106         sc = arg;
 1107         dev = sc->sc_dev;
 1108         mtx_lock_spin(&sc->sc_pcib_mtx);
 1109         if (sc->sc_mode == FIRE_MODE_OBERON) {
 1110                 intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT);
 1111                 device_printf(dev, "UBC error: interrupt status %#llx\n",
 1112                     (unsigned long long)intstat);
 1113                 if ((intstat & ~(OBERON_UBC_ERR_INT_DMARDUEB_P |
 1114                     OBERON_UBC_ERR_INT_DMARDUEA_P)) != 0)
 1115                         fatal = 1;
 1116                 else
 1117                         sc->sc_stats_ubc_dmardue++;
 1118                 if (fatal != 0) {
 1119                         mtx_unlock_spin(&sc->sc_pcib_mtx);
 1120                         panic("%s: fatal UBC core block error",
 1121                             device_get_nameunit(sc->sc_dev));
 1122                 } else {
 1123                         FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
 1124                         mtx_unlock_spin(&sc->sc_pcib_mtx);
 1125                 }
 1126         } else {
 1127                 errstat = FIRE_CTRL_READ_8(sc, FIRE_JBC_CORE_BLOCK_ERR_STAT);
 1128                 if ((errstat & (FIRE_JBC_CORE_BLOCK_ERR_STAT_MERGE |
 1129                     FIRE_JBC_CORE_BLOCK_ERR_STAT_JBCINT |
 1130                     FIRE_JBC_CORE_BLOCK_ERR_STAT_DMCINT)) != 0) {
 1131                         intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT);
 1132                         device_printf(dev, "JBC interrupt status %#llx\n",
 1133                             (unsigned long long)intstat);
 1134                         if ((intstat & FIRE_JBC_ERR_INT_EBUS_TO_P) != 0) {
 1135                                 val = FIRE_CTRL_READ_8(sc,
 1136                                     FIRE_JBC_CSR_ERR_LOG);
 1137                                 device_printf(dev, "CSR error log %#llx\n",
 1138                                     (unsigned long long)val);
 1139                         }
 1140                         if ((intstat & (FIRE_JBC_ERR_INT_UNSOL_RD_P |
 1141                             FIRE_JBC_ERR_INT_UNSOL_INT_P)) != 0) {
 1142                                 if ((intstat &
 1143                                     FIRE_JBC_ERR_INT_UNSOL_RD_P) != 0)
 1144                                         sc->sc_stats_jbc_unsol_rd++;
 1145                                 if ((intstat &
 1146                                     FIRE_JBC_ERR_INT_UNSOL_INT_P) != 0)
 1147                                         sc->sc_stats_jbc_unsol_int++;
 1148                                 val = FIRE_CTRL_READ_8(sc,
 1149                                     FIRE_DMCINT_IDC_ERR_LOG);
 1150                                 device_printf(dev,
 1151                                     "DMCINT IDC error log %#llx\n",
 1152                                     (unsigned long long)val);
 1153                         }
 1154                         if ((intstat & (FIRE_JBC_ERR_INT_MB_PER_P |
 1155                             FIRE_JBC_ERR_INT_MB_PEW_P)) != 0) {
 1156                                 fatal = 1;
 1157                                 val = FIRE_CTRL_READ_8(sc,
 1158                                     FIRE_MERGE_TRANS_ERR_LOG);
 1159                                 device_printf(dev,
 1160                                     "merge transaction error log %#llx\n",
 1161                                     (unsigned long long)val);
 1162                         }
 1163                         if ((intstat & FIRE_JBC_ERR_INT_IJP_P) != 0) {
 1164                                 fatal = 1;
 1165                                 val = FIRE_CTRL_READ_8(sc,
 1166                                     FIRE_JBCINT_OTRANS_ERR_LOG);
 1167                                 device_printf(dev,
 1168                                     "JBCINT out transaction error log "
 1169                                     "%#llx\n", (unsigned long long)val);
 1170                                 val = FIRE_CTRL_READ_8(sc,
 1171                                     FIRE_JBCINT_OTRANS_ERR_LOG2);
 1172                                 device_printf(dev,
 1173                                     "JBCINT out transaction error log 2 "
 1174                                     "%#llx\n", (unsigned long long)val);
 1175                         }
 1176                         if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P |
 1177                             FIRE_JBC_ERR_INT_CE_ASYN_P |
 1178                             FIRE_JBC_ERR_INT_JTE_P | FIRE_JBC_ERR_INT_JBE_P |
 1179                             FIRE_JBC_ERR_INT_JUE_P |
 1180                             FIRE_JBC_ERR_INT_ICISE_P |
 1181                             FIRE_JBC_ERR_INT_WR_DPE_P |
 1182                             FIRE_JBC_ERR_INT_RD_DPE_P |
 1183                             FIRE_JBC_ERR_INT_ILL_BMW_P |
 1184                             FIRE_JBC_ERR_INT_ILL_BMR_P |
 1185                             FIRE_JBC_ERR_INT_BJC_P)) != 0) {
 1186                                 if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P |
 1187                                     FIRE_JBC_ERR_INT_JTE_P |
 1188                                     FIRE_JBC_ERR_INT_JBE_P |
 1189                                     FIRE_JBC_ERR_INT_JUE_P |
 1190                                     FIRE_JBC_ERR_INT_ICISE_P |
 1191                                     FIRE_JBC_ERR_INT_WR_DPE_P |
 1192                                     FIRE_JBC_ERR_INT_RD_DPE_P |
 1193                                     FIRE_JBC_ERR_INT_ILL_BMW_P |
 1194                                     FIRE_JBC_ERR_INT_ILL_BMR_P |
 1195                                     FIRE_JBC_ERR_INT_BJC_P)) != 0)
 1196                                         fatal = 1;
 1197                                 else
 1198                                         sc->sc_stats_jbc_ce_async++;
 1199                                 val = FIRE_CTRL_READ_8(sc,
 1200                                     FIRE_JBCINT_ITRANS_ERR_LOG);
 1201                                 device_printf(dev,
 1202                                     "JBCINT in transaction error log %#llx\n",
 1203                                     (unsigned long long)val);
 1204                                 val = FIRE_CTRL_READ_8(sc,
 1205                                     FIRE_JBCINT_ITRANS_ERR_LOG2);
 1206                                 device_printf(dev,
 1207                                     "JBCINT in transaction error log 2 "
 1208                                     "%#llx\n", (unsigned long long)val);
 1209                         }
 1210                         if ((intstat & (FIRE_JBC_ERR_INT_PIO_UNMAP_RD_P |
 1211                             FIRE_JBC_ERR_INT_ILL_ACC_RD_P |
 1212                             FIRE_JBC_ERR_INT_PIO_UNMAP_P |
 1213                             FIRE_JBC_ERR_INT_PIO_DPE_P |
 1214                             FIRE_JBC_ERR_INT_PIO_CPE_P |
 1215                             FIRE_JBC_ERR_INT_ILL_ACC_P)) != 0) {
 1216                                 fatal = 1;
 1217                                 val = FIRE_CTRL_READ_8(sc,
 1218                                     FIRE_JBC_CSR_ERR_LOG);
 1219                                 device_printf(dev,
 1220                                     "DMCINT ODCD error log %#llx\n",
 1221                                     (unsigned long long)val);
 1222                         }
 1223                         if ((intstat & (FIRE_JBC_ERR_INT_MB_PEA_P |
 1224                             FIRE_JBC_ERR_INT_CPE_P | FIRE_JBC_ERR_INT_APE_P |
 1225                             FIRE_JBC_ERR_INT_PIO_CPE_P |
 1226                             FIRE_JBC_ERR_INT_JTCEEW_P |
 1227                             FIRE_JBC_ERR_INT_JTCEEI_P |
 1228                             FIRE_JBC_ERR_INT_JTCEER_P)) != 0) {
 1229                                 fatal = 1;
 1230                                 val = FIRE_CTRL_READ_8(sc,
 1231                                     FIRE_FATAL_ERR_LOG);
 1232                                 device_printf(dev, "fatal error log %#llx\n",
 1233                                     (unsigned long long)val);
 1234                                 val = FIRE_CTRL_READ_8(sc,
 1235                                     FIRE_FATAL_ERR_LOG2);
 1236                                 device_printf(dev, "fatal error log 2 "
 1237                                     "%#llx\n", (unsigned long long)val);
 1238                         }
 1239                         if (fatal != 0) {
 1240                                 mtx_unlock_spin(&sc->sc_pcib_mtx);
 1241                                 panic("%s: fatal JBC core block error",
 1242                                     device_get_nameunit(sc->sc_dev));
 1243                         } else {
 1244                                 FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
 1245                                 mtx_unlock_spin(&sc->sc_pcib_mtx);
 1246                         }
 1247                 } else {
 1248                         mtx_unlock_spin(&sc->sc_pcib_mtx);
 1249                         panic("%s: unknown JCB core block error status %#llx",
 1250                             device_get_nameunit(sc->sc_dev),
 1251                             (unsigned long long)errstat);
 1252                 }
 1253         }
 1254         return (FILTER_HANDLED);
 1255 }
 1256 
 1257 static int
 1258 fire_pcie(void *arg)
 1259 {
 1260         struct fire_msiqarg *fmqa;
 1261         struct fire_softc *sc;
 1262         struct fo_msiq_record *qrec;
 1263         device_t dev;
 1264         uint64_t word0;
 1265         u_int head, msg, msiq;
 1266 
 1267         fmqa = arg;
 1268         sc = fmqa->fmqa_fica.fica_sc;
 1269         dev = sc->sc_dev;
 1270         msiq = fmqa->fmqa_msiq;
 1271         mtx_lock_spin(&fmqa->fmqa_mtx);
 1272         head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >>
 1273             FO_PCI_EQ_HD_SHFT;
 1274         qrec = &fmqa->fmqa_base[head];
 1275         word0 = qrec->fomqr_word0;
 1276         for (;;) {
 1277                 KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSG) != 0,
 1278                     ("%s: received non-PCIe message in event queue %d "
 1279                     "(word0 %#llx)", device_get_nameunit(dev), msiq,
 1280                     (unsigned long long)word0));
 1281                 msg = (word0 & FO_MQR_WORD0_DATA0_MASK) >>
 1282                     FO_MQR_WORD0_DATA0_SHFT;
 1283 
 1284 #define PCIE_MSG_CODE_ERR_COR           0x30
 1285 #define PCIE_MSG_CODE_ERR_NONFATAL      0x31
 1286 #define PCIE_MSG_CODE_ERR_FATAL         0x33
 1287 
 1288                 if (msg == PCIE_MSG_CODE_ERR_COR)
 1289                         device_printf(dev, "correctable PCIe error\n");
 1290                 else if (msg == PCIE_MSG_CODE_ERR_NONFATAL ||
 1291                     msg == PCIE_MSG_CODE_ERR_FATAL)
 1292                         panic("%s: %sfatal PCIe error",
 1293                             device_get_nameunit(dev),
 1294                             msg == PCIE_MSG_CODE_ERR_NONFATAL ? "non-" : "");
 1295                 else
 1296                         panic("%s: received unknown PCIe message %#x",
 1297                             device_get_nameunit(dev), msg);
 1298                 qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK;
 1299                 head = (head + 1) % sc->sc_msiq_size;
 1300                 qrec = &fmqa->fmqa_base[head];
 1301                 word0 = qrec->fomqr_word0;
 1302                 if (__predict_true((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0))
 1303                         break;
 1304         }
 1305         FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) <<
 1306             FO_PCI_EQ_HD_SHFT);
 1307         if ((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) &
 1308             FO_PCI_EQ_TL_OVERR) != 0) {
 1309                 device_printf(dev, "event queue %d overflow\n", msiq);
 1310                 msiq <<= 3;
 1311                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
 1312                     FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) |
 1313                     FO_PCI_EQ_CTRL_CLR_COVERR);
 1314         }
 1315         mtx_unlock_spin(&fmqa->fmqa_mtx);
 1316         return (FILTER_HANDLED);
 1317 }
 1318 
 1319 static int
 1320 fire_maxslots(device_t dev)
 1321 {
 1322 
 1323         return (1);
 1324 }
 1325 
 1326 static uint32_t
 1327 fire_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
 1328     int width)
 1329 {
 1330 
 1331         return (ofw_pci_read_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus,
 1332             slot, func, reg), bus, slot, func, reg, width));
 1333 }
 1334 
 1335 static void
 1336 fire_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
 1337     uint32_t val, int width)
 1338 {
 1339 
 1340         ofw_pci_write_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus, slot,
 1341             func, reg), bus, slot, func, reg, val, width);
 1342 }
 1343 
 1344 static int
 1345 fire_route_interrupt(device_t bridge, device_t dev, int pin)
 1346 {
 1347         ofw_pci_intr_t mintr;
 1348 
 1349         mintr = ofw_pci_route_interrupt_common(bridge, dev, pin);
 1350         if (!PCI_INTERRUPT_VALID(mintr))
 1351                 device_printf(bridge,
 1352                     "could not route pin %d for device %d.%d\n",
 1353                     pin, pci_get_slot(dev), pci_get_function(dev));
 1354         return (mintr);
 1355 }
 1356 
 1357 static void
 1358 fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
 1359     bus_dmasync_op_t op)
 1360 {
 1361 
 1362         if ((map->dm_flags & DMF_LOADED) == 0)
 1363                 return;
 1364 
 1365         if ((op & BUS_DMASYNC_POSTREAD) != 0)
 1366                 ofw_pci_dmamap_sync_stst_order_common();
 1367         else if ((op & BUS_DMASYNC_PREWRITE) != 0)
 1368                 membar(Sync);
 1369 }
 1370 
 1371 static void
 1372 fire_intr_enable(void *arg)
 1373 {
 1374         struct intr_vector *iv;
 1375         struct fire_icarg *fica;
 1376         struct fire_softc *sc;
 1377         struct pcpu *pc;
 1378         uint64_t mr;
 1379         u_int ctrl, i;
 1380 
 1381         iv = arg;
 1382         fica = iv->iv_icarg;
 1383         sc = fica->fica_sc;
 1384         mr = FO_PCI_IMAP_V;
 1385         if (sc->sc_mode == FIRE_MODE_OBERON)
 1386                 mr |= (iv->iv_mid << OBERON_PCI_IMAP_T_DESTID_SHFT) &
 1387                     OBERON_PCI_IMAP_T_DESTID_MASK;
 1388         else
 1389                 mr |= (iv->iv_mid << FIRE_PCI_IMAP_T_JPID_SHFT) &
 1390                     FIRE_PCI_IMAP_T_JPID_MASK;
 1391         /*
 1392          * Given that all mondos for the same target are required to use the
 1393          * same interrupt controller we just use the CPU ID for indexing the
 1394          * latter.
 1395          */
 1396         ctrl = 0;
 1397         for (i = 0; i < mp_ncpus; ++i) {
 1398                 pc = pcpu_find(i);
 1399                 if (pc == NULL || iv->iv_mid != pc->pc_mid)
 1400                         continue;
 1401                 ctrl = pc->pc_cpuid % 4;
 1402                 break;
 1403         }
 1404         mr |= (1ULL << ctrl) << FO_PCI_IMAP_INT_CTRL_NUM_SHFT &
 1405             FO_PCI_IMAP_INT_CTRL_NUM_MASK;
 1406         FIRE_PCI_WRITE_8(sc, fica->fica_map, mr);
 1407 }
 1408 
 1409 static void
 1410 fire_intr_disable(void *arg)
 1411 {
 1412         struct intr_vector *iv;
 1413         struct fire_icarg *fica;
 1414         struct fire_softc *sc;
 1415 
 1416         iv = arg;
 1417         fica = iv->iv_icarg;
 1418         sc = fica->fica_sc;
 1419         FIRE_PCI_WRITE_8(sc, fica->fica_map,
 1420             FIRE_PCI_READ_8(sc, fica->fica_map) & ~FO_PCI_IMAP_V);
 1421 }
 1422 
 1423 static void
 1424 fire_intr_assign(void *arg)
 1425 {
 1426         struct intr_vector *iv;
 1427         struct fire_icarg *fica;
 1428         struct fire_softc *sc;
 1429         uint64_t mr;
 1430 
 1431         iv = arg;
 1432         fica = iv->iv_icarg;
 1433         sc = fica->fica_sc;
 1434         mr = FIRE_PCI_READ_8(sc, fica->fica_map);
 1435         if ((mr & FO_PCI_IMAP_V) != 0) {
 1436                 FIRE_PCI_WRITE_8(sc, fica->fica_map, mr & ~FO_PCI_IMAP_V);
 1437                 FIRE_PCI_BARRIER(sc, fica->fica_map, 8,
 1438                     BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 1439         }
 1440         while (FIRE_PCI_READ_8(sc, fica->fica_clr) != INTCLR_IDLE)
 1441                 ;
 1442         if ((mr & FO_PCI_IMAP_V) != 0)
 1443                 fire_intr_enable(arg);
 1444 }
 1445 
 1446 static void
 1447 fire_intr_clear(void *arg)
 1448 {
 1449         struct intr_vector *iv;
 1450         struct fire_icarg *fica;
 1451 
 1452         iv = arg;
 1453         fica = iv->iv_icarg;
 1454         FIRE_PCI_WRITE_8(fica->fica_sc, fica->fica_clr, INTCLR_IDLE);
 1455 }
 1456 
 1457 /*
 1458  * Given that the event queue implementation matches our current MD and MI
 1459  * interrupt frameworks like square pegs fit into round holes we are generous
 1460  * and use one event queue per MSI for now, which limits us to 35 MSIs/MSI-Xs
 1461  * per Host-PCIe-bridge (we use one event queue for the PCIe error messages).
 1462  * This seems tolerable as long as most devices just use one MSI/MSI-X anyway.
 1463  * Adding knowledge about MSIs/MSI-Xs to the MD interrupt code should allow us
 1464  * to decouple the 1:1 mapping at the cost of no longer being able to bind
 1465  * MSIs/MSI-Xs to specific CPUs as we currently have no reliable way to
 1466  * quiesce a device while we move its MSIs/MSI-Xs to another event queue.
 1467  */
 1468 
 1469 static int
 1470 fire_alloc_msi(device_t dev, device_t child, int count, int maxcount __unused,
 1471     int *irqs)
 1472 {
 1473         struct fire_softc *sc;
 1474         u_int i, j, msiqrun;
 1475 
 1476         if (powerof2(count) == 0 || count > 32)
 1477                 return (EINVAL);
 1478 
 1479         sc = device_get_softc(dev);
 1480         mtx_lock(&sc->sc_msi_mtx);
 1481         msiqrun = 0;
 1482         for (i = 0; i < sc->sc_msiq_count; i++) {
 1483                 for (j = i; j < i + count; j++) {
 1484                         if (isclr(sc->sc_msiq_bitmap, j) == 0)
 1485                                 break;
 1486                 }
 1487                 if (j == i + count) {
 1488                         msiqrun = i;
 1489                         break;
 1490                 }
 1491         }
 1492         if (i == sc->sc_msiq_count) {
 1493                 mtx_unlock(&sc->sc_msi_mtx);
 1494                 return (ENXIO);
 1495         }
 1496         for (i = 0; i + count < sc->sc_msi_count; i += count) {
 1497                 for (j = i; j < i + count; j++)
 1498                         if (isclr(sc->sc_msi_bitmap, j) == 0)
 1499                                 break;
 1500                 if (j == i + count) {
 1501                         for (j = 0; j < count; j++) {
 1502                                 setbit(sc->sc_msiq_bitmap, msiqrun + j);
 1503                                 setbit(sc->sc_msi_bitmap, i + j);
 1504                                 sc->sc_msi_msiq_table[i + j] = msiqrun + j;
 1505                                 irqs[j] = sc->sc_msi_first + i + j;
 1506                         }
 1507                         mtx_unlock(&sc->sc_msi_mtx);
 1508                         return (0);
 1509                 }
 1510         }
 1511         mtx_unlock(&sc->sc_msi_mtx);
 1512         return (ENXIO);
 1513 }
 1514 
 1515 static int
 1516 fire_release_msi(device_t dev, device_t child, int count, int *irqs)
 1517 {
 1518         struct fire_softc *sc;
 1519         u_int i;
 1520 
 1521         sc = device_get_softc(dev);
 1522         mtx_lock(&sc->sc_msi_mtx);
 1523         for (i = 0; i < count; i++) {
 1524                 clrbit(sc->sc_msiq_bitmap,
 1525                     sc->sc_msi_msiq_table[irqs[i] - sc->sc_msi_first]);
 1526                 clrbit(sc->sc_msi_bitmap, irqs[i] - sc->sc_msi_first);
 1527         }
 1528         mtx_unlock(&sc->sc_msi_mtx);
 1529         return (0);
 1530 }
 1531 
 1532 static int
 1533 fire_alloc_msix(device_t dev, device_t child, int *irq)
 1534 {
 1535         struct fire_softc *sc;
 1536         int i, msiq;
 1537 
 1538         sc = device_get_softc(dev);
 1539         if ((sc->sc_flags & FIRE_MSIX) == 0)
 1540                 return (ENXIO);
 1541         mtx_lock(&sc->sc_msi_mtx);
 1542         msiq = 0;
 1543         for (i = 0; i < sc->sc_msiq_count; i++) {
 1544                 if (isclr(sc->sc_msiq_bitmap, i) != 0) {
 1545                         msiq = i;
 1546                         break;
 1547                 }
 1548         }
 1549         if (i == sc->sc_msiq_count) {
 1550                 mtx_unlock(&sc->sc_msi_mtx);
 1551                 return (ENXIO);
 1552         }
 1553         for (i = sc->sc_msi_count - 1; i >= 0; i--) {
 1554                 if (isclr(sc->sc_msi_bitmap, i) != 0) {
 1555                         setbit(sc->sc_msiq_bitmap, msiq);
 1556                         setbit(sc->sc_msi_bitmap, i);
 1557                         sc->sc_msi_msiq_table[i] = msiq;
 1558                         *irq = sc->sc_msi_first + i;
 1559                         mtx_unlock(&sc->sc_msi_mtx);
 1560                         return (0);
 1561                 }
 1562         }
 1563         mtx_unlock(&sc->sc_msi_mtx);
 1564         return (ENXIO);
 1565 }
 1566 
 1567 static int
 1568 fire_release_msix(device_t dev, device_t child, int irq)
 1569 {
 1570         struct fire_softc *sc;
 1571 
 1572         sc = device_get_softc(dev);
 1573         if ((sc->sc_flags & FIRE_MSIX) == 0)
 1574                 return (ENXIO);
 1575         mtx_lock(&sc->sc_msi_mtx);
 1576         clrbit(sc->sc_msiq_bitmap,
 1577             sc->sc_msi_msiq_table[irq - sc->sc_msi_first]);
 1578         clrbit(sc->sc_msi_bitmap, irq - sc->sc_msi_first);
 1579         mtx_unlock(&sc->sc_msi_mtx);
 1580         return (0);
 1581 }
 1582 
 1583 static int
 1584 fire_map_msi(device_t dev, device_t child, int irq, uint64_t *addr,
 1585     uint32_t *data)
 1586 {
 1587         struct fire_softc *sc;
 1588         struct pci_devinfo *dinfo;
 1589 
 1590         sc = device_get_softc(dev);
 1591         dinfo = device_get_ivars(child);
 1592         if (dinfo->cfg.msi.msi_alloc > 0) {
 1593                 if ((irq & ~sc->sc_msi_data_mask) != 0) {
 1594                         device_printf(dev, "invalid MSI 0x%x\n", irq);
 1595                         return (EINVAL);
 1596                 }
 1597         } else {
 1598                 if ((sc->sc_flags & FIRE_MSIX) == 0)
 1599                         return (ENXIO);
 1600                 if (fls(irq) > sc->sc_msix_data_width) {
 1601                         device_printf(dev, "invalid MSI-X 0x%x\n", irq);
 1602                         return (EINVAL);
 1603                 }
 1604         }
 1605         if (dinfo->cfg.msi.msi_alloc > 0 &&
 1606             (dinfo->cfg.msi.msi_ctrl & PCIM_MSICTRL_64BIT) == 0)
 1607                 *addr = sc->sc_msi_addr32;
 1608         else
 1609                 *addr = sc->sc_msi_addr64;
 1610         *data = irq;
 1611         return (0);
 1612 }
 1613 
 1614 static void
 1615 fire_msiq_handler(void *cookie)
 1616 {
 1617         struct intr_vector *iv;
 1618         struct fire_msiqarg *fmqa;
 1619 
 1620         iv = cookie;
 1621         fmqa = iv->iv_icarg;
 1622         /*
 1623          * Note that since fire_intr_clear() will clear the event queue
 1624          * interrupt after the handler associated with the MSI [sic] has
 1625          * been executed we have to protect the access to the event queue as
 1626          * otherwise nested event queue interrupts cause corruption of the
 1627          * event queue on MP machines.  Obviously especially when abandoning
 1628          * the 1:1 mapping it would be better to not clear the event queue
 1629          * interrupt after each handler invocation but only once when the
 1630          * outstanding MSIs have been processed but unfortunately that
 1631          * doesn't work well and leads to interrupt storms with controllers/
 1632          * drivers which don't mask interrupts while the handler is executed.
 1633          * Maybe delaying clearing the MSI until after the handler has been
 1634          * executed could be used to work around this but that's not the
 1635          * intended usage and might in turn cause lost MSIs.
 1636          */
 1637         mtx_lock_spin(&fmqa->fmqa_mtx);
 1638         fire_msiq_common(iv, fmqa);
 1639         mtx_unlock_spin(&fmqa->fmqa_mtx);
 1640 }
 1641 
 1642 static void
 1643 fire_msiq_filter(void *cookie)
 1644 {
 1645         struct intr_vector *iv;
 1646         struct fire_msiqarg *fmqa;
 1647 
 1648         iv = cookie;
 1649         fmqa = iv->iv_icarg;
 1650         /*
 1651          * For filters we don't use fire_intr_clear() since it would clear
 1652          * the event queue interrupt while we're still processing the event
 1653          * queue as filters and associated post-filter handler are executed
 1654          * directly, which in turn would lead to lost MSIs.  So we clear the
 1655          * event queue interrupt only once after processing the event queue.
 1656          * Given that this still guarantees the filters to not be executed
 1657          * concurrently and no other CPU can clear the event queue interrupt
 1658          * while the event queue is still processed, we don't even need to
 1659          * interlock the access to the event queue in this case.
 1660          */
 1661         critical_enter();
 1662         fire_msiq_common(iv, fmqa);
 1663         FIRE_PCI_WRITE_8(fmqa->fmqa_fica.fica_sc, fmqa->fmqa_fica.fica_clr,
 1664             INTCLR_IDLE);
 1665         critical_exit();
 1666 }
 1667 
 1668 static inline void
 1669 fire_msiq_common(struct intr_vector *iv, struct fire_msiqarg *fmqa)
 1670 {
 1671         struct fire_softc *sc;
 1672         struct fo_msiq_record *qrec;
 1673         device_t dev;
 1674         uint64_t word0;
 1675         u_int head, msi, msiq;
 1676 
 1677         sc = fmqa->fmqa_fica.fica_sc;
 1678         dev = sc->sc_dev;
 1679         msiq = fmqa->fmqa_msiq;
 1680         head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >>
 1681             FO_PCI_EQ_HD_SHFT;
 1682         qrec = &fmqa->fmqa_base[head];
 1683         word0 = qrec->fomqr_word0;
 1684         for (;;) {
 1685                 if (__predict_false((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0))
 1686                         break;
 1687                 KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSI64) != 0 ||
 1688                     (word0 & FO_MQR_WORD0_FMT_TYPE_MSI32) != 0,
 1689                     ("%s: received non-MSI/MSI-X message in event queue %d "
 1690                     "(word0 %#llx)", device_get_nameunit(dev), msiq,
 1691                     (unsigned long long)word0));
 1692                 msi = (word0 & FO_MQR_WORD0_DATA0_MASK) >>
 1693                     FO_MQR_WORD0_DATA0_SHFT;
 1694                 /*
 1695                  * Sanity check the MSI/MSI-X as long as we use a 1:1 mapping.
 1696                  */
 1697                 KASSERT(msi == fmqa->fmqa_msi,
 1698                     ("%s: received non-matching MSI/MSI-X in event queue %d "
 1699                     "(%d versus %d)", device_get_nameunit(dev), msiq, msi,
 1700                     fmqa->fmqa_msi));
 1701                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + (msi << 3),
 1702                     FO_PCI_MSI_CLR_EQWR_N);
 1703                 if (__predict_false(intr_event_handle(iv->iv_event,
 1704                     NULL) != 0))
 1705                         printf("stray MSI/MSI-X in event queue %d\n", msiq);
 1706                 qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK;
 1707                 head = (head + 1) % sc->sc_msiq_size;
 1708                 qrec = &fmqa->fmqa_base[head];
 1709                 word0 = qrec->fomqr_word0;
 1710         }
 1711         FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) <<
 1712             FO_PCI_EQ_HD_SHFT);
 1713         if (__predict_false((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) &
 1714             FO_PCI_EQ_TL_OVERR) != 0)) {
 1715                 device_printf(dev, "event queue %d overflow\n", msiq);
 1716                 msiq <<= 3;
 1717                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
 1718                     FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) |
 1719                     FO_PCI_EQ_CTRL_CLR_COVERR);
 1720         }
 1721 }
 1722 
 1723 static int
 1724 fire_setup_intr(device_t dev, device_t child, struct resource *ires,
 1725     int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg,
 1726     void **cookiep)
 1727 {
 1728         struct fire_softc *sc;
 1729         struct fire_msiqarg *fmqa;
 1730         u_long vec;
 1731         int error;
 1732         u_int msi, msiq;
 1733 
 1734         sc = device_get_softc(dev);
 1735         /*
 1736          * XXX this assumes that a device only has one INTx, while in fact
 1737          * Cassini+ and Saturn can use all four the firmware has assigned
 1738          * to them, but so does pci(4).
 1739          */
 1740         if (rman_get_rid(ires) != 0) {
 1741                 msi = rman_get_start(ires);
 1742                 msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first];
 1743                 vec = INTMAP_VEC(sc->sc_ign, sc->sc_msiq_ino_first + msiq);
 1744                 msiq += sc->sc_msiq_first;
 1745                 if (intr_vectors[vec].iv_ic != &fire_ic) {
 1746                         device_printf(dev,
 1747                             "invalid interrupt controller for vector 0x%lx\n",
 1748                             vec);
 1749                         return (EINVAL);
 1750                 }
 1751                 /*
 1752                  * The MD interrupt code needs the vector rather than the MSI.
 1753                  */
 1754                 rman_set_start(ires, vec);
 1755                 rman_set_end(ires, vec);
 1756                 error = bus_generic_setup_intr(dev, child, ires, flags, filt,
 1757                     intr, arg, cookiep);
 1758                 rman_set_start(ires, msi);
 1759                 rman_set_end(ires, msi);
 1760                 if (error != 0)
 1761                         return (error);
 1762                 fmqa = intr_vectors[vec].iv_icarg;
 1763                 /*
 1764                  * XXX inject our event queue handler.
 1765                  */
 1766                 if (filt != NULL) {
 1767                         intr_vectors[vec].iv_func = fire_msiq_filter;
 1768                         intr_vectors[vec].iv_ic = &fire_msiqc_filter;
 1769                         /*
 1770                          * Ensure the event queue interrupt is cleared, it
 1771                          * might have triggered before.  Given we supply NULL
 1772                          * as ic_clear, inthand_add() won't do this for us.
 1773                          */
 1774                         FIRE_PCI_WRITE_8(sc, fmqa->fmqa_fica.fica_clr,
 1775                             INTCLR_IDLE);
 1776                 } else
 1777                         intr_vectors[vec].iv_func = fire_msiq_handler;
 1778                 /* Record the MSI/MSI-X as long as we we use a 1:1 mapping. */
 1779                 fmqa->fmqa_msi = msi;
 1780                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (msiq << 3),
 1781                     FO_PCI_EQ_CTRL_SET_EN);
 1782                 msi <<= 3;
 1783                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
 1784                     (FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) &
 1785                     ~FO_PCI_MSI_MAP_EQNUM_MASK) |
 1786                     ((msiq << FO_PCI_MSI_MAP_EQNUM_SHFT) &
 1787                     FO_PCI_MSI_MAP_EQNUM_MASK));
 1788                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + msi,
 1789                     FO_PCI_MSI_CLR_EQWR_N);
 1790                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
 1791                     FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) |
 1792                     FO_PCI_MSI_MAP_V);
 1793                 return (error);
 1794         }
 1795 
 1796         /*
 1797          * Make sure the vector is fully specified and we registered
 1798          * our interrupt controller for it.
 1799          */
 1800         vec = rman_get_start(ires);
 1801         if (INTIGN(vec) != sc->sc_ign) {
 1802                 device_printf(dev, "invalid interrupt vector 0x%lx\n", vec);
 1803                 return (EINVAL);
 1804         }
 1805         if (intr_vectors[vec].iv_ic != &fire_ic) {
 1806                 device_printf(dev,
 1807                     "invalid interrupt controller for vector 0x%lx\n", vec);
 1808                 return (EINVAL);
 1809         }
 1810         return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr,
 1811             arg, cookiep));
 1812 }
 1813 
 1814 static int
 1815 fire_teardown_intr(device_t dev, device_t child, struct resource *ires,
 1816     void *cookie)
 1817 {
 1818         struct fire_softc *sc;
 1819         u_long vec;
 1820         int error;
 1821         u_int msi, msiq;
 1822 
 1823         sc = device_get_softc(dev);
 1824         if (rman_get_rid(ires) != 0) {
 1825                 msi = rman_get_start(ires);
 1826                 msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first];
 1827                 vec = INTMAP_VEC(sc->sc_ign, msiq + sc->sc_msiq_ino_first);
 1828                 msiq += sc->sc_msiq_first;
 1829                 msi <<= 3;
 1830                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
 1831                     FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) &
 1832                     ~FO_PCI_MSI_MAP_V);
 1833                 msiq <<= 3;
 1834                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
 1835                     FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I |
 1836                     FO_PCI_EQ_CTRL_CLR_DIS);
 1837                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_TL_BASE + msiq,
 1838                     (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK);
 1839                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_HD_BASE + msiq,
 1840                     (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK);
 1841                 intr_vectors[vec].iv_ic = &fire_ic;
 1842                 /*
 1843                  * The MD interrupt code needs the vector rather than the MSI.
 1844                  */
 1845                 rman_set_start(ires, vec);
 1846                 rman_set_end(ires, vec);
 1847                 error = bus_generic_teardown_intr(dev, child, ires, cookie);
 1848                 msi >>= 3;
 1849                 rman_set_start(ires, msi);
 1850                 rman_set_end(ires, msi);
 1851                 return (error);
 1852         }
 1853         return (bus_generic_teardown_intr(dev, child, ires, cookie));
 1854 }
 1855 
 1856 static struct resource *
 1857 fire_alloc_resource(device_t bus, device_t child, int type, int *rid,
 1858     rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
 1859 {
 1860         struct fire_softc *sc;
 1861 
 1862         if (type == SYS_RES_IRQ && *rid == 0) {
 1863                 sc = device_get_softc(bus);
 1864                 start = end = INTMAP_VEC(sc->sc_ign, end);
 1865         }
 1866         return (ofw_pci_alloc_resource(bus, child, type, rid, start, end,
 1867             count, flags));
 1868 }
 1869 
 1870 static u_int
 1871 fire_get_timecount(struct timecounter *tc)
 1872 {
 1873         struct fire_softc *sc;
 1874 
 1875         sc = tc->tc_priv;
 1876         return (FIRE_CTRL_READ_8(sc, FO_XBC_PRF_CNT0) & TC_COUNTER_MAX_MASK);
 1877 }

Cache object: 22b5de5bca79119ce41e74add3171dbe


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.