FreeBSD/Linux Kernel Cross Reference
sys/sparc64/pci/fire.c
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1999, 2000 Matthew R. Green
5 * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org>
6 * Copyright (c) 2009 by Marius Strobl <marius@FreeBSD.org>
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. The name of the author may not be used to endorse or promote products
18 * derived from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp
33 * from: FreeBSD: psycho.c 183152 2008-09-18 19:45:22Z marius
34 */
35
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38
39 /*
40 * Driver for `Fire' JBus to PCI Express and `Oberon' Uranus to PCI Express
41 * bridges
42 */
43
44 #include "opt_fire.h"
45 #include "opt_ofw_pci.h"
46
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/bus.h>
50 #include <sys/interrupt.h>
51 #include <sys/kernel.h>
52 #include <sys/lock.h>
53 #include <sys/malloc.h>
54 #include <sys/module.h>
55 #include <sys/mutex.h>
56 #include <sys/pciio.h>
57 #include <sys/pcpu.h>
58 #include <sys/rman.h>
59 #include <sys/smp.h>
60 #include <sys/sysctl.h>
61 #include <sys/timetc.h>
62
63 #include <dev/ofw/ofw_bus.h>
64 #include <dev/ofw/openfirm.h>
65
66 #include <vm/vm.h>
67 #include <vm/pmap.h>
68
69 #include <machine/bus.h>
70 #include <machine/bus_common.h>
71 #include <machine/bus_private.h>
72 #include <machine/iommureg.h>
73 #include <machine/iommuvar.h>
74 #include <machine/resource.h>
75
76 #include <dev/pci/pcireg.h>
77 #include <dev/pci/pcivar.h>
78 #include <dev/pci/pcib_private.h>
79
80 #include <sparc64/pci/ofw_pci.h>
81 #include <sparc64/pci/firereg.h>
82 #include <sparc64/pci/firevar.h>
83
84 #include "pcib_if.h"
85
86 struct fire_msiqarg;
87
88 static const struct fire_desc *fire_get_desc(device_t dev);
89 static void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
90 bus_dmasync_op_t op);
91 static int fire_get_intrmap(struct fire_softc *sc, u_int ino,
92 bus_addr_t *intrmapptr, bus_addr_t *intrclrptr);
93 static void fire_intr_assign(void *arg);
94 static void fire_intr_clear(void *arg);
95 static void fire_intr_disable(void *arg);
96 static void fire_intr_enable(void *arg);
97 static int fire_intr_register(struct fire_softc *sc, u_int ino);
98 static inline void fire_msiq_common(struct intr_vector *iv,
99 struct fire_msiqarg *fmqa);
100 static void fire_msiq_filter(void *cookie);
101 static void fire_msiq_handler(void *cookie);
102 static void fire_set_intr(struct fire_softc *sc, u_int index, u_int ino,
103 driver_filter_t handler, void *arg);
104 static timecounter_get_t fire_get_timecount;
105
106 /* Interrupt handlers */
107 static driver_filter_t fire_dmc_pec;
108 static driver_filter_t fire_pcie;
109 static driver_filter_t fire_xcb;
110
111 /*
112 * Methods
113 */
114 static pcib_alloc_msi_t fire_alloc_msi;
115 static pcib_alloc_msix_t fire_alloc_msix;
116 static bus_alloc_resource_t fire_alloc_resource;
117 static device_attach_t fire_attach;
118 static pcib_map_msi_t fire_map_msi;
119 static pcib_maxslots_t fire_maxslots;
120 static device_probe_t fire_probe;
121 static pcib_read_config_t fire_read_config;
122 static pcib_release_msi_t fire_release_msi;
123 static pcib_release_msix_t fire_release_msix;
124 static pcib_route_interrupt_t fire_route_interrupt;
125 static bus_setup_intr_t fire_setup_intr;
126 static bus_teardown_intr_t fire_teardown_intr;
127 static pcib_write_config_t fire_write_config;
128
129 static device_method_t fire_methods[] = {
130 /* Device interface */
131 DEVMETHOD(device_probe, fire_probe),
132 DEVMETHOD(device_attach, fire_attach),
133 DEVMETHOD(device_shutdown, bus_generic_shutdown),
134 DEVMETHOD(device_suspend, bus_generic_suspend),
135 DEVMETHOD(device_resume, bus_generic_resume),
136
137 /* Bus interface */
138 DEVMETHOD(bus_read_ivar, ofw_pci_read_ivar),
139 DEVMETHOD(bus_setup_intr, fire_setup_intr),
140 DEVMETHOD(bus_teardown_intr, fire_teardown_intr),
141 DEVMETHOD(bus_alloc_resource, fire_alloc_resource),
142 DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource),
143 DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
144 DEVMETHOD(bus_adjust_resource, ofw_pci_adjust_resource),
145 DEVMETHOD(bus_release_resource, bus_generic_release_resource),
146 DEVMETHOD(bus_get_dma_tag, ofw_pci_get_dma_tag),
147
148 /* pcib interface */
149 DEVMETHOD(pcib_maxslots, fire_maxslots),
150 DEVMETHOD(pcib_read_config, fire_read_config),
151 DEVMETHOD(pcib_write_config, fire_write_config),
152 DEVMETHOD(pcib_route_interrupt, fire_route_interrupt),
153 DEVMETHOD(pcib_alloc_msi, fire_alloc_msi),
154 DEVMETHOD(pcib_release_msi, fire_release_msi),
155 DEVMETHOD(pcib_alloc_msix, fire_alloc_msix),
156 DEVMETHOD(pcib_release_msix, fire_release_msix),
157 DEVMETHOD(pcib_map_msi, fire_map_msi),
158 DEVMETHOD(pcib_request_feature, pcib_request_feature_allow),
159
160 /* ofw_bus interface */
161 DEVMETHOD(ofw_bus_get_node, ofw_pci_get_node),
162
163 DEVMETHOD_END
164 };
165
166 static devclass_t fire_devclass;
167
168 DEFINE_CLASS_0(pcib, fire_driver, fire_methods, sizeof(struct fire_softc));
169 EARLY_DRIVER_MODULE(fire, nexus, fire_driver, fire_devclass, 0, 0,
170 BUS_PASS_BUS);
171 MODULE_DEPEND(fire, nexus, 1, 1, 1);
172
173 static const struct intr_controller fire_ic = {
174 fire_intr_enable,
175 fire_intr_disable,
176 fire_intr_assign,
177 fire_intr_clear
178 };
179
180 struct fire_icarg {
181 struct fire_softc *fica_sc;
182 bus_addr_t fica_map;
183 bus_addr_t fica_clr;
184 };
185
186 static const struct intr_controller fire_msiqc_filter = {
187 fire_intr_enable,
188 fire_intr_disable,
189 fire_intr_assign,
190 NULL
191 };
192
193 struct fire_msiqarg {
194 struct fire_icarg fmqa_fica;
195 struct mtx fmqa_mtx;
196 struct fo_msiq_record *fmqa_base;
197 uint64_t fmqa_head;
198 uint64_t fmqa_tail;
199 uint32_t fmqa_msiq;
200 uint32_t fmqa_msi;
201 };
202
203 #define FIRE_PERF_CNT_QLTY 100
204
205 #define FIRE_SPC_BARRIER(spc, sc, offs, len, flags) \
206 bus_barrier((sc)->sc_mem_res[(spc)], (offs), (len), (flags))
207 #define FIRE_SPC_READ_8(spc, sc, offs) \
208 bus_read_8((sc)->sc_mem_res[(spc)], (offs))
209 #define FIRE_SPC_WRITE_8(spc, sc, offs, v) \
210 bus_write_8((sc)->sc_mem_res[(spc)], (offs), (v))
211
212 #ifndef FIRE_DEBUG
213 #define FIRE_SPC_SET(spc, sc, offs, reg, v) \
214 FIRE_SPC_WRITE_8((spc), (sc), (offs), (v))
215 #else
216 #define FIRE_SPC_SET(spc, sc, offs, reg, v) do { \
217 device_printf((sc)->sc_dev, reg " 0x%016llx -> 0x%016llx\n", \
218 (unsigned long long)FIRE_SPC_READ_8((spc), (sc), (offs)), \
219 (unsigned long long)(v)); \
220 FIRE_SPC_WRITE_8((spc), (sc), (offs), (v)); \
221 } while (0)
222 #endif
223
224 #define FIRE_PCI_BARRIER(sc, offs, len, flags) \
225 FIRE_SPC_BARRIER(FIRE_PCI, (sc), (offs), len, flags)
226 #define FIRE_PCI_READ_8(sc, offs) \
227 FIRE_SPC_READ_8(FIRE_PCI, (sc), (offs))
228 #define FIRE_PCI_WRITE_8(sc, offs, v) \
229 FIRE_SPC_WRITE_8(FIRE_PCI, (sc), (offs), (v))
230 #define FIRE_CTRL_BARRIER(sc, offs, len, flags) \
231 FIRE_SPC_BARRIER(FIRE_CTRL, (sc), (offs), len, flags)
232 #define FIRE_CTRL_READ_8(sc, offs) \
233 FIRE_SPC_READ_8(FIRE_CTRL, (sc), (offs))
234 #define FIRE_CTRL_WRITE_8(sc, offs, v) \
235 FIRE_SPC_WRITE_8(FIRE_CTRL, (sc), (offs), (v))
236
237 #define FIRE_PCI_SET(sc, offs, v) \
238 FIRE_SPC_SET(FIRE_PCI, (sc), (offs), # offs, (v))
239 #define FIRE_CTRL_SET(sc, offs, v) \
240 FIRE_SPC_SET(FIRE_CTRL, (sc), (offs), # offs, (v))
241
242 struct fire_desc {
243 const char *fd_string;
244 int fd_mode;
245 const char *fd_name;
246 };
247
248 static const struct fire_desc fire_compats[] = {
249 { "pciex108e,80f0", FIRE_MODE_FIRE, "Fire" },
250 #if 0
251 { "pciex108e,80f8", FIRE_MODE_OBERON, "Oberon" },
252 #endif
253 { NULL, 0, NULL }
254 };
255
256 static const struct fire_desc *
257 fire_get_desc(device_t dev)
258 {
259 const struct fire_desc *desc;
260 const char *compat;
261
262 compat = ofw_bus_get_compat(dev);
263 if (compat == NULL)
264 return (NULL);
265 for (desc = fire_compats; desc->fd_string != NULL; desc++)
266 if (strcmp(desc->fd_string, compat) == 0)
267 return (desc);
268 return (NULL);
269 }
270
271 static int
272 fire_probe(device_t dev)
273 {
274 const char *dtype;
275
276 dtype = ofw_bus_get_type(dev);
277 if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCIE) == 0 &&
278 fire_get_desc(dev) != NULL) {
279 device_set_desc(dev, "Sun Host-PCIe bridge");
280 return (BUS_PROBE_GENERIC);
281 }
282 return (ENXIO);
283 }
284
285 static int
286 fire_attach(device_t dev)
287 {
288 struct fire_softc *sc;
289 const struct fire_desc *desc;
290 struct ofw_pci_msi_ranges msi_ranges;
291 struct ofw_pci_msi_addr_ranges msi_addr_ranges;
292 struct ofw_pci_msi_eq_to_devino msi_eq_to_devino;
293 struct fire_msiqarg *fmqa;
294 struct timecounter *tc;
295 bus_dma_tag_t dmat;
296 uint64_t ino_bitmap, val;
297 phandle_t node;
298 uint32_t prop, prop_array[2];
299 int i, j, mode;
300 u_int lw;
301 uint16_t mps;
302
303 sc = device_get_softc(dev);
304 node = ofw_bus_get_node(dev);
305 desc = fire_get_desc(dev);
306 mode = desc->fd_mode;
307
308 sc->sc_dev = dev;
309 sc->sc_mode = mode;
310 sc->sc_flags = 0;
311
312 mtx_init(&sc->sc_msi_mtx, "msi_mtx", NULL, MTX_DEF);
313 mtx_init(&sc->sc_pcib_mtx, "pcib_mtx", NULL, MTX_SPIN);
314
315 /*
316 * Fire and Oberon have two register banks:
317 * (0) per-PBM PCI Express configuration and status registers
318 * (1) (shared) Fire/Oberon controller configuration and status
319 * registers
320 */
321 for (i = 0; i < FIRE_NREG; i++) {
322 j = i;
323 sc->sc_mem_res[i] = bus_alloc_resource_any(dev,
324 SYS_RES_MEMORY, &j, RF_ACTIVE);
325 if (sc->sc_mem_res[i] == NULL)
326 panic("%s: could not allocate register bank %d",
327 __func__, i);
328 }
329
330 if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1)
331 panic("%s: could not determine IGN", __func__);
332 if (OF_getprop(node, "module-revision#", &prop, sizeof(prop)) == -1)
333 panic("%s: could not determine module-revision", __func__);
334
335 device_printf(dev, "%s, module-revision %d, IGN %#x\n",
336 desc->fd_name, prop, sc->sc_ign);
337
338 /*
339 * Hunt through all the interrupt mapping regs and register
340 * the interrupt controller for our interrupt vectors. We do
341 * this early in order to be able to catch stray interrupts.
342 */
343 i = OF_getprop(node, "ino-bitmap", (void *)prop_array,
344 sizeof(prop_array));
345 if (i == -1)
346 panic("%s: could not get ino-bitmap", __func__);
347 ino_bitmap = ((uint64_t)prop_array[1] << 32) | prop_array[0];
348 for (i = 0; i <= FO_MAX_INO; i++) {
349 if ((ino_bitmap & (1ULL << i)) == 0)
350 continue;
351 j = fire_intr_register(sc, i);
352 if (j != 0)
353 device_printf(dev, "could not register interrupt "
354 "controller for INO %d (%d)\n", i, j);
355 }
356
357 /* JBC/UBC module initialization */
358 FIRE_CTRL_SET(sc, FO_XBC_ERR_LOG_EN, ~0ULL);
359 FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
360 /* not enabled by OpenSolaris */
361 FIRE_CTRL_SET(sc, FO_XBC_INT_EN, ~0ULL);
362 if (sc->sc_mode == FIRE_MODE_FIRE) {
363 FIRE_CTRL_SET(sc, FIRE_JBUS_PAR_CTRL,
364 FIRE_JBUS_PAR_CTRL_P_EN);
365 FIRE_CTRL_SET(sc, FIRE_JBC_FATAL_RST_EN,
366 ((1ULL << FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_SHFT) &
367 FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_MASK) |
368 FIRE_JBC_FATAL_RST_EN_MB_PEA_P_INT |
369 FIRE_JBC_FATAL_RST_EN_CPE_P_INT |
370 FIRE_JBC_FATAL_RST_EN_APE_P_INT |
371 FIRE_JBC_FATAL_RST_EN_PIO_CPE_INT |
372 FIRE_JBC_FATAL_RST_EN_JTCEEW_P_INT |
373 FIRE_JBC_FATAL_RST_EN_JTCEEI_P_INT |
374 FIRE_JBC_FATAL_RST_EN_JTCEER_P_INT);
375 FIRE_CTRL_SET(sc, FIRE_JBC_CORE_BLOCK_INT_EN, ~0ULL);
376 }
377
378 /* TLU initialization */
379 FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_STAT_CLR,
380 FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK);
381 /* not enabled by OpenSolaris */
382 FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_INT_EN,
383 FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK);
384 FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_STAT_CLR,
385 FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK);
386 /* not enabled by OpenSolaris */
387 FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_INT_EN,
388 FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK);
389 FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_STAT_CLR,
390 FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK);
391 /* not enabled by OpenSolaris */
392 FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_INT_EN,
393 FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK);
394 val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) |
395 ((FO_PCI_TLU_CTRL_L0S_TIM_DFLT << FO_PCI_TLU_CTRL_L0S_TIM_SHFT) &
396 FO_PCI_TLU_CTRL_L0S_TIM_MASK) |
397 ((FO_PCI_TLU_CTRL_CFG_DFLT << FO_PCI_TLU_CTRL_CFG_SHFT) &
398 FO_PCI_TLU_CTRL_CFG_MASK);
399 if (sc->sc_mode == FIRE_MODE_OBERON)
400 val &= ~FO_PCI_TLU_CTRL_NWPR_EN;
401 val |= FO_PCI_TLU_CTRL_CFG_REMAIN_DETECT_QUIET;
402 FIRE_PCI_SET(sc, FO_PCI_TLU_CTRL, val);
403 FIRE_PCI_SET(sc, FO_PCI_TLU_DEV_CTRL, 0);
404 FIRE_PCI_SET(sc, FO_PCI_TLU_LNK_CTRL, FO_PCI_TLU_LNK_CTRL_CLK);
405
406 /* DLU/LPU initialization */
407 if (sc->sc_mode == FIRE_MODE_OBERON)
408 FIRE_PCI_SET(sc, FO_PCI_LPU_INT_MASK, 0);
409 else
410 FIRE_PCI_SET(sc, FO_PCI_LPU_RST, 0);
411 FIRE_PCI_SET(sc, FO_PCI_LPU_LNK_LYR_CFG,
412 FO_PCI_LPU_LNK_LYR_CFG_VC0_EN);
413 FIRE_PCI_SET(sc, FO_PCI_LPU_FLW_CTRL_UPDT_CTRL,
414 FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_NP_EN |
415 FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_P_EN);
416 if (sc->sc_mode == FIRE_MODE_OBERON)
417 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS,
418 (OBERON_PCI_LPU_TXLNK_RPLY_TMR_THRS_DFLT <<
419 FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) &
420 FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK);
421 else {
422 switch ((FIRE_PCI_READ_8(sc, FO_PCI_TLU_LNK_STAT) &
423 FO_PCI_TLU_LNK_STAT_WDTH_MASK) >>
424 FO_PCI_TLU_LNK_STAT_WDTH_SHFT) {
425 case 1:
426 lw = 0;
427 break;
428 case 4:
429 lw = 1;
430 break;
431 case 8:
432 lw = 2;
433 break;
434 case 16:
435 lw = 3;
436 break;
437 default:
438 lw = 0;
439 }
440 mps = (FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) &
441 FO_PCI_TLU_CTRL_CFG_MPS_MASK) >>
442 FO_PCI_TLU_CTRL_CFG_MPS_SHFT;
443 i = sizeof(fire_freq_nak_tmr_thrs) /
444 sizeof(*fire_freq_nak_tmr_thrs);
445 if (mps >= i)
446 mps = i - 1;
447 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS,
448 (fire_freq_nak_tmr_thrs[mps][lw] <<
449 FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_SHFT) &
450 FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_MASK);
451 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS,
452 (fire_rply_tmr_thrs[mps][lw] <<
453 FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) &
454 FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK);
455 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RTR_FIFO_PTR,
456 ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_DFLT <<
457 FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_SHFT) &
458 FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_MASK) |
459 ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_DFLT <<
460 FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_SHFT) &
461 FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_MASK));
462 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG2,
463 (FO_PCI_LPU_LTSSM_CFG2_12_TO_DFLT <<
464 FO_PCI_LPU_LTSSM_CFG2_12_TO_SHFT) &
465 FO_PCI_LPU_LTSSM_CFG2_12_TO_MASK);
466 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG3,
467 (FO_PCI_LPU_LTSSM_CFG3_2_TO_DFLT <<
468 FO_PCI_LPU_LTSSM_CFG3_2_TO_SHFT) &
469 FO_PCI_LPU_LTSSM_CFG3_2_TO_MASK);
470 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG4,
471 ((FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_DFLT <<
472 FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_SHFT) &
473 FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_MASK) |
474 ((FO_PCI_LPU_LTSSM_CFG4_N_FTS_DFLT <<
475 FO_PCI_LPU_LTSSM_CFG4_N_FTS_SHFT) &
476 FO_PCI_LPU_LTSSM_CFG4_N_FTS_MASK));
477 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG5, 0);
478 }
479
480 /* ILU initialization */
481 FIRE_PCI_SET(sc, FO_PCI_ILU_ERR_STAT_CLR, ~0ULL);
482 /* not enabled by OpenSolaris */
483 FIRE_PCI_SET(sc, FO_PCI_ILU_INT_EN, ~0ULL);
484
485 /* IMU initialization */
486 FIRE_PCI_SET(sc, FO_PCI_IMU_ERR_STAT_CLR, ~0ULL);
487 FIRE_PCI_SET(sc, FO_PCI_IMU_INT_EN,
488 FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_EN) &
489 ~(FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_S |
490 FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_S |
491 FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_S |
492 FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P |
493 FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P |
494 FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P));
495
496 /* MMU initialization */
497 FIRE_PCI_SET(sc, FO_PCI_MMU_ERR_STAT_CLR,
498 FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK);
499 /* not enabled by OpenSolaris */
500 FIRE_PCI_SET(sc, FO_PCI_MMU_INT_EN,
501 FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK);
502
503 /* DMC initialization */
504 FIRE_PCI_SET(sc, FO_PCI_DMC_CORE_BLOCK_INT_EN, ~0ULL);
505 FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTA, 0);
506 FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTB, 0);
507
508 /* PEC initialization */
509 FIRE_PCI_SET(sc, FO_PCI_PEC_CORE_BLOCK_INT_EN, ~0ULL);
510
511 /* Establish handlers for interesting interrupts. */
512 if ((ino_bitmap & (1ULL << FO_DMC_PEC_INO)) != 0)
513 fire_set_intr(sc, 1, FO_DMC_PEC_INO, fire_dmc_pec, sc);
514 if ((ino_bitmap & (1ULL << FO_XCB_INO)) != 0)
515 fire_set_intr(sc, 0, FO_XCB_INO, fire_xcb, sc);
516
517 /* MSI/MSI-X support */
518 if (OF_getprop(node, "#msi", &sc->sc_msi_count,
519 sizeof(sc->sc_msi_count)) == -1)
520 panic("%s: could not determine MSI count", __func__);
521 if (OF_getprop(node, "msi-ranges", &msi_ranges,
522 sizeof(msi_ranges)) == -1)
523 sc->sc_msi_first = 0;
524 else
525 sc->sc_msi_first = msi_ranges.first;
526 if (OF_getprop(node, "msi-data-mask", &sc->sc_msi_data_mask,
527 sizeof(sc->sc_msi_data_mask)) == -1)
528 panic("%s: could not determine MSI data mask", __func__);
529 if (OF_getprop(node, "msix-data-width", &sc->sc_msix_data_width,
530 sizeof(sc->sc_msix_data_width)) > 0)
531 sc->sc_flags |= FIRE_MSIX;
532 if (OF_getprop(node, "msi-address-ranges", &msi_addr_ranges,
533 sizeof(msi_addr_ranges)) == -1)
534 panic("%s: could not determine MSI address ranges", __func__);
535 sc->sc_msi_addr32 = OFW_PCI_MSI_ADDR_RANGE_32(&msi_addr_ranges);
536 sc->sc_msi_addr64 = OFW_PCI_MSI_ADDR_RANGE_64(&msi_addr_ranges);
537 if (OF_getprop(node, "#msi-eqs", &sc->sc_msiq_count,
538 sizeof(sc->sc_msiq_count)) == -1)
539 panic("%s: could not determine MSI event queue count",
540 __func__);
541 if (OF_getprop(node, "msi-eq-size", &sc->sc_msiq_size,
542 sizeof(sc->sc_msiq_size)) == -1)
543 panic("%s: could not determine MSI event queue size",
544 __func__);
545 if (OF_getprop(node, "msi-eq-to-devino", &msi_eq_to_devino,
546 sizeof(msi_eq_to_devino)) == -1 &&
547 OF_getprop(node, "msi-eq-devino", &msi_eq_to_devino,
548 sizeof(msi_eq_to_devino)) == -1) {
549 sc->sc_msiq_first = 0;
550 sc->sc_msiq_ino_first = FO_EQ_FIRST_INO;
551 } else {
552 sc->sc_msiq_first = msi_eq_to_devino.eq_first;
553 sc->sc_msiq_ino_first = msi_eq_to_devino.devino_first;
554 }
555 if (sc->sc_msiq_ino_first < FO_EQ_FIRST_INO ||
556 sc->sc_msiq_ino_first + sc->sc_msiq_count - 1 > FO_EQ_LAST_INO)
557 panic("%s: event queues exceed INO range", __func__);
558 sc->sc_msi_bitmap = malloc(roundup2(sc->sc_msi_count, NBBY) / NBBY,
559 M_DEVBUF, M_NOWAIT | M_ZERO);
560 if (sc->sc_msi_bitmap == NULL)
561 panic("%s: could not malloc MSI bitmap", __func__);
562 sc->sc_msi_msiq_table = malloc(sc->sc_msi_count *
563 sizeof(*sc->sc_msi_msiq_table), M_DEVBUF, M_NOWAIT | M_ZERO);
564 if (sc->sc_msi_msiq_table == NULL)
565 panic("%s: could not malloc MSI-MSI event queue table",
566 __func__);
567 sc->sc_msiq_bitmap = malloc(roundup2(sc->sc_msiq_count, NBBY) / NBBY,
568 M_DEVBUF, M_NOWAIT | M_ZERO);
569 if (sc->sc_msiq_bitmap == NULL)
570 panic("%s: could not malloc MSI event queue bitmap", __func__);
571 j = FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * sc->sc_msiq_count;
572 sc->sc_msiq = contigmalloc(j, M_DEVBUF, M_NOWAIT, 0, ~0UL,
573 FO_EQ_ALIGNMENT, 0);
574 if (sc->sc_msiq == NULL)
575 panic("%s: could not contigmalloc MSI event queue", __func__);
576 memset(sc->sc_msiq, 0, j);
577 FIRE_PCI_SET(sc, FO_PCI_EQ_BASE_ADDR, FO_PCI_EQ_BASE_ADDR_BYPASS |
578 (pmap_kextract((vm_offset_t)sc->sc_msiq) &
579 FO_PCI_EQ_BASE_ADDR_MASK));
580 for (i = 0; i < sc->sc_msi_count; i++) {
581 j = (i + sc->sc_msi_first) << 3;
582 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + j,
583 FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + j) &
584 ~FO_PCI_MSI_MAP_V);
585 }
586 for (i = 0; i < sc->sc_msiq_count; i++) {
587 j = i + sc->sc_msiq_ino_first;
588 if ((ino_bitmap & (1ULL << j)) == 0) {
589 mtx_lock(&sc->sc_msi_mtx);
590 setbit(sc->sc_msiq_bitmap, i);
591 mtx_unlock(&sc->sc_msi_mtx);
592 }
593 fmqa = intr_vectors[INTMAP_VEC(sc->sc_ign, j)].iv_icarg;
594 mtx_init(&fmqa->fmqa_mtx, "msiq_mtx", NULL, MTX_SPIN);
595 fmqa->fmqa_base =
596 (struct fo_msiq_record *)((caddr_t)sc->sc_msiq +
597 (FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * i));
598 j = i + sc->sc_msiq_first;
599 fmqa->fmqa_msiq = j;
600 j <<= 3;
601 fmqa->fmqa_head = FO_PCI_EQ_HD_BASE + j;
602 fmqa->fmqa_tail = FO_PCI_EQ_TL_BASE + j;
603 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + j,
604 FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I |
605 FO_PCI_EQ_CTRL_CLR_DIS);
606 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_tail,
607 (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK);
608 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head,
609 (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK);
610 }
611 FIRE_PCI_SET(sc, FO_PCI_MSI_32_BIT_ADDR, sc->sc_msi_addr32 &
612 FO_PCI_MSI_32_BIT_ADDR_MASK);
613 FIRE_PCI_SET(sc, FO_PCI_MSI_64_BIT_ADDR, sc->sc_msi_addr64 &
614 FO_PCI_MSI_64_BIT_ADDR_MASK);
615
616 /*
617 * Establish a handler for interesting PCIe messages and disable
618 * unintersting ones.
619 */
620 mtx_lock(&sc->sc_msi_mtx);
621 for (i = 0; i < sc->sc_msiq_count; i++) {
622 if (isclr(sc->sc_msiq_bitmap, i) != 0) {
623 j = i;
624 break;
625 }
626 }
627 if (i == sc->sc_msiq_count) {
628 mtx_unlock(&sc->sc_msi_mtx);
629 panic("%s: no spare event queue for PCIe messages", __func__);
630 }
631 setbit(sc->sc_msiq_bitmap, j);
632 mtx_unlock(&sc->sc_msi_mtx);
633 i = INTMAP_VEC(sc->sc_ign, j + sc->sc_msiq_ino_first);
634 if (bus_set_resource(dev, SYS_RES_IRQ, 2, i, 1) != 0)
635 panic("%s: failed to add interrupt for PCIe messages",
636 __func__);
637 fire_set_intr(sc, 2, INTINO(i), fire_pcie, intr_vectors[i].iv_icarg);
638 j += sc->sc_msiq_first;
639 /*
640 * "Please note that setting the EQNUM field to a value larger than
641 * 35 will yield unpredictable results."
642 */
643 if (j > 35)
644 panic("%s: invalid queue for PCIe messages (%d)",
645 __func__, j);
646 FIRE_PCI_SET(sc, FO_PCI_ERR_COR, FO_PCI_ERR_PME_V |
647 ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
648 FIRE_PCI_SET(sc, FO_PCI_ERR_NONFATAL, FO_PCI_ERR_PME_V |
649 ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
650 FIRE_PCI_SET(sc, FO_PCI_ERR_FATAL, FO_PCI_ERR_PME_V |
651 ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
652 FIRE_PCI_SET(sc, FO_PCI_PM_PME, 0);
653 FIRE_PCI_SET(sc, FO_PCI_PME_TO_ACK, 0);
654 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (j << 3),
655 FO_PCI_EQ_CTRL_SET_EN);
656
657 #define TC_COUNTER_MAX_MASK 0xffffffff
658
659 /*
660 * Setup JBC/UBC performance counter 0 in bus cycle counting
661 * mode as timecounter.
662 */
663 if (device_get_unit(dev) == 0) {
664 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT0, 0);
665 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT1, 0);
666 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT_SEL,
667 (FO_XBC_PRF_CNT_NONE << FO_XBC_PRF_CNT_CNT1_SHFT) |
668 (FO_XBC_PRF_CNT_XB_CLK << FO_XBC_PRF_CNT_CNT0_SHFT));
669 tc = malloc(sizeof(*tc), M_DEVBUF, M_NOWAIT | M_ZERO);
670 if (tc == NULL)
671 panic("%s: could not malloc timecounter", __func__);
672 tc->tc_get_timecount = fire_get_timecount;
673 tc->tc_counter_mask = TC_COUNTER_MAX_MASK;
674 if (OF_getprop(OF_peer(0), "clock-frequency", &prop,
675 sizeof(prop)) == -1)
676 panic("%s: could not determine clock frequency",
677 __func__);
678 tc->tc_frequency = prop;
679 tc->tc_name = strdup(device_get_nameunit(dev), M_DEVBUF);
680 tc->tc_priv = sc;
681 /*
682 * Due to initial problems with the JBus-driven performance
683 * counters not advancing which might be firmware dependent
684 * ensure that it actually works.
685 */
686 if (fire_get_timecount(tc) - fire_get_timecount(tc) != 0)
687 tc->tc_quality = FIRE_PERF_CNT_QLTY;
688 else
689 tc->tc_quality = -FIRE_PERF_CNT_QLTY;
690 tc_init(tc);
691 }
692
693 /*
694 * Set up the IOMMU. Both Fire and Oberon have one per PBM, but
695 * neither has a streaming buffer.
696 */
697 memcpy(&sc->sc_dma_methods, &iommu_dma_methods,
698 sizeof(sc->sc_dma_methods));
699 sc->sc_is.is_flags = IOMMU_FIRE | IOMMU_PRESERVE_PROM;
700 if (sc->sc_mode == FIRE_MODE_OBERON) {
701 sc->sc_is.is_flags |= IOMMU_FLUSH_CACHE;
702 sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(OBERON_IOMMU_BITS);
703 } else {
704 sc->sc_dma_methods.dm_dmamap_sync = fire_dmamap_sync;
705 sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(FIRE_IOMMU_BITS);
706 }
707 sc->sc_is.is_sb[0] = sc->sc_is.is_sb[1] = 0;
708 /* Punch in our copies. */
709 sc->sc_is.is_bustag = rman_get_bustag(sc->sc_mem_res[FIRE_PCI]);
710 sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[FIRE_PCI]);
711 sc->sc_is.is_iommu = FO_PCI_MMU;
712 val = FIRE_PCI_READ_8(sc, FO_PCI_MMU + IMR_CTL);
713 iommu_init(device_get_nameunit(dev), &sc->sc_is, 7, -1, 0);
714 #ifdef FIRE_DEBUG
715 device_printf(dev, "FO_PCI_MMU + IMR_CTL 0x%016llx -> 0x%016llx\n",
716 (long long unsigned)val, (long long unsigned)sc->sc_is.is_cr);
717 #endif
718 /* Create our DMA tag. */
719 if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0x100000000,
720 sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr,
721 0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0)
722 panic("%s: could not create PCI DMA tag", __func__);
723 dmat->dt_cookie = &sc->sc_is;
724 dmat->dt_mt = &sc->sc_dma_methods;
725
726 if (ofw_pci_attach_common(dev, dmat, FO_IO_SIZE, FO_MEM_SIZE) != 0)
727 panic("%s: ofw_pci_attach_common() failed", __func__);
728
729 #define FIRE_SYSCTL_ADD_UINT(name, arg, desc) \
730 SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), \
731 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, \
732 (name), CTLFLAG_RD, (arg), 0, (desc))
733
734 FIRE_SYSCTL_ADD_UINT("ilu_err", &sc->sc_stats_ilu_err,
735 "ILU unknown errors");
736 FIRE_SYSCTL_ADD_UINT("jbc_ce_async", &sc->sc_stats_jbc_ce_async,
737 "JBC correctable errors");
738 FIRE_SYSCTL_ADD_UINT("jbc_unsol_int", &sc->sc_stats_jbc_unsol_int,
739 "JBC unsolicited interrupt ACK/NACK errors");
740 FIRE_SYSCTL_ADD_UINT("jbc_unsol_rd", &sc->sc_stats_jbc_unsol_rd,
741 "JBC unsolicited read response errors");
742 FIRE_SYSCTL_ADD_UINT("mmu_err", &sc->sc_stats_mmu_err, "MMU errors");
743 FIRE_SYSCTL_ADD_UINT("tlu_ce", &sc->sc_stats_tlu_ce,
744 "DLU/TLU correctable errors");
745 FIRE_SYSCTL_ADD_UINT("tlu_oe_non_fatal",
746 &sc->sc_stats_tlu_oe_non_fatal,
747 "DLU/TLU other event non-fatal errors summary");
748 FIRE_SYSCTL_ADD_UINT("tlu_oe_rx_err", &sc->sc_stats_tlu_oe_rx_err,
749 "DLU/TLU receive other event errors");
750 FIRE_SYSCTL_ADD_UINT("tlu_oe_tx_err", &sc->sc_stats_tlu_oe_tx_err,
751 "DLU/TLU transmit other event errors");
752 FIRE_SYSCTL_ADD_UINT("ubc_dmardue", &sc->sc_stats_ubc_dmardue,
753 "UBC DMARDUE erros");
754
755 #undef FIRE_SYSCTL_ADD_UINT
756
757 device_add_child(dev, "pci", -1);
758 return (bus_generic_attach(dev));
759 }
760
761 static void
762 fire_set_intr(struct fire_softc *sc, u_int index, u_int ino,
763 driver_filter_t handler, void *arg)
764 {
765 u_long vec;
766 int rid;
767
768 rid = index;
769 sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev,
770 SYS_RES_IRQ, &rid, RF_ACTIVE);
771 if (sc->sc_irq_res[index] == NULL ||
772 INTINO(vec = rman_get_start(sc->sc_irq_res[index])) != ino ||
773 INTIGN(vec) != sc->sc_ign ||
774 intr_vectors[vec].iv_ic != &fire_ic ||
775 bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index],
776 INTR_TYPE_MISC | INTR_BRIDGE, handler, NULL, arg,
777 &sc->sc_ihand[index]) != 0)
778 panic("%s: failed to set up interrupt %d", __func__, index);
779 }
780
781 static int
782 fire_intr_register(struct fire_softc *sc, u_int ino)
783 {
784 struct fire_icarg *fica;
785 bus_addr_t intrclr, intrmap;
786 int error;
787
788 if (fire_get_intrmap(sc, ino, &intrmap, &intrclr) == 0)
789 return (ENXIO);
790 fica = malloc((ino >= FO_EQ_FIRST_INO && ino <= FO_EQ_LAST_INO) ?
791 sizeof(struct fire_msiqarg) : sizeof(struct fire_icarg), M_DEVBUF,
792 M_NOWAIT | M_ZERO);
793 if (fica == NULL)
794 return (ENOMEM);
795 fica->fica_sc = sc;
796 fica->fica_map = intrmap;
797 fica->fica_clr = intrclr;
798 error = (intr_controller_register(INTMAP_VEC(sc->sc_ign, ino),
799 &fire_ic, fica));
800 if (error != 0)
801 free(fica, M_DEVBUF);
802 return (error);
803 }
804
805 static int
806 fire_get_intrmap(struct fire_softc *sc, u_int ino, bus_addr_t *intrmapptr,
807 bus_addr_t *intrclrptr)
808 {
809
810 if (ino > FO_MAX_INO) {
811 device_printf(sc->sc_dev, "out of range INO %d requested\n",
812 ino);
813 return (0);
814 }
815
816 ino <<= 3;
817 if (intrmapptr != NULL)
818 *intrmapptr = FO_PCI_INT_MAP_BASE + ino;
819 if (intrclrptr != NULL)
820 *intrclrptr = FO_PCI_INT_CLR_BASE + ino;
821 return (1);
822 }
823
824 /*
825 * Interrupt handlers
826 */
827 static int
828 fire_dmc_pec(void *arg)
829 {
830 struct fire_softc *sc;
831 device_t dev;
832 uint64_t cestat, dmcstat, ilustat, imustat, mcstat, mmustat, mmutfar;
833 uint64_t mmutfsr, oestat, pecstat, uestat, val;
834 u_int fatal, oenfatal;
835
836 fatal = 0;
837 sc = arg;
838 dev = sc->sc_dev;
839 mtx_lock_spin(&sc->sc_pcib_mtx);
840 mcstat = FIRE_PCI_READ_8(sc, FO_PCI_MULTI_CORE_ERR_STAT);
841 if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_DMC) != 0) {
842 dmcstat = FIRE_PCI_READ_8(sc, FO_PCI_DMC_CORE_BLOCK_ERR_STAT);
843 if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_IMU) != 0) {
844 imustat = FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_STAT);
845 device_printf(dev, "IMU error %#llx\n",
846 (unsigned long long)imustat);
847 if ((imustat &
848 FO_PCI_IMU_ERR_INT_EQ_NOT_EN_P) != 0) {
849 fatal = 1;
850 val = FIRE_PCI_READ_8(sc,
851 FO_PCI_IMU_SCS_ERR_LOG);
852 device_printf(dev, "SCS error log %#llx\n",
853 (unsigned long long)val);
854 }
855 if ((imustat & FO_PCI_IMU_ERR_INT_EQ_OVER_P) != 0) {
856 fatal = 1;
857 val = FIRE_PCI_READ_8(sc,
858 FO_PCI_IMU_EQS_ERR_LOG);
859 device_printf(dev, "EQS error log %#llx\n",
860 (unsigned long long)val);
861 }
862 if ((imustat & (FO_PCI_IMU_ERR_INT_MSI_MAL_ERR_P |
863 FO_PCI_IMU_ERR_INT_MSI_PAR_ERR_P |
864 FO_PCI_IMU_ERR_INT_PMEACK_MES_NOT_EN_P |
865 FO_PCI_IMU_ERR_INT_PMPME_MES_NOT_EN_P |
866 FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P |
867 FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P |
868 FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P |
869 FO_PCI_IMU_ERR_INT_MSI_NOT_EN_P)) != 0) {
870 fatal = 1;
871 val = FIRE_PCI_READ_8(sc,
872 FO_PCI_IMU_RDS_ERR_LOG);
873 device_printf(dev, "RDS error log %#llx\n",
874 (unsigned long long)val);
875 }
876 }
877 if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_MMU) != 0) {
878 fatal = 1;
879 mmustat = FIRE_PCI_READ_8(sc, FO_PCI_MMU_INT_STAT);
880 mmutfar = FIRE_PCI_READ_8(sc,
881 FO_PCI_MMU_TRANS_FAULT_ADDR);
882 mmutfsr = FIRE_PCI_READ_8(sc,
883 FO_PCI_MMU_TRANS_FAULT_STAT);
884 if ((mmustat & (FO_PCI_MMU_ERR_INT_TBW_DPE_P |
885 FO_PCI_MMU_ERR_INT_TBW_ERR_P |
886 FO_PCI_MMU_ERR_INT_TBW_UDE_P |
887 FO_PCI_MMU_ERR_INT_TBW_DME_P |
888 FO_PCI_MMU_ERR_INT_TTC_CAE_P |
889 FIRE_PCI_MMU_ERR_INT_TTC_DPE_P |
890 OBERON_PCI_MMU_ERR_INT_TTC_DUE_P |
891 FO_PCI_MMU_ERR_INT_TRN_ERR_P)) != 0)
892 fatal = 1;
893 else {
894 sc->sc_stats_mmu_err++;
895 FIRE_PCI_WRITE_8(sc, FO_PCI_MMU_ERR_STAT_CLR,
896 mmustat);
897 }
898 device_printf(dev,
899 "MMU error %#llx: TFAR %#llx TFSR %#llx\n",
900 (unsigned long long)mmustat,
901 (unsigned long long)mmutfar,
902 (unsigned long long)mmutfsr);
903 }
904 }
905 if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_PEC) != 0) {
906 pecstat = FIRE_PCI_READ_8(sc, FO_PCI_PEC_CORE_BLOCK_INT_STAT);
907 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_UERR) != 0) {
908 fatal = 1;
909 uestat = FIRE_PCI_READ_8(sc,
910 FO_PCI_TLU_UERR_INT_STAT);
911 device_printf(dev,
912 "DLU/TLU uncorrectable error %#llx\n",
913 (unsigned long long)uestat);
914 if ((uestat & (FO_PCI_TLU_UERR_INT_UR_P |
915 OBERON_PCI_TLU_UERR_INT_POIS_P |
916 FO_PCI_TLU_UERR_INT_MFP_P |
917 FO_PCI_TLU_UERR_INT_ROF_P |
918 FO_PCI_TLU_UERR_INT_UC_P |
919 FIRE_PCI_TLU_UERR_INT_PP_P |
920 OBERON_PCI_TLU_UERR_INT_POIS_P)) != 0) {
921 val = FIRE_PCI_READ_8(sc,
922 FO_PCI_TLU_RX_UERR_HDR1_LOG);
923 device_printf(dev,
924 "receive header log %#llx\n",
925 (unsigned long long)val);
926 val = FIRE_PCI_READ_8(sc,
927 FO_PCI_TLU_RX_UERR_HDR2_LOG);
928 device_printf(dev,
929 "receive header log 2 %#llx\n",
930 (unsigned long long)val);
931 }
932 if ((uestat & FO_PCI_TLU_UERR_INT_CTO_P) != 0) {
933 val = FIRE_PCI_READ_8(sc,
934 FO_PCI_TLU_TX_UERR_HDR1_LOG);
935 device_printf(dev,
936 "transmit header log %#llx\n",
937 (unsigned long long)val);
938 val = FIRE_PCI_READ_8(sc,
939 FO_PCI_TLU_TX_UERR_HDR2_LOG);
940 device_printf(dev,
941 "transmit header log 2 %#llx\n",
942 (unsigned long long)val);
943 }
944 if ((uestat & FO_PCI_TLU_UERR_INT_DLP_P) != 0) {
945 val = FIRE_PCI_READ_8(sc,
946 FO_PCI_LPU_LNK_LYR_INT_STAT);
947 device_printf(dev,
948 "link layer interrupt and status %#llx\n",
949 (unsigned long long)val);
950 }
951 if ((uestat & FO_PCI_TLU_UERR_INT_TE_P) != 0) {
952 val = FIRE_PCI_READ_8(sc,
953 FO_PCI_LPU_PHY_LYR_INT_STAT);
954 device_printf(dev,
955 "phy layer interrupt and status %#llx\n",
956 (unsigned long long)val);
957 }
958 }
959 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_CERR) != 0) {
960 sc->sc_stats_tlu_ce++;
961 cestat = FIRE_PCI_READ_8(sc,
962 FO_PCI_TLU_CERR_INT_STAT);
963 device_printf(dev,
964 "DLU/TLU correctable error %#llx\n",
965 (unsigned long long)cestat);
966 val = FIRE_PCI_READ_8(sc,
967 FO_PCI_LPU_LNK_LYR_INT_STAT);
968 device_printf(dev,
969 "link layer interrupt and status %#llx\n",
970 (unsigned long long)val);
971 if ((cestat & FO_PCI_TLU_CERR_INT_RE_P) != 0) {
972 FIRE_PCI_WRITE_8(sc,
973 FO_PCI_LPU_LNK_LYR_INT_STAT, val);
974 val = FIRE_PCI_READ_8(sc,
975 FO_PCI_LPU_PHY_LYR_INT_STAT);
976 device_printf(dev,
977 "phy layer interrupt and status %#llx\n",
978 (unsigned long long)val);
979 }
980 FIRE_PCI_WRITE_8(sc, FO_PCI_TLU_CERR_STAT_CLR,
981 cestat);
982 }
983 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_OEVENT) != 0) {
984 oenfatal = 0;
985 oestat = FIRE_PCI_READ_8(sc,
986 FO_PCI_TLU_OEVENT_INT_STAT);
987 device_printf(dev, "DLU/TLU other event %#llx\n",
988 (unsigned long long)oestat);
989 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
990 FO_PCI_TLU_OEVENT_MRC_P |
991 FO_PCI_TLU_OEVENT_WUC_P |
992 FO_PCI_TLU_OEVENT_RUC_P |
993 FO_PCI_TLU_OEVENT_CRS_P)) != 0) {
994 val = FIRE_PCI_READ_8(sc,
995 FO_PCI_TLU_RX_OEVENT_HDR1_LOG);
996 device_printf(dev,
997 "receive header log %#llx\n",
998 (unsigned long long)val);
999 val = FIRE_PCI_READ_8(sc,
1000 FO_PCI_TLU_RX_OEVENT_HDR2_LOG);
1001 device_printf(dev,
1002 "receive header log 2 %#llx\n",
1003 (unsigned long long)val);
1004 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
1005 FO_PCI_TLU_OEVENT_MRC_P |
1006 FO_PCI_TLU_OEVENT_WUC_P |
1007 FO_PCI_TLU_OEVENT_RUC_P)) != 0)
1008 fatal = 1;
1009 else {
1010 sc->sc_stats_tlu_oe_rx_err++;
1011 oenfatal = 1;
1012 }
1013 }
1014 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
1015 FO_PCI_TLU_OEVENT_CTO_P |
1016 FO_PCI_TLU_OEVENT_WUC_P |
1017 FO_PCI_TLU_OEVENT_RUC_P)) != 0) {
1018 val = FIRE_PCI_READ_8(sc,
1019 FO_PCI_TLU_TX_OEVENT_HDR1_LOG);
1020 device_printf(dev,
1021 "transmit header log %#llx\n",
1022 (unsigned long long)val);
1023 val = FIRE_PCI_READ_8(sc,
1024 FO_PCI_TLU_TX_OEVENT_HDR2_LOG);
1025 device_printf(dev,
1026 "transmit header log 2 %#llx\n",
1027 (unsigned long long)val);
1028 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
1029 FO_PCI_TLU_OEVENT_CTO_P |
1030 FO_PCI_TLU_OEVENT_WUC_P |
1031 FO_PCI_TLU_OEVENT_RUC_P)) != 0)
1032 fatal = 1;
1033 else {
1034 sc->sc_stats_tlu_oe_tx_err++;
1035 oenfatal = 1;
1036 }
1037 }
1038 if ((oestat & (FO_PCI_TLU_OEVENT_ERO_P |
1039 FO_PCI_TLU_OEVENT_EMP_P |
1040 FO_PCI_TLU_OEVENT_EPE_P |
1041 FIRE_PCI_TLU_OEVENT_ERP_P |
1042 OBERON_PCI_TLU_OEVENT_ERBU_P |
1043 FIRE_PCI_TLU_OEVENT_EIP_P |
1044 OBERON_PCI_TLU_OEVENT_EIUE_P)) != 0) {
1045 fatal = 1;
1046 val = FIRE_PCI_READ_8(sc,
1047 FO_PCI_LPU_LNK_LYR_INT_STAT);
1048 device_printf(dev,
1049 "link layer interrupt and status %#llx\n",
1050 (unsigned long long)val);
1051 }
1052 if ((oestat & (FO_PCI_TLU_OEVENT_IIP_P |
1053 FO_PCI_TLU_OEVENT_EDP_P |
1054 FIRE_PCI_TLU_OEVENT_EHP_P |
1055 OBERON_PCI_TLU_OEVENT_TLUEITMO_S |
1056 FO_PCI_TLU_OEVENT_ERU_P)) != 0)
1057 fatal = 1;
1058 if ((oestat & (FO_PCI_TLU_OEVENT_NFP_P |
1059 FO_PCI_TLU_OEVENT_LWC_P |
1060 FO_PCI_TLU_OEVENT_LIN_P |
1061 FO_PCI_TLU_OEVENT_LRS_P |
1062 FO_PCI_TLU_OEVENT_LDN_P |
1063 FO_PCI_TLU_OEVENT_LUP_P)) != 0)
1064 oenfatal = 1;
1065 if (oenfatal != 0) {
1066 sc->sc_stats_tlu_oe_non_fatal++;
1067 FIRE_PCI_WRITE_8(sc,
1068 FO_PCI_TLU_OEVENT_STAT_CLR, oestat);
1069 if ((oestat & FO_PCI_TLU_OEVENT_LIN_P) != 0)
1070 FIRE_PCI_WRITE_8(sc,
1071 FO_PCI_LPU_LNK_LYR_INT_STAT,
1072 FIRE_PCI_READ_8(sc,
1073 FO_PCI_LPU_LNK_LYR_INT_STAT));
1074 }
1075 }
1076 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_ILU) != 0) {
1077 ilustat = FIRE_PCI_READ_8(sc, FO_PCI_ILU_INT_STAT);
1078 device_printf(dev, "ILU error %#llx\n",
1079 (unsigned long long)ilustat);
1080 if ((ilustat & (FIRE_PCI_ILU_ERR_INT_IHB_PE_P |
1081 FIRE_PCI_ILU_ERR_INT_IHB_PE_P)) != 0)
1082 fatal = 1;
1083 else {
1084 sc->sc_stats_ilu_err++;
1085 FIRE_PCI_WRITE_8(sc, FO_PCI_ILU_INT_STAT,
1086 ilustat);
1087 }
1088 }
1089 }
1090 mtx_unlock_spin(&sc->sc_pcib_mtx);
1091 if (fatal != 0)
1092 panic("%s: fatal DMC/PEC error",
1093 device_get_nameunit(sc->sc_dev));
1094 return (FILTER_HANDLED);
1095 }
1096
1097 static int
1098 fire_xcb(void *arg)
1099 {
1100 struct fire_softc *sc;
1101 device_t dev;
1102 uint64_t errstat, intstat, val;
1103 u_int fatal;
1104
1105 fatal = 0;
1106 sc = arg;
1107 dev = sc->sc_dev;
1108 mtx_lock_spin(&sc->sc_pcib_mtx);
1109 if (sc->sc_mode == FIRE_MODE_OBERON) {
1110 intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT);
1111 device_printf(dev, "UBC error: interrupt status %#llx\n",
1112 (unsigned long long)intstat);
1113 if ((intstat & ~(OBERON_UBC_ERR_INT_DMARDUEB_P |
1114 OBERON_UBC_ERR_INT_DMARDUEA_P)) != 0)
1115 fatal = 1;
1116 else
1117 sc->sc_stats_ubc_dmardue++;
1118 if (fatal != 0) {
1119 mtx_unlock_spin(&sc->sc_pcib_mtx);
1120 panic("%s: fatal UBC core block error",
1121 device_get_nameunit(sc->sc_dev));
1122 } else {
1123 FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
1124 mtx_unlock_spin(&sc->sc_pcib_mtx);
1125 }
1126 } else {
1127 errstat = FIRE_CTRL_READ_8(sc, FIRE_JBC_CORE_BLOCK_ERR_STAT);
1128 if ((errstat & (FIRE_JBC_CORE_BLOCK_ERR_STAT_MERGE |
1129 FIRE_JBC_CORE_BLOCK_ERR_STAT_JBCINT |
1130 FIRE_JBC_CORE_BLOCK_ERR_STAT_DMCINT)) != 0) {
1131 intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT);
1132 device_printf(dev, "JBC interrupt status %#llx\n",
1133 (unsigned long long)intstat);
1134 if ((intstat & FIRE_JBC_ERR_INT_EBUS_TO_P) != 0) {
1135 val = FIRE_CTRL_READ_8(sc,
1136 FIRE_JBC_CSR_ERR_LOG);
1137 device_printf(dev, "CSR error log %#llx\n",
1138 (unsigned long long)val);
1139 }
1140 if ((intstat & (FIRE_JBC_ERR_INT_UNSOL_RD_P |
1141 FIRE_JBC_ERR_INT_UNSOL_INT_P)) != 0) {
1142 if ((intstat &
1143 FIRE_JBC_ERR_INT_UNSOL_RD_P) != 0)
1144 sc->sc_stats_jbc_unsol_rd++;
1145 if ((intstat &
1146 FIRE_JBC_ERR_INT_UNSOL_INT_P) != 0)
1147 sc->sc_stats_jbc_unsol_int++;
1148 val = FIRE_CTRL_READ_8(sc,
1149 FIRE_DMCINT_IDC_ERR_LOG);
1150 device_printf(dev,
1151 "DMCINT IDC error log %#llx\n",
1152 (unsigned long long)val);
1153 }
1154 if ((intstat & (FIRE_JBC_ERR_INT_MB_PER_P |
1155 FIRE_JBC_ERR_INT_MB_PEW_P)) != 0) {
1156 fatal = 1;
1157 val = FIRE_CTRL_READ_8(sc,
1158 FIRE_MERGE_TRANS_ERR_LOG);
1159 device_printf(dev,
1160 "merge transaction error log %#llx\n",
1161 (unsigned long long)val);
1162 }
1163 if ((intstat & FIRE_JBC_ERR_INT_IJP_P) != 0) {
1164 fatal = 1;
1165 val = FIRE_CTRL_READ_8(sc,
1166 FIRE_JBCINT_OTRANS_ERR_LOG);
1167 device_printf(dev,
1168 "JBCINT out transaction error log "
1169 "%#llx\n", (unsigned long long)val);
1170 val = FIRE_CTRL_READ_8(sc,
1171 FIRE_JBCINT_OTRANS_ERR_LOG2);
1172 device_printf(dev,
1173 "JBCINT out transaction error log 2 "
1174 "%#llx\n", (unsigned long long)val);
1175 }
1176 if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P |
1177 FIRE_JBC_ERR_INT_CE_ASYN_P |
1178 FIRE_JBC_ERR_INT_JTE_P | FIRE_JBC_ERR_INT_JBE_P |
1179 FIRE_JBC_ERR_INT_JUE_P |
1180 FIRE_JBC_ERR_INT_ICISE_P |
1181 FIRE_JBC_ERR_INT_WR_DPE_P |
1182 FIRE_JBC_ERR_INT_RD_DPE_P |
1183 FIRE_JBC_ERR_INT_ILL_BMW_P |
1184 FIRE_JBC_ERR_INT_ILL_BMR_P |
1185 FIRE_JBC_ERR_INT_BJC_P)) != 0) {
1186 if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P |
1187 FIRE_JBC_ERR_INT_JTE_P |
1188 FIRE_JBC_ERR_INT_JBE_P |
1189 FIRE_JBC_ERR_INT_JUE_P |
1190 FIRE_JBC_ERR_INT_ICISE_P |
1191 FIRE_JBC_ERR_INT_WR_DPE_P |
1192 FIRE_JBC_ERR_INT_RD_DPE_P |
1193 FIRE_JBC_ERR_INT_ILL_BMW_P |
1194 FIRE_JBC_ERR_INT_ILL_BMR_P |
1195 FIRE_JBC_ERR_INT_BJC_P)) != 0)
1196 fatal = 1;
1197 else
1198 sc->sc_stats_jbc_ce_async++;
1199 val = FIRE_CTRL_READ_8(sc,
1200 FIRE_JBCINT_ITRANS_ERR_LOG);
1201 device_printf(dev,
1202 "JBCINT in transaction error log %#llx\n",
1203 (unsigned long long)val);
1204 val = FIRE_CTRL_READ_8(sc,
1205 FIRE_JBCINT_ITRANS_ERR_LOG2);
1206 device_printf(dev,
1207 "JBCINT in transaction error log 2 "
1208 "%#llx\n", (unsigned long long)val);
1209 }
1210 if ((intstat & (FIRE_JBC_ERR_INT_PIO_UNMAP_RD_P |
1211 FIRE_JBC_ERR_INT_ILL_ACC_RD_P |
1212 FIRE_JBC_ERR_INT_PIO_UNMAP_P |
1213 FIRE_JBC_ERR_INT_PIO_DPE_P |
1214 FIRE_JBC_ERR_INT_PIO_CPE_P |
1215 FIRE_JBC_ERR_INT_ILL_ACC_P)) != 0) {
1216 fatal = 1;
1217 val = FIRE_CTRL_READ_8(sc,
1218 FIRE_JBC_CSR_ERR_LOG);
1219 device_printf(dev,
1220 "DMCINT ODCD error log %#llx\n",
1221 (unsigned long long)val);
1222 }
1223 if ((intstat & (FIRE_JBC_ERR_INT_MB_PEA_P |
1224 FIRE_JBC_ERR_INT_CPE_P | FIRE_JBC_ERR_INT_APE_P |
1225 FIRE_JBC_ERR_INT_PIO_CPE_P |
1226 FIRE_JBC_ERR_INT_JTCEEW_P |
1227 FIRE_JBC_ERR_INT_JTCEEI_P |
1228 FIRE_JBC_ERR_INT_JTCEER_P)) != 0) {
1229 fatal = 1;
1230 val = FIRE_CTRL_READ_8(sc,
1231 FIRE_FATAL_ERR_LOG);
1232 device_printf(dev, "fatal error log %#llx\n",
1233 (unsigned long long)val);
1234 val = FIRE_CTRL_READ_8(sc,
1235 FIRE_FATAL_ERR_LOG2);
1236 device_printf(dev, "fatal error log 2 "
1237 "%#llx\n", (unsigned long long)val);
1238 }
1239 if (fatal != 0) {
1240 mtx_unlock_spin(&sc->sc_pcib_mtx);
1241 panic("%s: fatal JBC core block error",
1242 device_get_nameunit(sc->sc_dev));
1243 } else {
1244 FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
1245 mtx_unlock_spin(&sc->sc_pcib_mtx);
1246 }
1247 } else {
1248 mtx_unlock_spin(&sc->sc_pcib_mtx);
1249 panic("%s: unknown JCB core block error status %#llx",
1250 device_get_nameunit(sc->sc_dev),
1251 (unsigned long long)errstat);
1252 }
1253 }
1254 return (FILTER_HANDLED);
1255 }
1256
1257 static int
1258 fire_pcie(void *arg)
1259 {
1260 struct fire_msiqarg *fmqa;
1261 struct fire_softc *sc;
1262 struct fo_msiq_record *qrec;
1263 device_t dev;
1264 uint64_t word0;
1265 u_int head, msg, msiq;
1266
1267 fmqa = arg;
1268 sc = fmqa->fmqa_fica.fica_sc;
1269 dev = sc->sc_dev;
1270 msiq = fmqa->fmqa_msiq;
1271 mtx_lock_spin(&fmqa->fmqa_mtx);
1272 head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >>
1273 FO_PCI_EQ_HD_SHFT;
1274 qrec = &fmqa->fmqa_base[head];
1275 word0 = qrec->fomqr_word0;
1276 for (;;) {
1277 KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSG) != 0,
1278 ("%s: received non-PCIe message in event queue %d "
1279 "(word0 %#llx)", device_get_nameunit(dev), msiq,
1280 (unsigned long long)word0));
1281 msg = (word0 & FO_MQR_WORD0_DATA0_MASK) >>
1282 FO_MQR_WORD0_DATA0_SHFT;
1283
1284 #define PCIE_MSG_CODE_ERR_COR 0x30
1285 #define PCIE_MSG_CODE_ERR_NONFATAL 0x31
1286 #define PCIE_MSG_CODE_ERR_FATAL 0x33
1287
1288 if (msg == PCIE_MSG_CODE_ERR_COR)
1289 device_printf(dev, "correctable PCIe error\n");
1290 else if (msg == PCIE_MSG_CODE_ERR_NONFATAL ||
1291 msg == PCIE_MSG_CODE_ERR_FATAL)
1292 panic("%s: %sfatal PCIe error",
1293 device_get_nameunit(dev),
1294 msg == PCIE_MSG_CODE_ERR_NONFATAL ? "non-" : "");
1295 else
1296 panic("%s: received unknown PCIe message %#x",
1297 device_get_nameunit(dev), msg);
1298 qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK;
1299 head = (head + 1) % sc->sc_msiq_size;
1300 qrec = &fmqa->fmqa_base[head];
1301 word0 = qrec->fomqr_word0;
1302 if (__predict_true((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0))
1303 break;
1304 }
1305 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) <<
1306 FO_PCI_EQ_HD_SHFT);
1307 if ((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) &
1308 FO_PCI_EQ_TL_OVERR) != 0) {
1309 device_printf(dev, "event queue %d overflow\n", msiq);
1310 msiq <<= 3;
1311 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
1312 FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) |
1313 FO_PCI_EQ_CTRL_CLR_COVERR);
1314 }
1315 mtx_unlock_spin(&fmqa->fmqa_mtx);
1316 return (FILTER_HANDLED);
1317 }
1318
1319 static int
1320 fire_maxslots(device_t dev)
1321 {
1322
1323 return (1);
1324 }
1325
1326 static uint32_t
1327 fire_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
1328 int width)
1329 {
1330
1331 return (ofw_pci_read_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus,
1332 slot, func, reg), bus, slot, func, reg, width));
1333 }
1334
1335 static void
1336 fire_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
1337 uint32_t val, int width)
1338 {
1339
1340 ofw_pci_write_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus, slot,
1341 func, reg), bus, slot, func, reg, val, width);
1342 }
1343
1344 static int
1345 fire_route_interrupt(device_t bridge, device_t dev, int pin)
1346 {
1347 ofw_pci_intr_t mintr;
1348
1349 mintr = ofw_pci_route_interrupt_common(bridge, dev, pin);
1350 if (!PCI_INTERRUPT_VALID(mintr))
1351 device_printf(bridge,
1352 "could not route pin %d for device %d.%d\n",
1353 pin, pci_get_slot(dev), pci_get_function(dev));
1354 return (mintr);
1355 }
1356
1357 static void
1358 fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
1359 bus_dmasync_op_t op)
1360 {
1361
1362 if ((map->dm_flags & DMF_LOADED) == 0)
1363 return;
1364
1365 if ((op & BUS_DMASYNC_POSTREAD) != 0)
1366 ofw_pci_dmamap_sync_stst_order_common();
1367 else if ((op & BUS_DMASYNC_PREWRITE) != 0)
1368 membar(Sync);
1369 }
1370
1371 static void
1372 fire_intr_enable(void *arg)
1373 {
1374 struct intr_vector *iv;
1375 struct fire_icarg *fica;
1376 struct fire_softc *sc;
1377 struct pcpu *pc;
1378 uint64_t mr;
1379 u_int ctrl, i;
1380
1381 iv = arg;
1382 fica = iv->iv_icarg;
1383 sc = fica->fica_sc;
1384 mr = FO_PCI_IMAP_V;
1385 if (sc->sc_mode == FIRE_MODE_OBERON)
1386 mr |= (iv->iv_mid << OBERON_PCI_IMAP_T_DESTID_SHFT) &
1387 OBERON_PCI_IMAP_T_DESTID_MASK;
1388 else
1389 mr |= (iv->iv_mid << FIRE_PCI_IMAP_T_JPID_SHFT) &
1390 FIRE_PCI_IMAP_T_JPID_MASK;
1391 /*
1392 * Given that all mondos for the same target are required to use the
1393 * same interrupt controller we just use the CPU ID for indexing the
1394 * latter.
1395 */
1396 ctrl = 0;
1397 for (i = 0; i < mp_ncpus; ++i) {
1398 pc = pcpu_find(i);
1399 if (pc == NULL || iv->iv_mid != pc->pc_mid)
1400 continue;
1401 ctrl = pc->pc_cpuid % 4;
1402 break;
1403 }
1404 mr |= (1ULL << ctrl) << FO_PCI_IMAP_INT_CTRL_NUM_SHFT &
1405 FO_PCI_IMAP_INT_CTRL_NUM_MASK;
1406 FIRE_PCI_WRITE_8(sc, fica->fica_map, mr);
1407 }
1408
1409 static void
1410 fire_intr_disable(void *arg)
1411 {
1412 struct intr_vector *iv;
1413 struct fire_icarg *fica;
1414 struct fire_softc *sc;
1415
1416 iv = arg;
1417 fica = iv->iv_icarg;
1418 sc = fica->fica_sc;
1419 FIRE_PCI_WRITE_8(sc, fica->fica_map,
1420 FIRE_PCI_READ_8(sc, fica->fica_map) & ~FO_PCI_IMAP_V);
1421 }
1422
1423 static void
1424 fire_intr_assign(void *arg)
1425 {
1426 struct intr_vector *iv;
1427 struct fire_icarg *fica;
1428 struct fire_softc *sc;
1429 uint64_t mr;
1430
1431 iv = arg;
1432 fica = iv->iv_icarg;
1433 sc = fica->fica_sc;
1434 mr = FIRE_PCI_READ_8(sc, fica->fica_map);
1435 if ((mr & FO_PCI_IMAP_V) != 0) {
1436 FIRE_PCI_WRITE_8(sc, fica->fica_map, mr & ~FO_PCI_IMAP_V);
1437 FIRE_PCI_BARRIER(sc, fica->fica_map, 8,
1438 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
1439 }
1440 while (FIRE_PCI_READ_8(sc, fica->fica_clr) != INTCLR_IDLE)
1441 ;
1442 if ((mr & FO_PCI_IMAP_V) != 0)
1443 fire_intr_enable(arg);
1444 }
1445
1446 static void
1447 fire_intr_clear(void *arg)
1448 {
1449 struct intr_vector *iv;
1450 struct fire_icarg *fica;
1451
1452 iv = arg;
1453 fica = iv->iv_icarg;
1454 FIRE_PCI_WRITE_8(fica->fica_sc, fica->fica_clr, INTCLR_IDLE);
1455 }
1456
1457 /*
1458 * Given that the event queue implementation matches our current MD and MI
1459 * interrupt frameworks like square pegs fit into round holes we are generous
1460 * and use one event queue per MSI for now, which limits us to 35 MSIs/MSI-Xs
1461 * per Host-PCIe-bridge (we use one event queue for the PCIe error messages).
1462 * This seems tolerable as long as most devices just use one MSI/MSI-X anyway.
1463 * Adding knowledge about MSIs/MSI-Xs to the MD interrupt code should allow us
1464 * to decouple the 1:1 mapping at the cost of no longer being able to bind
1465 * MSIs/MSI-Xs to specific CPUs as we currently have no reliable way to
1466 * quiesce a device while we move its MSIs/MSI-Xs to another event queue.
1467 */
1468
1469 static int
1470 fire_alloc_msi(device_t dev, device_t child, int count, int maxcount __unused,
1471 int *irqs)
1472 {
1473 struct fire_softc *sc;
1474 u_int i, j, msiqrun;
1475
1476 if (powerof2(count) == 0 || count > 32)
1477 return (EINVAL);
1478
1479 sc = device_get_softc(dev);
1480 mtx_lock(&sc->sc_msi_mtx);
1481 msiqrun = 0;
1482 for (i = 0; i < sc->sc_msiq_count; i++) {
1483 for (j = i; j < i + count; j++) {
1484 if (isclr(sc->sc_msiq_bitmap, j) == 0)
1485 break;
1486 }
1487 if (j == i + count) {
1488 msiqrun = i;
1489 break;
1490 }
1491 }
1492 if (i == sc->sc_msiq_count) {
1493 mtx_unlock(&sc->sc_msi_mtx);
1494 return (ENXIO);
1495 }
1496 for (i = 0; i + count < sc->sc_msi_count; i += count) {
1497 for (j = i; j < i + count; j++)
1498 if (isclr(sc->sc_msi_bitmap, j) == 0)
1499 break;
1500 if (j == i + count) {
1501 for (j = 0; j < count; j++) {
1502 setbit(sc->sc_msiq_bitmap, msiqrun + j);
1503 setbit(sc->sc_msi_bitmap, i + j);
1504 sc->sc_msi_msiq_table[i + j] = msiqrun + j;
1505 irqs[j] = sc->sc_msi_first + i + j;
1506 }
1507 mtx_unlock(&sc->sc_msi_mtx);
1508 return (0);
1509 }
1510 }
1511 mtx_unlock(&sc->sc_msi_mtx);
1512 return (ENXIO);
1513 }
1514
1515 static int
1516 fire_release_msi(device_t dev, device_t child, int count, int *irqs)
1517 {
1518 struct fire_softc *sc;
1519 u_int i;
1520
1521 sc = device_get_softc(dev);
1522 mtx_lock(&sc->sc_msi_mtx);
1523 for (i = 0; i < count; i++) {
1524 clrbit(sc->sc_msiq_bitmap,
1525 sc->sc_msi_msiq_table[irqs[i] - sc->sc_msi_first]);
1526 clrbit(sc->sc_msi_bitmap, irqs[i] - sc->sc_msi_first);
1527 }
1528 mtx_unlock(&sc->sc_msi_mtx);
1529 return (0);
1530 }
1531
1532 static int
1533 fire_alloc_msix(device_t dev, device_t child, int *irq)
1534 {
1535 struct fire_softc *sc;
1536 int i, msiq;
1537
1538 sc = device_get_softc(dev);
1539 if ((sc->sc_flags & FIRE_MSIX) == 0)
1540 return (ENXIO);
1541 mtx_lock(&sc->sc_msi_mtx);
1542 msiq = 0;
1543 for (i = 0; i < sc->sc_msiq_count; i++) {
1544 if (isclr(sc->sc_msiq_bitmap, i) != 0) {
1545 msiq = i;
1546 break;
1547 }
1548 }
1549 if (i == sc->sc_msiq_count) {
1550 mtx_unlock(&sc->sc_msi_mtx);
1551 return (ENXIO);
1552 }
1553 for (i = sc->sc_msi_count - 1; i >= 0; i--) {
1554 if (isclr(sc->sc_msi_bitmap, i) != 0) {
1555 setbit(sc->sc_msiq_bitmap, msiq);
1556 setbit(sc->sc_msi_bitmap, i);
1557 sc->sc_msi_msiq_table[i] = msiq;
1558 *irq = sc->sc_msi_first + i;
1559 mtx_unlock(&sc->sc_msi_mtx);
1560 return (0);
1561 }
1562 }
1563 mtx_unlock(&sc->sc_msi_mtx);
1564 return (ENXIO);
1565 }
1566
1567 static int
1568 fire_release_msix(device_t dev, device_t child, int irq)
1569 {
1570 struct fire_softc *sc;
1571
1572 sc = device_get_softc(dev);
1573 if ((sc->sc_flags & FIRE_MSIX) == 0)
1574 return (ENXIO);
1575 mtx_lock(&sc->sc_msi_mtx);
1576 clrbit(sc->sc_msiq_bitmap,
1577 sc->sc_msi_msiq_table[irq - sc->sc_msi_first]);
1578 clrbit(sc->sc_msi_bitmap, irq - sc->sc_msi_first);
1579 mtx_unlock(&sc->sc_msi_mtx);
1580 return (0);
1581 }
1582
1583 static int
1584 fire_map_msi(device_t dev, device_t child, int irq, uint64_t *addr,
1585 uint32_t *data)
1586 {
1587 struct fire_softc *sc;
1588 struct pci_devinfo *dinfo;
1589
1590 sc = device_get_softc(dev);
1591 dinfo = device_get_ivars(child);
1592 if (dinfo->cfg.msi.msi_alloc > 0) {
1593 if ((irq & ~sc->sc_msi_data_mask) != 0) {
1594 device_printf(dev, "invalid MSI 0x%x\n", irq);
1595 return (EINVAL);
1596 }
1597 } else {
1598 if ((sc->sc_flags & FIRE_MSIX) == 0)
1599 return (ENXIO);
1600 if (fls(irq) > sc->sc_msix_data_width) {
1601 device_printf(dev, "invalid MSI-X 0x%x\n", irq);
1602 return (EINVAL);
1603 }
1604 }
1605 if (dinfo->cfg.msi.msi_alloc > 0 &&
1606 (dinfo->cfg.msi.msi_ctrl & PCIM_MSICTRL_64BIT) == 0)
1607 *addr = sc->sc_msi_addr32;
1608 else
1609 *addr = sc->sc_msi_addr64;
1610 *data = irq;
1611 return (0);
1612 }
1613
1614 static void
1615 fire_msiq_handler(void *cookie)
1616 {
1617 struct intr_vector *iv;
1618 struct fire_msiqarg *fmqa;
1619
1620 iv = cookie;
1621 fmqa = iv->iv_icarg;
1622 /*
1623 * Note that since fire_intr_clear() will clear the event queue
1624 * interrupt after the handler associated with the MSI [sic] has
1625 * been executed we have to protect the access to the event queue as
1626 * otherwise nested event queue interrupts cause corruption of the
1627 * event queue on MP machines. Obviously especially when abandoning
1628 * the 1:1 mapping it would be better to not clear the event queue
1629 * interrupt after each handler invocation but only once when the
1630 * outstanding MSIs have been processed but unfortunately that
1631 * doesn't work well and leads to interrupt storms with controllers/
1632 * drivers which don't mask interrupts while the handler is executed.
1633 * Maybe delaying clearing the MSI until after the handler has been
1634 * executed could be used to work around this but that's not the
1635 * intended usage and might in turn cause lost MSIs.
1636 */
1637 mtx_lock_spin(&fmqa->fmqa_mtx);
1638 fire_msiq_common(iv, fmqa);
1639 mtx_unlock_spin(&fmqa->fmqa_mtx);
1640 }
1641
1642 static void
1643 fire_msiq_filter(void *cookie)
1644 {
1645 struct intr_vector *iv;
1646 struct fire_msiqarg *fmqa;
1647
1648 iv = cookie;
1649 fmqa = iv->iv_icarg;
1650 /*
1651 * For filters we don't use fire_intr_clear() since it would clear
1652 * the event queue interrupt while we're still processing the event
1653 * queue as filters and associated post-filter handler are executed
1654 * directly, which in turn would lead to lost MSIs. So we clear the
1655 * event queue interrupt only once after processing the event queue.
1656 * Given that this still guarantees the filters to not be executed
1657 * concurrently and no other CPU can clear the event queue interrupt
1658 * while the event queue is still processed, we don't even need to
1659 * interlock the access to the event queue in this case.
1660 */
1661 critical_enter();
1662 fire_msiq_common(iv, fmqa);
1663 FIRE_PCI_WRITE_8(fmqa->fmqa_fica.fica_sc, fmqa->fmqa_fica.fica_clr,
1664 INTCLR_IDLE);
1665 critical_exit();
1666 }
1667
1668 static inline void
1669 fire_msiq_common(struct intr_vector *iv, struct fire_msiqarg *fmqa)
1670 {
1671 struct fire_softc *sc;
1672 struct fo_msiq_record *qrec;
1673 device_t dev;
1674 uint64_t word0;
1675 u_int head, msi, msiq;
1676
1677 sc = fmqa->fmqa_fica.fica_sc;
1678 dev = sc->sc_dev;
1679 msiq = fmqa->fmqa_msiq;
1680 head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >>
1681 FO_PCI_EQ_HD_SHFT;
1682 qrec = &fmqa->fmqa_base[head];
1683 word0 = qrec->fomqr_word0;
1684 for (;;) {
1685 if (__predict_false((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0))
1686 break;
1687 KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSI64) != 0 ||
1688 (word0 & FO_MQR_WORD0_FMT_TYPE_MSI32) != 0,
1689 ("%s: received non-MSI/MSI-X message in event queue %d "
1690 "(word0 %#llx)", device_get_nameunit(dev), msiq,
1691 (unsigned long long)word0));
1692 msi = (word0 & FO_MQR_WORD0_DATA0_MASK) >>
1693 FO_MQR_WORD0_DATA0_SHFT;
1694 /*
1695 * Sanity check the MSI/MSI-X as long as we use a 1:1 mapping.
1696 */
1697 KASSERT(msi == fmqa->fmqa_msi,
1698 ("%s: received non-matching MSI/MSI-X in event queue %d "
1699 "(%d versus %d)", device_get_nameunit(dev), msiq, msi,
1700 fmqa->fmqa_msi));
1701 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + (msi << 3),
1702 FO_PCI_MSI_CLR_EQWR_N);
1703 if (__predict_false(intr_event_handle(iv->iv_event,
1704 NULL) != 0))
1705 printf("stray MSI/MSI-X in event queue %d\n", msiq);
1706 qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK;
1707 head = (head + 1) % sc->sc_msiq_size;
1708 qrec = &fmqa->fmqa_base[head];
1709 word0 = qrec->fomqr_word0;
1710 }
1711 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) <<
1712 FO_PCI_EQ_HD_SHFT);
1713 if (__predict_false((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) &
1714 FO_PCI_EQ_TL_OVERR) != 0)) {
1715 device_printf(dev, "event queue %d overflow\n", msiq);
1716 msiq <<= 3;
1717 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
1718 FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) |
1719 FO_PCI_EQ_CTRL_CLR_COVERR);
1720 }
1721 }
1722
1723 static int
1724 fire_setup_intr(device_t dev, device_t child, struct resource *ires,
1725 int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg,
1726 void **cookiep)
1727 {
1728 struct fire_softc *sc;
1729 struct fire_msiqarg *fmqa;
1730 u_long vec;
1731 int error;
1732 u_int msi, msiq;
1733
1734 sc = device_get_softc(dev);
1735 /*
1736 * XXX this assumes that a device only has one INTx, while in fact
1737 * Cassini+ and Saturn can use all four the firmware has assigned
1738 * to them, but so does pci(4).
1739 */
1740 if (rman_get_rid(ires) != 0) {
1741 msi = rman_get_start(ires);
1742 msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first];
1743 vec = INTMAP_VEC(sc->sc_ign, sc->sc_msiq_ino_first + msiq);
1744 msiq += sc->sc_msiq_first;
1745 if (intr_vectors[vec].iv_ic != &fire_ic) {
1746 device_printf(dev,
1747 "invalid interrupt controller for vector 0x%lx\n",
1748 vec);
1749 return (EINVAL);
1750 }
1751 /*
1752 * The MD interrupt code needs the vector rather than the MSI.
1753 */
1754 rman_set_start(ires, vec);
1755 rman_set_end(ires, vec);
1756 error = bus_generic_setup_intr(dev, child, ires, flags, filt,
1757 intr, arg, cookiep);
1758 rman_set_start(ires, msi);
1759 rman_set_end(ires, msi);
1760 if (error != 0)
1761 return (error);
1762 fmqa = intr_vectors[vec].iv_icarg;
1763 /*
1764 * XXX inject our event queue handler.
1765 */
1766 if (filt != NULL) {
1767 intr_vectors[vec].iv_func = fire_msiq_filter;
1768 intr_vectors[vec].iv_ic = &fire_msiqc_filter;
1769 /*
1770 * Ensure the event queue interrupt is cleared, it
1771 * might have triggered before. Given we supply NULL
1772 * as ic_clear, inthand_add() won't do this for us.
1773 */
1774 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_fica.fica_clr,
1775 INTCLR_IDLE);
1776 } else
1777 intr_vectors[vec].iv_func = fire_msiq_handler;
1778 /* Record the MSI/MSI-X as long as we we use a 1:1 mapping. */
1779 fmqa->fmqa_msi = msi;
1780 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (msiq << 3),
1781 FO_PCI_EQ_CTRL_SET_EN);
1782 msi <<= 3;
1783 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
1784 (FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) &
1785 ~FO_PCI_MSI_MAP_EQNUM_MASK) |
1786 ((msiq << FO_PCI_MSI_MAP_EQNUM_SHFT) &
1787 FO_PCI_MSI_MAP_EQNUM_MASK));
1788 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + msi,
1789 FO_PCI_MSI_CLR_EQWR_N);
1790 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
1791 FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) |
1792 FO_PCI_MSI_MAP_V);
1793 return (error);
1794 }
1795
1796 /*
1797 * Make sure the vector is fully specified and we registered
1798 * our interrupt controller for it.
1799 */
1800 vec = rman_get_start(ires);
1801 if (INTIGN(vec) != sc->sc_ign) {
1802 device_printf(dev, "invalid interrupt vector 0x%lx\n", vec);
1803 return (EINVAL);
1804 }
1805 if (intr_vectors[vec].iv_ic != &fire_ic) {
1806 device_printf(dev,
1807 "invalid interrupt controller for vector 0x%lx\n", vec);
1808 return (EINVAL);
1809 }
1810 return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr,
1811 arg, cookiep));
1812 }
1813
1814 static int
1815 fire_teardown_intr(device_t dev, device_t child, struct resource *ires,
1816 void *cookie)
1817 {
1818 struct fire_softc *sc;
1819 u_long vec;
1820 int error;
1821 u_int msi, msiq;
1822
1823 sc = device_get_softc(dev);
1824 if (rman_get_rid(ires) != 0) {
1825 msi = rman_get_start(ires);
1826 msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first];
1827 vec = INTMAP_VEC(sc->sc_ign, msiq + sc->sc_msiq_ino_first);
1828 msiq += sc->sc_msiq_first;
1829 msi <<= 3;
1830 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
1831 FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) &
1832 ~FO_PCI_MSI_MAP_V);
1833 msiq <<= 3;
1834 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
1835 FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I |
1836 FO_PCI_EQ_CTRL_CLR_DIS);
1837 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_TL_BASE + msiq,
1838 (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK);
1839 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_HD_BASE + msiq,
1840 (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK);
1841 intr_vectors[vec].iv_ic = &fire_ic;
1842 /*
1843 * The MD interrupt code needs the vector rather than the MSI.
1844 */
1845 rman_set_start(ires, vec);
1846 rman_set_end(ires, vec);
1847 error = bus_generic_teardown_intr(dev, child, ires, cookie);
1848 msi >>= 3;
1849 rman_set_start(ires, msi);
1850 rman_set_end(ires, msi);
1851 return (error);
1852 }
1853 return (bus_generic_teardown_intr(dev, child, ires, cookie));
1854 }
1855
1856 static struct resource *
1857 fire_alloc_resource(device_t bus, device_t child, int type, int *rid,
1858 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
1859 {
1860 struct fire_softc *sc;
1861
1862 if (type == SYS_RES_IRQ && *rid == 0) {
1863 sc = device_get_softc(bus);
1864 start = end = INTMAP_VEC(sc->sc_ign, end);
1865 }
1866 return (ofw_pci_alloc_resource(bus, child, type, rid, start, end,
1867 count, flags));
1868 }
1869
1870 static u_int
1871 fire_get_timecount(struct timecounter *tc)
1872 {
1873 struct fire_softc *sc;
1874
1875 sc = tc->tc_priv;
1876 return (FIRE_CTRL_READ_8(sc, FO_XBC_PRF_CNT0) & TC_COUNTER_MAX_MASK);
1877 }
Cache object: 22b5de5bca79119ce41e74add3171dbe
|