FreeBSD/Linux Kernel Cross Reference
sys/dev/pci/pci.c
1 /*-
2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
3 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
4 * Copyright (c) 2000, BSDi
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD: releng/11.2/sys/dev/pci/pci.c 331722 2018-03-29 02:50:57Z eadler $");
31
32 #include "opt_bus.h"
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/malloc.h>
37 #include <sys/module.h>
38 #include <sys/limits.h>
39 #include <sys/linker.h>
40 #include <sys/fcntl.h>
41 #include <sys/conf.h>
42 #include <sys/kernel.h>
43 #include <sys/queue.h>
44 #include <sys/sysctl.h>
45 #include <sys/endian.h>
46
47 #include <vm/vm.h>
48 #include <vm/pmap.h>
49 #include <vm/vm_extern.h>
50
51 #include <sys/bus.h>
52 #include <machine/bus.h>
53 #include <sys/rman.h>
54 #include <machine/resource.h>
55 #include <machine/stdarg.h>
56
57 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
58 #include <machine/intr_machdep.h>
59 #endif
60
61 #include <sys/pciio.h>
62 #include <dev/pci/pcireg.h>
63 #include <dev/pci/pcivar.h>
64 #include <dev/pci/pci_private.h>
65
66 #ifdef PCI_IOV
67 #include <sys/nv.h>
68 #include <dev/pci/pci_iov_private.h>
69 #endif
70
71 #include <dev/usb/controller/xhcireg.h>
72 #include <dev/usb/controller/ehcireg.h>
73 #include <dev/usb/controller/ohcireg.h>
74 #include <dev/usb/controller/uhcireg.h>
75
76 #include "pcib_if.h"
77 #include "pci_if.h"
78
79 #define PCIR_IS_BIOS(cfg, reg) \
80 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \
81 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
82
83 static int pci_has_quirk(uint32_t devid, int quirk);
84 static pci_addr_t pci_mapbase(uint64_t mapreg);
85 static const char *pci_maptype(uint64_t mapreg);
86 static int pci_maprange(uint64_t mapreg);
87 static pci_addr_t pci_rombase(uint64_t mapreg);
88 static int pci_romsize(uint64_t testval);
89 static void pci_fixancient(pcicfgregs *cfg);
90 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...);
91
92 static int pci_porten(device_t dev);
93 static int pci_memen(device_t dev);
94 static void pci_assign_interrupt(device_t bus, device_t dev,
95 int force_route);
96 static int pci_add_map(device_t bus, device_t dev, int reg,
97 struct resource_list *rl, int force, int prefetch);
98 static int pci_probe(device_t dev);
99 static int pci_attach(device_t dev);
100 static int pci_detach(device_t dev);
101 static void pci_load_vendor_data(void);
102 static int pci_describe_parse_line(char **ptr, int *vendor,
103 int *device, char **desc);
104 static char *pci_describe_device(device_t dev);
105 static int pci_modevent(module_t mod, int what, void *arg);
106 static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
107 pcicfgregs *cfg);
108 static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
109 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
110 int reg, uint32_t *data);
111 #if 0
112 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
113 int reg, uint32_t data);
114 #endif
115 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
116 static void pci_mask_msix(device_t dev, u_int index);
117 static void pci_unmask_msix(device_t dev, u_int index);
118 static int pci_msi_blacklisted(void);
119 static int pci_msix_blacklisted(void);
120 static void pci_resume_msi(device_t dev);
121 static void pci_resume_msix(device_t dev);
122 static int pci_remap_intr_method(device_t bus, device_t dev,
123 u_int irq);
124
125 static int pci_get_id_method(device_t dev, device_t child,
126 enum pci_id_type type, uintptr_t *rid);
127
128 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
129 int b, int s, int f, uint16_t vid, uint16_t did);
130
131 static device_method_t pci_methods[] = {
132 /* Device interface */
133 DEVMETHOD(device_probe, pci_probe),
134 DEVMETHOD(device_attach, pci_attach),
135 DEVMETHOD(device_detach, pci_detach),
136 DEVMETHOD(device_shutdown, bus_generic_shutdown),
137 DEVMETHOD(device_suspend, bus_generic_suspend),
138 DEVMETHOD(device_resume, pci_resume),
139
140 /* Bus interface */
141 DEVMETHOD(bus_print_child, pci_print_child),
142 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
143 DEVMETHOD(bus_read_ivar, pci_read_ivar),
144 DEVMETHOD(bus_write_ivar, pci_write_ivar),
145 DEVMETHOD(bus_driver_added, pci_driver_added),
146 DEVMETHOD(bus_setup_intr, pci_setup_intr),
147 DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
148
149 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
150 DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
151 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
152 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
153 DEVMETHOD(bus_delete_resource, pci_delete_resource),
154 DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
155 DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource),
156 DEVMETHOD(bus_release_resource, pci_release_resource),
157 DEVMETHOD(bus_activate_resource, pci_activate_resource),
158 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
159 DEVMETHOD(bus_child_deleted, pci_child_deleted),
160 DEVMETHOD(bus_child_detached, pci_child_detached),
161 DEVMETHOD(bus_child_pnpinfo_str, pci_child_pnpinfo_str_method),
162 DEVMETHOD(bus_child_location_str, pci_child_location_str_method),
163 DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
164 DEVMETHOD(bus_suspend_child, pci_suspend_child),
165 DEVMETHOD(bus_resume_child, pci_resume_child),
166 DEVMETHOD(bus_rescan, pci_rescan_method),
167
168 /* PCI interface */
169 DEVMETHOD(pci_read_config, pci_read_config_method),
170 DEVMETHOD(pci_write_config, pci_write_config_method),
171 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
172 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
173 DEVMETHOD(pci_enable_io, pci_enable_io_method),
174 DEVMETHOD(pci_disable_io, pci_disable_io_method),
175 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
176 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
177 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
178 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
179 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
180 DEVMETHOD(pci_find_cap, pci_find_cap_method),
181 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
182 DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
183 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
184 DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
185 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
186 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
187 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
188 DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
189 DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
190 DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
191 DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
192 DEVMETHOD(pci_release_msi, pci_release_msi_method),
193 DEVMETHOD(pci_msi_count, pci_msi_count_method),
194 DEVMETHOD(pci_msix_count, pci_msix_count_method),
195 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
196 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
197 DEVMETHOD(pci_get_id, pci_get_id_method),
198 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
199 DEVMETHOD(pci_child_added, pci_child_added_method),
200 #ifdef PCI_IOV
201 DEVMETHOD(pci_iov_attach_name, pci_iov_attach_method),
202 DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
203 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
204 #endif
205
206 DEVMETHOD_END
207 };
208
209 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
210
211 static devclass_t pci_devclass;
212 DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, NULL);
213 MODULE_VERSION(pci, 1);
214
215 static char *pci_vendordata;
216 static size_t pci_vendordata_size;
217
218 struct pci_quirk {
219 uint32_t devid; /* Vendor/device of the card */
220 int type;
221 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
222 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
223 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
224 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
225 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
226 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
227 int arg1;
228 int arg2;
229 };
230
231 static const struct pci_quirk pci_quirks[] = {
232 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
233 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
234 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
235 /* As does the Serverworks OSB4 (the SMBus mapping register) */
236 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },
237
238 /*
239 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
240 * or the CMIC-SL (AKA ServerWorks GC_LE).
241 */
242 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
243 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
244
245 /*
246 * MSI doesn't work on earlier Intel chipsets including
247 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
248 */
249 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
250 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
251 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
252 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
253 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
254 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
255 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
256
257 /*
258 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX
259 * bridge.
260 */
261 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },
262
263 /*
264 * MSI-X allocation doesn't work properly for devices passed through
265 * by VMware up to at least ESXi 5.1.
266 */
267 { 0x079015ad, PCI_QUIRK_DISABLE_MSIX, 0, 0 }, /* PCI/PCI-X */
268 { 0x07a015ad, PCI_QUIRK_DISABLE_MSIX, 0, 0 }, /* PCIe */
269
270 /*
271 * Some virtualization environments emulate an older chipset
272 * but support MSI just fine. QEMU uses the Intel 82440.
273 */
274 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },
275
276 /*
277 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
278 * controller depending on SoftPciRst register (PM_IO 0x55 [7]).
279 * It prevents us from attaching hpet(4) when the bit is unset.
280 * Note this quirk only affects SB600 revision A13 and earlier.
281 * For SB600 A21 and later, firmware must set the bit to hide it.
282 * For SB700 and later, it is unused and hardcoded to zero.
283 */
284 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },
285
286 /*
287 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
288 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
289 * of the command register is set.
290 */
291 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
292 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
293 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
294 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
295 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
296
297 /*
298 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
299 * issue MSI interrupts with PCIM_CMD_INTxDIS set either.
300 */
301 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
302 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
303 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
304 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
305 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
306 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */
307
308 { 0 }
309 };
310
311 /* map register information */
312 #define PCI_MAPMEM 0x01 /* memory map */
313 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */
314 #define PCI_MAPPORT 0x04 /* port map */
315
316 struct devlist pci_devq;
317 uint32_t pci_generation;
318 uint32_t pci_numdevs = 0;
319 static int pcie_chipset, pcix_chipset;
320
321 /* sysctl vars */
322 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD, 0, "PCI bus tuning parameters");
323
324 static int pci_enable_io_modes = 1;
325 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
326 &pci_enable_io_modes, 1,
327 "Enable I/O and memory bits in the config register. Some BIOSes do not"
328 " enable these bits correctly. We'd like to do this all the time, but"
329 " there are some peripherals that this causes problems with.");
330
331 static int pci_do_realloc_bars = 0;
332 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
333 &pci_do_realloc_bars, 0,
334 "Attempt to allocate a new range for any BARs whose original "
335 "firmware-assigned ranges fail to allocate during the initial device scan.");
336
337 static int pci_do_power_nodriver = 0;
338 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
339 &pci_do_power_nodriver, 0,
340 "Place a function into D3 state when no driver attaches to it. 0 means"
341 " disable. 1 means conservatively place devices into D3 state. 2 means"
342 " aggressively place devices into D3 state. 3 means put absolutely"
343 " everything in D3 state.");
344
345 int pci_do_power_resume = 1;
346 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
347 &pci_do_power_resume, 1,
348 "Transition from D3 -> D0 on resume.");
349
350 int pci_do_power_suspend = 1;
351 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
352 &pci_do_power_suspend, 1,
353 "Transition from D0 -> D3 on suspend.");
354
355 static int pci_do_msi = 1;
356 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
357 "Enable support for MSI interrupts");
358
359 static int pci_do_msix = 1;
360 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
361 "Enable support for MSI-X interrupts");
362
363 static int pci_msix_rewrite_table = 0;
364 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
365 &pci_msix_rewrite_table, 0,
366 "Rewrite entire MSI-X table when updating MSI-X entries");
367
368 static int pci_honor_msi_blacklist = 1;
369 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
370 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
371
372 #if defined(__i386__) || defined(__amd64__)
373 static int pci_usb_takeover = 1;
374 #else
375 static int pci_usb_takeover = 0;
376 #endif
377 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
378 &pci_usb_takeover, 1,
379 "Enable early takeover of USB controllers. Disable this if you depend on"
380 " BIOS emulation of USB devices, that is you use USB devices (like"
381 " keyboard or mouse) but do not load USB drivers");
382
383 static int pci_clear_bars;
384 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
385 "Ignore firmware-assigned resources for BARs.");
386
387 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
388 static int pci_clear_buses;
389 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
390 "Ignore firmware-assigned bus numbers.");
391 #endif
392
393 static int pci_enable_ari = 1;
394 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
395 0, "Enable support for PCIe Alternative RID Interpretation");
396
397 static int
398 pci_has_quirk(uint32_t devid, int quirk)
399 {
400 const struct pci_quirk *q;
401
402 for (q = &pci_quirks[0]; q->devid; q++) {
403 if (q->devid == devid && q->type == quirk)
404 return (1);
405 }
406 return (0);
407 }
408
409 /* Find a device_t by bus/slot/function in domain 0 */
410
411 device_t
412 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
413 {
414
415 return (pci_find_dbsf(0, bus, slot, func));
416 }
417
418 /* Find a device_t by domain/bus/slot/function */
419
420 device_t
421 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
422 {
423 struct pci_devinfo *dinfo;
424
425 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
426 if ((dinfo->cfg.domain == domain) &&
427 (dinfo->cfg.bus == bus) &&
428 (dinfo->cfg.slot == slot) &&
429 (dinfo->cfg.func == func)) {
430 return (dinfo->cfg.dev);
431 }
432 }
433
434 return (NULL);
435 }
436
437 /* Find a device_t by vendor/device ID */
438
439 device_t
440 pci_find_device(uint16_t vendor, uint16_t device)
441 {
442 struct pci_devinfo *dinfo;
443
444 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
445 if ((dinfo->cfg.vendor == vendor) &&
446 (dinfo->cfg.device == device)) {
447 return (dinfo->cfg.dev);
448 }
449 }
450
451 return (NULL);
452 }
453
454 device_t
455 pci_find_class(uint8_t class, uint8_t subclass)
456 {
457 struct pci_devinfo *dinfo;
458
459 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
460 if (dinfo->cfg.baseclass == class &&
461 dinfo->cfg.subclass == subclass) {
462 return (dinfo->cfg.dev);
463 }
464 }
465
466 return (NULL);
467 }
468
469 static int
470 pci_printf(pcicfgregs *cfg, const char *fmt, ...)
471 {
472 va_list ap;
473 int retval;
474
475 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
476 cfg->func);
477 va_start(ap, fmt);
478 retval += vprintf(fmt, ap);
479 va_end(ap);
480 return (retval);
481 }
482
483 /* return base address of memory or port map */
484
485 static pci_addr_t
486 pci_mapbase(uint64_t mapreg)
487 {
488
489 if (PCI_BAR_MEM(mapreg))
490 return (mapreg & PCIM_BAR_MEM_BASE);
491 else
492 return (mapreg & PCIM_BAR_IO_BASE);
493 }
494
495 /* return map type of memory or port map */
496
497 static const char *
498 pci_maptype(uint64_t mapreg)
499 {
500
501 if (PCI_BAR_IO(mapreg))
502 return ("I/O Port");
503 if (mapreg & PCIM_BAR_MEM_PREFETCH)
504 return ("Prefetchable Memory");
505 return ("Memory");
506 }
507
508 /* return log2 of map size decoded for memory or port map */
509
510 int
511 pci_mapsize(uint64_t testval)
512 {
513 int ln2size;
514
515 testval = pci_mapbase(testval);
516 ln2size = 0;
517 if (testval != 0) {
518 while ((testval & 1) == 0)
519 {
520 ln2size++;
521 testval >>= 1;
522 }
523 }
524 return (ln2size);
525 }
526
527 /* return base address of device ROM */
528
529 static pci_addr_t
530 pci_rombase(uint64_t mapreg)
531 {
532
533 return (mapreg & PCIM_BIOS_ADDR_MASK);
534 }
535
536 /* return log2 of map size decided for device ROM */
537
538 static int
539 pci_romsize(uint64_t testval)
540 {
541 int ln2size;
542
543 testval = pci_rombase(testval);
544 ln2size = 0;
545 if (testval != 0) {
546 while ((testval & 1) == 0)
547 {
548 ln2size++;
549 testval >>= 1;
550 }
551 }
552 return (ln2size);
553 }
554
555 /* return log2 of address range supported by map register */
556
557 static int
558 pci_maprange(uint64_t mapreg)
559 {
560 int ln2range = 0;
561
562 if (PCI_BAR_IO(mapreg))
563 ln2range = 32;
564 else
565 switch (mapreg & PCIM_BAR_MEM_TYPE) {
566 case PCIM_BAR_MEM_32:
567 ln2range = 32;
568 break;
569 case PCIM_BAR_MEM_1MB:
570 ln2range = 20;
571 break;
572 case PCIM_BAR_MEM_64:
573 ln2range = 64;
574 break;
575 }
576 return (ln2range);
577 }
578
579 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */
580
581 static void
582 pci_fixancient(pcicfgregs *cfg)
583 {
584 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
585 return;
586
587 /* PCI to PCI bridges use header type 1 */
588 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
589 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
590 }
591
592 /* extract header type specific config data */
593
594 static void
595 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
596 {
597 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
598 switch (cfg->hdrtype & PCIM_HDRTYPE) {
599 case PCIM_HDRTYPE_NORMAL:
600 cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
601 cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
602 cfg->mingnt = REG(PCIR_MINGNT, 1);
603 cfg->maxlat = REG(PCIR_MAXLAT, 1);
604 cfg->nummaps = PCI_MAXMAPS_0;
605 break;
606 case PCIM_HDRTYPE_BRIDGE:
607 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
608 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
609 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
610 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
611 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
612 cfg->nummaps = PCI_MAXMAPS_1;
613 break;
614 case PCIM_HDRTYPE_CARDBUS:
615 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
616 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
617 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
618 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
619 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
620 cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
621 cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
622 cfg->nummaps = PCI_MAXMAPS_2;
623 break;
624 }
625 #undef REG
626 }
627
628 /* read configuration header into pcicfgregs structure */
629 struct pci_devinfo *
630 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
631 {
632 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
633 uint16_t vid, did;
634
635 vid = REG(PCIR_VENDOR, 2);
636 did = REG(PCIR_DEVICE, 2);
637 if (vid != 0xffff)
638 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));
639
640 return (NULL);
641 }
642
643 struct pci_devinfo *
644 pci_alloc_devinfo_method(device_t dev)
645 {
646
647 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
648 M_WAITOK | M_ZERO));
649 }
650
651 static struct pci_devinfo *
652 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
653 uint16_t vid, uint16_t did)
654 {
655 struct pci_devinfo *devlist_entry;
656 pcicfgregs *cfg;
657
658 devlist_entry = PCI_ALLOC_DEVINFO(bus);
659
660 cfg = &devlist_entry->cfg;
661
662 cfg->domain = d;
663 cfg->bus = b;
664 cfg->slot = s;
665 cfg->func = f;
666 cfg->vendor = vid;
667 cfg->device = did;
668 cfg->cmdreg = REG(PCIR_COMMAND, 2);
669 cfg->statreg = REG(PCIR_STATUS, 2);
670 cfg->baseclass = REG(PCIR_CLASS, 1);
671 cfg->subclass = REG(PCIR_SUBCLASS, 1);
672 cfg->progif = REG(PCIR_PROGIF, 1);
673 cfg->revid = REG(PCIR_REVID, 1);
674 cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
675 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
676 cfg->lattimer = REG(PCIR_LATTIMER, 1);
677 cfg->intpin = REG(PCIR_INTPIN, 1);
678 cfg->intline = REG(PCIR_INTLINE, 1);
679
680 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
681 cfg->hdrtype &= ~PCIM_MFDEV;
682 STAILQ_INIT(&cfg->maps);
683
684 cfg->iov = NULL;
685
686 pci_fixancient(cfg);
687 pci_hdrtypedata(pcib, b, s, f, cfg);
688
689 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
690 pci_read_cap(pcib, cfg);
691
692 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);
693
694 devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
695 devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
696 devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
697 devlist_entry->conf.pc_sel.pc_func = cfg->func;
698 devlist_entry->conf.pc_hdr = cfg->hdrtype;
699
700 devlist_entry->conf.pc_subvendor = cfg->subvendor;
701 devlist_entry->conf.pc_subdevice = cfg->subdevice;
702 devlist_entry->conf.pc_vendor = cfg->vendor;
703 devlist_entry->conf.pc_device = cfg->device;
704
705 devlist_entry->conf.pc_class = cfg->baseclass;
706 devlist_entry->conf.pc_subclass = cfg->subclass;
707 devlist_entry->conf.pc_progif = cfg->progif;
708 devlist_entry->conf.pc_revid = cfg->revid;
709
710 pci_numdevs++;
711 pci_generation++;
712
713 return (devlist_entry);
714 }
715 #undef REG
716
717 static void
718 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
719 {
720 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
721 cfg->ea.ea_location + (n), w)
722 int num_ent;
723 int ptr;
724 int a, b;
725 uint32_t val;
726 int ent_size;
727 uint32_t dw[4];
728 uint64_t base, max_offset;
729 struct pci_ea_entry *eae;
730
731 if (cfg->ea.ea_location == 0)
732 return;
733
734 STAILQ_INIT(&cfg->ea.ea_entries);
735
736 /* Determine the number of entries */
737 num_ent = REG(PCIR_EA_NUM_ENT, 2);
738 num_ent &= PCIM_EA_NUM_ENT_MASK;
739
740 /* Find the first entry to care of */
741 ptr = PCIR_EA_FIRST_ENT;
742
743 /* Skip DWORD 2 for type 1 functions */
744 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
745 ptr += 4;
746
747 for (a = 0; a < num_ent; a++) {
748
749 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO);
750 eae->eae_cfg_offset = cfg->ea.ea_location + ptr;
751
752 /* Read a number of dwords in the entry */
753 val = REG(ptr, 4);
754 ptr += 4;
755 ent_size = (val & PCIM_EA_ES);
756
757 for (b = 0; b < ent_size; b++) {
758 dw[b] = REG(ptr, 4);
759 ptr += 4;
760 }
761
762 eae->eae_flags = val;
763 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;
764
765 base = dw[0] & PCIM_EA_FIELD_MASK;
766 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK;
767 b = 2;
768 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
769 base |= (uint64_t)dw[b] << 32UL;
770 b++;
771 }
772 if (((dw[1] & PCIM_EA_IS_64) != 0)
773 && (b < ent_size)) {
774 max_offset |= (uint64_t)dw[b] << 32UL;
775 b++;
776 }
777
778 eae->eae_base = base;
779 eae->eae_max_offset = max_offset;
780
781 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);
782
783 if (bootverbose) {
784 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
785 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
786 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
787 }
788 }
789 }
790 #undef REG
791
792 static void
793 pci_read_cap(device_t pcib, pcicfgregs *cfg)
794 {
795 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
796 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
797 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
798 uint64_t addr;
799 #endif
800 uint32_t val;
801 int ptr, nextptr, ptrptr;
802
803 switch (cfg->hdrtype & PCIM_HDRTYPE) {
804 case PCIM_HDRTYPE_NORMAL:
805 case PCIM_HDRTYPE_BRIDGE:
806 ptrptr = PCIR_CAP_PTR;
807 break;
808 case PCIM_HDRTYPE_CARDBUS:
809 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
810 break;
811 default:
812 return; /* no extended capabilities support */
813 }
814 nextptr = REG(ptrptr, 1); /* sanity check? */
815
816 /*
817 * Read capability entries.
818 */
819 while (nextptr != 0) {
820 /* Sanity check */
821 if (nextptr > 255) {
822 printf("illegal PCI extended capability offset %d\n",
823 nextptr);
824 return;
825 }
826 /* Find the next entry */
827 ptr = nextptr;
828 nextptr = REG(ptr + PCICAP_NEXTPTR, 1);
829
830 /* Process this entry */
831 switch (REG(ptr + PCICAP_ID, 1)) {
832 case PCIY_PMG: /* PCI power management */
833 if (cfg->pp.pp_cap == 0) {
834 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
835 cfg->pp.pp_status = ptr + PCIR_POWER_STATUS;
836 cfg->pp.pp_bse = ptr + PCIR_POWER_BSE;
837 if ((nextptr - ptr) > PCIR_POWER_DATA)
838 cfg->pp.pp_data = ptr + PCIR_POWER_DATA;
839 }
840 break;
841 case PCIY_HT: /* HyperTransport */
842 /* Determine HT-specific capability type. */
843 val = REG(ptr + PCIR_HT_COMMAND, 2);
844
845 if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
846 cfg->ht.ht_slave = ptr;
847
848 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
849 switch (val & PCIM_HTCMD_CAP_MASK) {
850 case PCIM_HTCAP_MSI_MAPPING:
851 if (!(val & PCIM_HTCMD_MSI_FIXED)) {
852 /* Sanity check the mapping window. */
853 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
854 4);
855 addr <<= 32;
856 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
857 4);
858 if (addr != MSI_INTEL_ADDR_BASE)
859 device_printf(pcib,
860 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
861 cfg->domain, cfg->bus,
862 cfg->slot, cfg->func,
863 (long long)addr);
864 } else
865 addr = MSI_INTEL_ADDR_BASE;
866
867 cfg->ht.ht_msimap = ptr;
868 cfg->ht.ht_msictrl = val;
869 cfg->ht.ht_msiaddr = addr;
870 break;
871 }
872 #endif
873 break;
874 case PCIY_MSI: /* PCI MSI */
875 cfg->msi.msi_location = ptr;
876 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
877 cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl &
878 PCIM_MSICTRL_MMC_MASK)>>1);
879 break;
880 case PCIY_MSIX: /* PCI MSI-X */
881 cfg->msix.msix_location = ptr;
882 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
883 cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl &
884 PCIM_MSIXCTRL_TABLE_SIZE) + 1;
885 val = REG(ptr + PCIR_MSIX_TABLE, 4);
886 cfg->msix.msix_table_bar = PCIR_BAR(val &
887 PCIM_MSIX_BIR_MASK);
888 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
889 val = REG(ptr + PCIR_MSIX_PBA, 4);
890 cfg->msix.msix_pba_bar = PCIR_BAR(val &
891 PCIM_MSIX_BIR_MASK);
892 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
893 break;
894 case PCIY_VPD: /* PCI Vital Product Data */
895 cfg->vpd.vpd_reg = ptr;
896 break;
897 case PCIY_SUBVENDOR:
898 /* Should always be true. */
899 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
900 PCIM_HDRTYPE_BRIDGE) {
901 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
902 cfg->subvendor = val & 0xffff;
903 cfg->subdevice = val >> 16;
904 }
905 break;
906 case PCIY_PCIX: /* PCI-X */
907 /*
908 * Assume we have a PCI-X chipset if we have
909 * at least one PCI-PCI bridge with a PCI-X
910 * capability. Note that some systems with
911 * PCI-express or HT chipsets might match on
912 * this check as well.
913 */
914 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
915 PCIM_HDRTYPE_BRIDGE)
916 pcix_chipset = 1;
917 cfg->pcix.pcix_location = ptr;
918 break;
919 case PCIY_EXPRESS: /* PCI-express */
920 /*
921 * Assume we have a PCI-express chipset if we have
922 * at least one PCI-express device.
923 */
924 pcie_chipset = 1;
925 cfg->pcie.pcie_location = ptr;
926 val = REG(ptr + PCIER_FLAGS, 2);
927 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
928 break;
929 case PCIY_EA: /* Enhanced Allocation */
930 cfg->ea.ea_location = ptr;
931 pci_ea_fill_info(pcib, cfg);
932 break;
933 default:
934 break;
935 }
936 }
937
938 #if defined(__powerpc__)
939 /*
940 * Enable the MSI mapping window for all HyperTransport
941 * slaves. PCI-PCI bridges have their windows enabled via
942 * PCIB_MAP_MSI().
943 */
944 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
945 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
946 device_printf(pcib,
947 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
948 cfg->domain, cfg->bus, cfg->slot, cfg->func);
949 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
950 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
951 2);
952 }
953 #endif
954 /* REG and WREG use carry through to next functions */
955 }
956
957 /*
958 * PCI Vital Product Data
959 */
960
961 #define PCI_VPD_TIMEOUT 1000000
962
963 static int
964 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data)
965 {
966 int count = PCI_VPD_TIMEOUT;
967
968 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
969
970 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);
971
972 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
973 if (--count < 0)
974 return (ENXIO);
975 DELAY(1); /* limit looping */
976 }
977 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));
978
979 return (0);
980 }
981
982 #if 0
983 static int
984 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
985 {
986 int count = PCI_VPD_TIMEOUT;
987
988 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
989
990 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
991 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2);
992 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
993 if (--count < 0)
994 return (ENXIO);
995 DELAY(1); /* limit looping */
996 }
997
998 return (0);
999 }
1000 #endif
1001
1002 #undef PCI_VPD_TIMEOUT
1003
1004 struct vpd_readstate {
1005 device_t pcib;
1006 pcicfgregs *cfg;
1007 uint32_t val;
1008 int bytesinval;
1009 int off;
1010 uint8_t cksum;
1011 };
1012
1013 static int
1014 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data)
1015 {
1016 uint32_t reg;
1017 uint8_t byte;
1018
1019 if (vrs->bytesinval == 0) {
1020 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®))
1021 return (ENXIO);
1022 vrs->val = le32toh(reg);
1023 vrs->off += 4;
1024 byte = vrs->val & 0xff;
1025 vrs->bytesinval = 3;
1026 } else {
1027 vrs->val = vrs->val >> 8;
1028 byte = vrs->val & 0xff;
1029 vrs->bytesinval--;
1030 }
1031
1032 vrs->cksum += byte;
1033 *data = byte;
1034 return (0);
1035 }
1036
1037 static void
1038 pci_read_vpd(device_t pcib, pcicfgregs *cfg)
1039 {
1040 struct vpd_readstate vrs;
1041 int state;
1042 int name;
1043 int remain;
1044 int i;
1045 int alloc, off; /* alloc/off for RO/W arrays */
1046 int cksumvalid;
1047 int dflen;
1048 uint8_t byte;
1049 uint8_t byte2;
1050
1051 /* init vpd reader */
1052 vrs.bytesinval = 0;
1053 vrs.off = 0;
1054 vrs.pcib = pcib;
1055 vrs.cfg = cfg;
1056 vrs.cksum = 0;
1057
1058 state = 0;
1059 name = remain = i = 0; /* shut up stupid gcc */
1060 alloc = off = 0; /* shut up stupid gcc */
1061 dflen = 0; /* shut up stupid gcc */
1062 cksumvalid = -1;
1063 while (state >= 0) {
1064 if (vpd_nextbyte(&vrs, &byte)) {
1065 state = -2;
1066 break;
1067 }
1068 #if 0
1069 printf("vpd: val: %#x, off: %d, bytesinval: %d, byte: %#hhx, " \
1070 "state: %d, remain: %d, name: %#x, i: %d\n", vrs.val,
1071 vrs.off, vrs.bytesinval, byte, state, remain, name, i);
1072 #endif
1073 switch (state) {
1074 case 0: /* item name */
1075 if (byte & 0x80) {
1076 if (vpd_nextbyte(&vrs, &byte2)) {
1077 state = -2;
1078 break;
1079 }
1080 remain = byte2;
1081 if (vpd_nextbyte(&vrs, &byte2)) {
1082 state = -2;
1083 break;
1084 }
1085 remain |= byte2 << 8;
1086 if (remain > (0x7f*4 - vrs.off)) {
1087 state = -1;
1088 pci_printf(cfg,
1089 "invalid VPD data, remain %#x\n",
1090 remain);
1091 }
1092 name = byte & 0x7f;
1093 } else {
1094 remain = byte & 0x7;
1095 name = (byte >> 3) & 0xf;
1096 }
1097 switch (name) {
1098 case 0x2: /* String */
1099 cfg->vpd.vpd_ident = malloc(remain + 1,
1100 M_DEVBUF, M_WAITOK);
1101 i = 0;
1102 state = 1;
1103 break;
1104 case 0xf: /* End */
1105 state = -1;
1106 break;
1107 case 0x10: /* VPD-R */
1108 alloc = 8;
1109 off = 0;
1110 cfg->vpd.vpd_ros = malloc(alloc *
1111 sizeof(*cfg->vpd.vpd_ros), M_DEVBUF,
1112 M_WAITOK | M_ZERO);
1113 state = 2;
1114 break;
1115 case 0x11: /* VPD-W */
1116 alloc = 8;
1117 off = 0;
1118 cfg->vpd.vpd_w = malloc(alloc *
1119 sizeof(*cfg->vpd.vpd_w), M_DEVBUF,
1120 M_WAITOK | M_ZERO);
1121 state = 5;
1122 break;
1123 default: /* Invalid data, abort */
1124 state = -1;
1125 break;
1126 }
1127 break;
1128
1129 case 1: /* Identifier String */
1130 cfg->vpd.vpd_ident[i++] = byte;
1131 remain--;
1132 if (remain == 0) {
1133 cfg->vpd.vpd_ident[i] = '\0';
1134 state = 0;
1135 }
1136 break;
1137
1138 case 2: /* VPD-R Keyword Header */
1139 if (off == alloc) {
1140 cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros,
1141 (alloc *= 2) * sizeof(*cfg->vpd.vpd_ros),
1142 M_DEVBUF, M_WAITOK | M_ZERO);
1143 }
1144 cfg->vpd.vpd_ros[off].keyword[0] = byte;
1145 if (vpd_nextbyte(&vrs, &byte2)) {
1146 state = -2;
1147 break;
1148 }
1149 cfg->vpd.vpd_ros[off].keyword[1] = byte2;
1150 if (vpd_nextbyte(&vrs, &byte2)) {
1151 state = -2;
1152 break;
1153 }
1154 cfg->vpd.vpd_ros[off].len = dflen = byte2;
1155 if (dflen == 0 &&
1156 strncmp(cfg->vpd.vpd_ros[off].keyword, "RV",
1157 2) == 0) {
1158 /*
1159 * if this happens, we can't trust the rest
1160 * of the VPD.
1161 */
1162 pci_printf(cfg, "bad keyword length: %d\n",
1163 dflen);
1164 cksumvalid = 0;
1165 state = -1;
1166 break;
1167 } else if (dflen == 0) {
1168 cfg->vpd.vpd_ros[off].value = malloc(1 *
1169 sizeof(*cfg->vpd.vpd_ros[off].value),
1170 M_DEVBUF, M_WAITOK);
1171 cfg->vpd.vpd_ros[off].value[0] = '\x00';
1172 } else
1173 cfg->vpd.vpd_ros[off].value = malloc(
1174 (dflen + 1) *
1175 sizeof(*cfg->vpd.vpd_ros[off].value),
1176 M_DEVBUF, M_WAITOK);
1177 remain -= 3;
1178 i = 0;
1179 /* keep in sync w/ state 3's transistions */
1180 if (dflen == 0 && remain == 0)
1181 state = 0;
1182 else if (dflen == 0)
1183 state = 2;
1184 else
1185 state = 3;
1186 break;
1187
1188 case 3: /* VPD-R Keyword Value */
1189 cfg->vpd.vpd_ros[off].value[i++] = byte;
1190 if (strncmp(cfg->vpd.vpd_ros[off].keyword,
1191 "RV", 2) == 0 && cksumvalid == -1) {
1192 if (vrs.cksum == 0)
1193 cksumvalid = 1;
1194 else {
1195 if (bootverbose)
1196 pci_printf(cfg,
1197 "bad VPD cksum, remain %hhu\n",
1198 vrs.cksum);
1199 cksumvalid = 0;
1200 state = -1;
1201 break;
1202 }
1203 }
1204 dflen--;
1205 remain--;
1206 /* keep in sync w/ state 2's transistions */
1207 if (dflen == 0)
1208 cfg->vpd.vpd_ros[off++].value[i++] = '\0';
1209 if (dflen == 0 && remain == 0) {
1210 cfg->vpd.vpd_rocnt = off;
1211 cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros,
1212 off * sizeof(*cfg->vpd.vpd_ros),
1213 M_DEVBUF, M_WAITOK | M_ZERO);
1214 state = 0;
1215 } else if (dflen == 0)
1216 state = 2;
1217 break;
1218
1219 case 4:
1220 remain--;
1221 if (remain == 0)
1222 state = 0;
1223 break;
1224
1225 case 5: /* VPD-W Keyword Header */
1226 if (off == alloc) {
1227 cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w,
1228 (alloc *= 2) * sizeof(*cfg->vpd.vpd_w),
1229 M_DEVBUF, M_WAITOK | M_ZERO);
1230 }
1231 cfg->vpd.vpd_w[off].keyword[0] = byte;
1232 if (vpd_nextbyte(&vrs, &byte2)) {
1233 state = -2;
1234 break;
1235 }
1236 cfg->vpd.vpd_w[off].keyword[1] = byte2;
1237 if (vpd_nextbyte(&vrs, &byte2)) {
1238 state = -2;
1239 break;
1240 }
1241 cfg->vpd.vpd_w[off].len = dflen = byte2;
1242 cfg->vpd.vpd_w[off].start = vrs.off - vrs.bytesinval;
1243 cfg->vpd.vpd_w[off].value = malloc((dflen + 1) *
1244 sizeof(*cfg->vpd.vpd_w[off].value),
1245 M_DEVBUF, M_WAITOK);
1246 remain -= 3;
1247 i = 0;
1248 /* keep in sync w/ state 6's transistions */
1249 if (dflen == 0 && remain == 0)
1250 state = 0;
1251 else if (dflen == 0)
1252 state = 5;
1253 else
1254 state = 6;
1255 break;
1256
1257 case 6: /* VPD-W Keyword Value */
1258 cfg->vpd.vpd_w[off].value[i++] = byte;
1259 dflen--;
1260 remain--;
1261 /* keep in sync w/ state 5's transistions */
1262 if (dflen == 0)
1263 cfg->vpd.vpd_w[off++].value[i++] = '\0';
1264 if (dflen == 0 && remain == 0) {
1265 cfg->vpd.vpd_wcnt = off;
1266 cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w,
1267 off * sizeof(*cfg->vpd.vpd_w),
1268 M_DEVBUF, M_WAITOK | M_ZERO);
1269 state = 0;
1270 } else if (dflen == 0)
1271 state = 5;
1272 break;
1273
1274 default:
1275 pci_printf(cfg, "invalid state: %d\n", state);
1276 state = -1;
1277 break;
1278 }
1279 }
1280
1281 if (cksumvalid == 0 || state < -1) {
1282 /* read-only data bad, clean up */
1283 if (cfg->vpd.vpd_ros != NULL) {
1284 for (off = 0; cfg->vpd.vpd_ros[off].value; off++)
1285 free(cfg->vpd.vpd_ros[off].value, M_DEVBUF);
1286 free(cfg->vpd.vpd_ros, M_DEVBUF);
1287 cfg->vpd.vpd_ros = NULL;
1288 }
1289 }
1290 if (state < -1) {
1291 /* I/O error, clean up */
1292 pci_printf(cfg, "failed to read VPD data.\n");
1293 if (cfg->vpd.vpd_ident != NULL) {
1294 free(cfg->vpd.vpd_ident, M_DEVBUF);
1295 cfg->vpd.vpd_ident = NULL;
1296 }
1297 if (cfg->vpd.vpd_w != NULL) {
1298 for (off = 0; cfg->vpd.vpd_w[off].value; off++)
1299 free(cfg->vpd.vpd_w[off].value, M_DEVBUF);
1300 free(cfg->vpd.vpd_w, M_DEVBUF);
1301 cfg->vpd.vpd_w = NULL;
1302 }
1303 }
1304 cfg->vpd.vpd_cached = 1;
1305 #undef REG
1306 #undef WREG
1307 }
1308
1309 int
1310 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
1311 {
1312 struct pci_devinfo *dinfo = device_get_ivars(child);
1313 pcicfgregs *cfg = &dinfo->cfg;
1314
1315 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1316 pci_read_vpd(device_get_parent(dev), cfg);
1317
1318 *identptr = cfg->vpd.vpd_ident;
1319
1320 if (*identptr == NULL)
1321 return (ENXIO);
1322
1323 return (0);
1324 }
1325
1326 int
1327 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
1328 const char **vptr)
1329 {
1330 struct pci_devinfo *dinfo = device_get_ivars(child);
1331 pcicfgregs *cfg = &dinfo->cfg;
1332 int i;
1333
1334 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1335 pci_read_vpd(device_get_parent(dev), cfg);
1336
1337 for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
1338 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
1339 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
1340 *vptr = cfg->vpd.vpd_ros[i].value;
1341 return (0);
1342 }
1343
1344 *vptr = NULL;
1345 return (ENXIO);
1346 }
1347
1348 struct pcicfg_vpd *
1349 pci_fetch_vpd_list(device_t dev)
1350 {
1351 struct pci_devinfo *dinfo = device_get_ivars(dev);
1352 pcicfgregs *cfg = &dinfo->cfg;
1353
1354 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1355 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
1356 return (&cfg->vpd);
1357 }
1358
1359 /*
1360 * Find the requested HyperTransport capability and return the offset
1361 * in configuration space via the pointer provided. The function
1362 * returns 0 on success and an error code otherwise.
1363 */
1364 int
1365 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
1366 {
1367 int ptr, error;
1368 uint16_t val;
1369
1370 error = pci_find_cap(child, PCIY_HT, &ptr);
1371 if (error)
1372 return (error);
1373
1374 /*
1375 * Traverse the capabilities list checking each HT capability
1376 * to see if it matches the requested HT capability.
1377 */
1378 for (;;) {
1379 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1380 if (capability == PCIM_HTCAP_SLAVE ||
1381 capability == PCIM_HTCAP_HOST)
1382 val &= 0xe000;
1383 else
1384 val &= PCIM_HTCMD_CAP_MASK;
1385 if (val == capability) {
1386 if (capreg != NULL)
1387 *capreg = ptr;
1388 return (0);
1389 }
1390
1391 /* Skip to the next HT capability. */
1392 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1393 break;
1394 }
1395
1396 return (ENOENT);
1397 }
1398
1399 /*
1400 * Find the next requested HyperTransport capability after start and return
1401 * the offset in configuration space via the pointer provided. The function
1402 * returns 0 on success and an error code otherwise.
1403 */
1404 int
1405 pci_find_next_htcap_method(device_t dev, device_t child, int capability,
1406 int start, int *capreg)
1407 {
1408 int ptr;
1409 uint16_t val;
1410
1411 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
1412 ("start capability is not HyperTransport capability"));
1413 ptr = start;
1414
1415 /*
1416 * Traverse the capabilities list checking each HT capability
1417 * to see if it matches the requested HT capability.
1418 */
1419 for (;;) {
1420 /* Skip to the next HT capability. */
1421 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1422 break;
1423
1424 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1425 if (capability == PCIM_HTCAP_SLAVE ||
1426 capability == PCIM_HTCAP_HOST)
1427 val &= 0xe000;
1428 else
1429 val &= PCIM_HTCMD_CAP_MASK;
1430 if (val == capability) {
1431 if (capreg != NULL)
1432 *capreg = ptr;
1433 return (0);
1434 }
1435 }
1436
1437 return (ENOENT);
1438 }
1439
1440 /*
1441 * Find the requested capability and return the offset in
1442 * configuration space via the pointer provided. The function returns
1443 * 0 on success and an error code otherwise.
1444 */
1445 int
1446 pci_find_cap_method(device_t dev, device_t child, int capability,
1447 int *capreg)
1448 {
1449 struct pci_devinfo *dinfo = device_get_ivars(child);
1450 pcicfgregs *cfg = &dinfo->cfg;
1451 uint32_t status;
1452 uint8_t ptr;
1453
1454 /*
1455 * Check the CAP_LIST bit of the PCI status register first.
1456 */
1457 status = pci_read_config(child, PCIR_STATUS, 2);
1458 if (!(status & PCIM_STATUS_CAPPRESENT))
1459 return (ENXIO);
1460
1461 /*
1462 * Determine the start pointer of the capabilities list.
1463 */
1464 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1465 case PCIM_HDRTYPE_NORMAL:
1466 case PCIM_HDRTYPE_BRIDGE:
1467 ptr = PCIR_CAP_PTR;
1468 break;
1469 case PCIM_HDRTYPE_CARDBUS:
1470 ptr = PCIR_CAP_PTR_2;
1471 break;
1472 default:
1473 /* XXX: panic? */
1474 return (ENXIO); /* no extended capabilities support */
1475 }
1476 ptr = pci_read_config(child, ptr, 1);
1477
1478 /*
1479 * Traverse the capabilities list.
1480 */
1481 while (ptr != 0) {
1482 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1483 if (capreg != NULL)
1484 *capreg = ptr;
1485 return (0);
1486 }
1487 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1488 }
1489
1490 return (ENOENT);
1491 }
1492
1493 /*
1494 * Find the next requested capability after start and return the offset in
1495 * configuration space via the pointer provided. The function returns
1496 * 0 on success and an error code otherwise.
1497 */
1498 int
1499 pci_find_next_cap_method(device_t dev, device_t child, int capability,
1500 int start, int *capreg)
1501 {
1502 uint8_t ptr;
1503
1504 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
1505 ("start capability is not expected capability"));
1506
1507 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
1508 while (ptr != 0) {
1509 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1510 if (capreg != NULL)
1511 *capreg = ptr;
1512 return (0);
1513 }
1514 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1515 }
1516
1517 return (ENOENT);
1518 }
1519
1520 /*
1521 * Find the requested extended capability and return the offset in
1522 * configuration space via the pointer provided. The function returns
1523 * 0 on success and an error code otherwise.
1524 */
1525 int
1526 pci_find_extcap_method(device_t dev, device_t child, int capability,
1527 int *capreg)
1528 {
1529 struct pci_devinfo *dinfo = device_get_ivars(child);
1530 pcicfgregs *cfg = &dinfo->cfg;
1531 uint32_t ecap;
1532 uint16_t ptr;
1533
1534 /* Only supported for PCI-express devices. */
1535 if (cfg->pcie.pcie_location == 0)
1536 return (ENXIO);
1537
1538 ptr = PCIR_EXTCAP;
1539 ecap = pci_read_config(child, ptr, 4);
1540 if (ecap == 0xffffffff || ecap == 0)
1541 return (ENOENT);
1542 for (;;) {
1543 if (PCI_EXTCAP_ID(ecap) == capability) {
1544 if (capreg != NULL)
1545 *capreg = ptr;
1546 return (0);
1547 }
1548 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1549 if (ptr == 0)
1550 break;
1551 ecap = pci_read_config(child, ptr, 4);
1552 }
1553
1554 return (ENOENT);
1555 }
1556
1557 /*
1558 * Find the next requested extended capability after start and return the
1559 * offset in configuration space via the pointer provided. The function
1560 * returns 0 on success and an error code otherwise.
1561 */
1562 int
1563 pci_find_next_extcap_method(device_t dev, device_t child, int capability,
1564 int start, int *capreg)
1565 {
1566 struct pci_devinfo *dinfo = device_get_ivars(child);
1567 pcicfgregs *cfg = &dinfo->cfg;
1568 uint32_t ecap;
1569 uint16_t ptr;
1570
1571 /* Only supported for PCI-express devices. */
1572 if (cfg->pcie.pcie_location == 0)
1573 return (ENXIO);
1574
1575 ecap = pci_read_config(child, start, 4);
1576 KASSERT(PCI_EXTCAP_ID(ecap) == capability,
1577 ("start extended capability is not expected capability"));
1578 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1579 while (ptr != 0) {
1580 ecap = pci_read_config(child, ptr, 4);
1581 if (PCI_EXTCAP_ID(ecap) == capability) {
1582 if (capreg != NULL)
1583 *capreg = ptr;
1584 return (0);
1585 }
1586 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1587 }
1588
1589 return (ENOENT);
1590 }
1591
1592 /*
1593 * Support for MSI-X message interrupts.
1594 */
1595 static void
1596 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
1597 {
1598 struct pci_devinfo *dinfo = device_get_ivars(dev);
1599 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1600 uint32_t offset;
1601
1602 KASSERT(msix->msix_table_len > index, ("bogus index"));
1603 offset = msix->msix_table_offset + index * 16;
1604 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
1605 bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
1606 bus_write_4(msix->msix_table_res, offset + 8, data);
1607 }
1608
1609 void
1610 pci_enable_msix_method(device_t dev, device_t child, u_int index,
1611 uint64_t address, uint32_t data)
1612 {
1613
1614 if (pci_msix_rewrite_table) {
1615 struct pci_devinfo *dinfo = device_get_ivars(child);
1616 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1617
1618 /*
1619 * Some VM hosts require MSIX to be disabled in the
1620 * control register before updating the MSIX table
1621 * entries are allowed. It is not enough to only
1622 * disable MSIX while updating a single entry. MSIX
1623 * must be disabled while updating all entries in the
1624 * table.
1625 */
1626 pci_write_config(child,
1627 msix->msix_location + PCIR_MSIX_CTRL,
1628 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
1629 pci_resume_msix(child);
1630 } else
1631 pci_write_msix_entry(child, index, address, data);
1632
1633 /* Enable MSI -> HT mapping. */
1634 pci_ht_map_msi(child, address);
1635 }
1636
1637 void
1638 pci_mask_msix(device_t dev, u_int index)
1639 {
1640 struct pci_devinfo *dinfo = device_get_ivars(dev);
1641 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1642 uint32_t offset, val;
1643
1644 KASSERT(msix->msix_msgnum > index, ("bogus index"));
1645 offset = msix->msix_table_offset + index * 16 + 12;
1646 val = bus_read_4(msix->msix_table_res, offset);
1647 if (!(val & PCIM_MSIX_VCTRL_MASK)) {
1648 val |= PCIM_MSIX_VCTRL_MASK;
1649 bus_write_4(msix->msix_table_res, offset, val);
1650 }
1651 }
1652
1653 void
1654 pci_unmask_msix(device_t dev, u_int index)
1655 {
1656 struct pci_devinfo *dinfo = device_get_ivars(dev);
1657 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1658 uint32_t offset, val;
1659
1660 KASSERT(msix->msix_table_len > index, ("bogus index"));
1661 offset = msix->msix_table_offset + index * 16 + 12;
1662 val = bus_read_4(msix->msix_table_res, offset);
1663 if (val & PCIM_MSIX_VCTRL_MASK) {
1664 val &= ~PCIM_MSIX_VCTRL_MASK;
1665 bus_write_4(msix->msix_table_res, offset, val);
1666 }
1667 }
1668
1669 int
1670 pci_pending_msix(device_t dev, u_int index)
1671 {
1672 struct pci_devinfo *dinfo = device_get_ivars(dev);
1673 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1674 uint32_t offset, bit;
1675
1676 KASSERT(msix->msix_table_len > index, ("bogus index"));
1677 offset = msix->msix_pba_offset + (index / 32) * 4;
1678 bit = 1 << index % 32;
1679 return (bus_read_4(msix->msix_pba_res, offset) & bit);
1680 }
1681
1682 /*
1683 * Restore MSI-X registers and table during resume. If MSI-X is
1684 * enabled then walk the virtual table to restore the actual MSI-X
1685 * table.
1686 */
1687 static void
1688 pci_resume_msix(device_t dev)
1689 {
1690 struct pci_devinfo *dinfo = device_get_ivars(dev);
1691 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1692 struct msix_table_entry *mte;
1693 struct msix_vector *mv;
1694 int i;
1695
1696 if (msix->msix_alloc > 0) {
1697 /* First, mask all vectors. */
1698 for (i = 0; i < msix->msix_msgnum; i++)
1699 pci_mask_msix(dev, i);
1700
1701 /* Second, program any messages with at least one handler. */
1702 for (i = 0; i < msix->msix_table_len; i++) {
1703 mte = &msix->msix_table[i];
1704 if (mte->mte_vector == 0 || mte->mte_handlers == 0)
1705 continue;
1706 mv = &msix->msix_vectors[mte->mte_vector - 1];
1707 pci_write_msix_entry(dev, i, mv->mv_address,
1708 mv->mv_data);
1709 pci_unmask_msix(dev, i);
1710 }
1711 }
1712 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
1713 msix->msix_ctrl, 2);
1714 }
1715
1716 /*
1717 * Attempt to allocate *count MSI-X messages. The actual number allocated is
1718 * returned in *count. After this function returns, each message will be
1719 * available to the driver as SYS_RES_IRQ resources starting at rid 1.
1720 */
1721 int
1722 pci_alloc_msix_method(device_t dev, device_t child, int *count)
1723 {
1724 struct pci_devinfo *dinfo = device_get_ivars(child);
1725 pcicfgregs *cfg = &dinfo->cfg;
1726 struct resource_list_entry *rle;
1727 int actual, error, i, irq, max;
1728
1729 /* Don't let count == 0 get us into trouble. */
1730 if (*count == 0)
1731 return (EINVAL);
1732
1733 /* If rid 0 is allocated, then fail. */
1734 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
1735 if (rle != NULL && rle->res != NULL)
1736 return (ENXIO);
1737
1738 /* Already have allocated messages? */
1739 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
1740 return (ENXIO);
1741
1742 /* If MSI-X is blacklisted for this system, fail. */
1743 if (pci_msix_blacklisted())
1744 return (ENXIO);
1745
1746 /* MSI-X capability present? */
1747 if (cfg->msix.msix_location == 0 || !pci_do_msix)
1748 return (ENODEV);
1749
1750 /* Make sure the appropriate BARs are mapped. */
1751 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1752 cfg->msix.msix_table_bar);
1753 if (rle == NULL || rle->res == NULL ||
1754 !(rman_get_flags(rle->res) & RF_ACTIVE))
1755 return (ENXIO);
1756 cfg->msix.msix_table_res = rle->res;
1757 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
1758 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1759 cfg->msix.msix_pba_bar);
1760 if (rle == NULL || rle->res == NULL ||
1761 !(rman_get_flags(rle->res) & RF_ACTIVE))
1762 return (ENXIO);
1763 }
1764 cfg->msix.msix_pba_res = rle->res;
1765
1766 if (bootverbose)
1767 device_printf(child,
1768 "attempting to allocate %d MSI-X vectors (%d supported)\n",
1769 *count, cfg->msix.msix_msgnum);
1770 max = min(*count, cfg->msix.msix_msgnum);
1771 for (i = 0; i < max; i++) {
1772 /* Allocate a message. */
1773 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
1774 if (error) {
1775 if (i == 0)
1776 return (error);
1777 break;
1778 }
1779 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1780 irq, 1);
1781 }
1782 actual = i;
1783
1784 if (bootverbose) {
1785 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
1786 if (actual == 1)
1787 device_printf(child, "using IRQ %ju for MSI-X\n",
1788 rle->start);
1789 else {
1790 int run;
1791
1792 /*
1793 * Be fancy and try to print contiguous runs of
1794 * IRQ values as ranges. 'irq' is the previous IRQ.
1795 * 'run' is true if we are in a range.
1796 */
1797 device_printf(child, "using IRQs %ju", rle->start);
1798 irq = rle->start;
1799 run = 0;
1800 for (i = 1; i < actual; i++) {
1801 rle = resource_list_find(&dinfo->resources,
1802 SYS_RES_IRQ, i + 1);
1803
1804 /* Still in a run? */
1805 if (rle->start == irq + 1) {
1806 run = 1;
1807 irq++;
1808 continue;
1809 }
1810
1811 /* Finish previous range. */
1812 if (run) {
1813 printf("-%d", irq);
1814 run = 0;
1815 }
1816
1817 /* Start new range. */
1818 printf(",%ju", rle->start);
1819 irq = rle->start;
1820 }
1821
1822 /* Unfinished range? */
1823 if (run)
1824 printf("-%d", irq);
1825 printf(" for MSI-X\n");
1826 }
1827 }
1828
1829 /* Mask all vectors. */
1830 for (i = 0; i < cfg->msix.msix_msgnum; i++)
1831 pci_mask_msix(child, i);
1832
1833 /* Allocate and initialize vector data and virtual table. */
1834 cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual,
1835 M_DEVBUF, M_WAITOK | M_ZERO);
1836 cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual,
1837 M_DEVBUF, M_WAITOK | M_ZERO);
1838 for (i = 0; i < actual; i++) {
1839 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1840 cfg->msix.msix_vectors[i].mv_irq = rle->start;
1841 cfg->msix.msix_table[i].mte_vector = i + 1;
1842 }
1843
1844 /* Update control register to enable MSI-X. */
1845 cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1846 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1847 cfg->msix.msix_ctrl, 2);
1848
1849 /* Update counts of alloc'd messages. */
1850 cfg->msix.msix_alloc = actual;
1851 cfg->msix.msix_table_len = actual;
1852 *count = actual;
1853 return (0);
1854 }
1855
1856 /*
1857 * By default, pci_alloc_msix() will assign the allocated IRQ
1858 * resources consecutively to the first N messages in the MSI-X table.
1859 * However, device drivers may want to use different layouts if they
1860 * either receive fewer messages than they asked for, or they wish to
1861 * populate the MSI-X table sparsely. This method allows the driver
1862 * to specify what layout it wants. It must be called after a
1863 * successful pci_alloc_msix() but before any of the associated
1864 * SYS_RES_IRQ resources are allocated via bus_alloc_resource().
1865 *
1866 * The 'vectors' array contains 'count' message vectors. The array
1867 * maps directly to the MSI-X table in that index 0 in the array
1868 * specifies the vector for the first message in the MSI-X table, etc.
1869 * The vector value in each array index can either be 0 to indicate
1870 * that no vector should be assigned to a message slot, or it can be a
1871 * number from 1 to N (where N is the count returned from a
1872 * succcessful call to pci_alloc_msix()) to indicate which message
1873 * vector (IRQ) to be used for the corresponding message.
1874 *
1875 * On successful return, each message with a non-zero vector will have
1876 * an associated SYS_RES_IRQ whose rid is equal to the array index +
1877 * 1. Additionally, if any of the IRQs allocated via the previous
1878 * call to pci_alloc_msix() are not used in the mapping, those IRQs
1879 * will be freed back to the system automatically.
1880 *
1881 * For example, suppose a driver has a MSI-X table with 6 messages and
1882 * asks for 6 messages, but pci_alloc_msix() only returns a count of
1883 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
1884 * C. After the call to pci_alloc_msix(), the device will be setup to
1885 * have an MSI-X table of ABC--- (where - means no vector assigned).
1886 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
1887 * then the MSI-X table will look like A-AB-B, and the 'C' vector will
1888 * be freed back to the system. This device will also have valid
1889 * SYS_RES_IRQ rids of 1, 3, 4, and 6.
1890 *
1891 * In any case, the SYS_RES_IRQ rid X will always map to the message
1892 * at MSI-X table index X - 1 and will only be valid if a vector is
1893 * assigned to that table entry.
1894 */
1895 int
1896 pci_remap_msix_method(device_t dev, device_t child, int count,
1897 const u_int *vectors)
1898 {
1899 struct pci_devinfo *dinfo = device_get_ivars(child);
1900 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1901 struct resource_list_entry *rle;
1902 int i, irq, j, *used;
1903
1904 /*
1905 * Have to have at least one message in the table but the
1906 * table can't be bigger than the actual MSI-X table in the
1907 * device.
1908 */
1909 if (count == 0 || count > msix->msix_msgnum)
1910 return (EINVAL);
1911
1912 /* Sanity check the vectors. */
1913 for (i = 0; i < count; i++)
1914 if (vectors[i] > msix->msix_alloc)
1915 return (EINVAL);
1916
1917 /*
1918 * Make sure there aren't any holes in the vectors to be used.
1919 * It's a big pain to support it, and it doesn't really make
1920 * sense anyway. Also, at least one vector must be used.
1921 */
1922 used = malloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK |
1923 M_ZERO);
1924 for (i = 0; i < count; i++)
1925 if (vectors[i] != 0)
1926 used[vectors[i] - 1] = 1;
1927 for (i = 0; i < msix->msix_alloc - 1; i++)
1928 if (used[i] == 0 && used[i + 1] == 1) {
1929 free(used, M_DEVBUF);
1930 return (EINVAL);
1931 }
1932 if (used[0] != 1) {
1933 free(used, M_DEVBUF);
1934 return (EINVAL);
1935 }
1936
1937 /* Make sure none of the resources are allocated. */
1938 for (i = 0; i < msix->msix_table_len; i++) {
1939 if (msix->msix_table[i].mte_vector == 0)
1940 continue;
1941 if (msix->msix_table[i].mte_handlers > 0) {
1942 free(used, M_DEVBUF);
1943 return (EBUSY);
1944 }
1945 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1946 KASSERT(rle != NULL, ("missing resource"));
1947 if (rle->res != NULL) {
1948 free(used, M_DEVBUF);
1949 return (EBUSY);
1950 }
1951 }
1952
1953 /* Free the existing resource list entries. */
1954 for (i = 0; i < msix->msix_table_len; i++) {
1955 if (msix->msix_table[i].mte_vector == 0)
1956 continue;
1957 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
1958 }
1959
1960 /*
1961 * Build the new virtual table keeping track of which vectors are
1962 * used.
1963 */
1964 free(msix->msix_table, M_DEVBUF);
1965 msix->msix_table = malloc(sizeof(struct msix_table_entry) * count,
1966 M_DEVBUF, M_WAITOK | M_ZERO);
1967 for (i = 0; i < count; i++)
1968 msix->msix_table[i].mte_vector = vectors[i];
1969 msix->msix_table_len = count;
1970
1971 /* Free any unused IRQs and resize the vectors array if necessary. */
1972 j = msix->msix_alloc - 1;
1973 if (used[j] == 0) {
1974 struct msix_vector *vec;
1975
1976 while (used[j] == 0) {
1977 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
1978 msix->msix_vectors[j].mv_irq);
1979 j--;
1980 }
1981 vec = malloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF,
1982 M_WAITOK);
1983 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
1984 (j + 1));
1985 free(msix->msix_vectors, M_DEVBUF);
1986 msix->msix_vectors = vec;
1987 msix->msix_alloc = j + 1;
1988 }
1989 free(used, M_DEVBUF);
1990
1991 /* Map the IRQs onto the rids. */
1992 for (i = 0; i < count; i++) {
1993 if (vectors[i] == 0)
1994 continue;
1995 irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
1996 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1997 irq, 1);
1998 }
1999
2000 if (bootverbose) {
2001 device_printf(child, "Remapped MSI-X IRQs as: ");
2002 for (i = 0; i < count; i++) {
2003 if (i != 0)
2004 printf(", ");
2005 if (vectors[i] == 0)
2006 printf("---");
2007 else
2008 printf("%d",
2009 msix->msix_vectors[vectors[i] - 1].mv_irq);
2010 }
2011 printf("\n");
2012 }
2013
2014 return (0);
2015 }
2016
2017 static int
2018 pci_release_msix(device_t dev, device_t child)
2019 {
2020 struct pci_devinfo *dinfo = device_get_ivars(child);
2021 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2022 struct resource_list_entry *rle;
2023 int i;
2024
2025 /* Do we have any messages to release? */
2026 if (msix->msix_alloc == 0)
2027 return (ENODEV);
2028
2029 /* Make sure none of the resources are allocated. */
2030 for (i = 0; i < msix->msix_table_len; i++) {
2031 if (msix->msix_table[i].mte_vector == 0)
2032 continue;
2033 if (msix->msix_table[i].mte_handlers > 0)
2034 return (EBUSY);
2035 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2036 KASSERT(rle != NULL, ("missing resource"));
2037 if (rle->res != NULL)
2038 return (EBUSY);
2039 }
2040
2041 /* Update control register to disable MSI-X. */
2042 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
2043 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
2044 msix->msix_ctrl, 2);
2045
2046 /* Free the resource list entries. */
2047 for (i = 0; i < msix->msix_table_len; i++) {
2048 if (msix->msix_table[i].mte_vector == 0)
2049 continue;
2050 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2051 }
2052 free(msix->msix_table, M_DEVBUF);
2053 msix->msix_table_len = 0;
2054
2055 /* Release the IRQs. */
2056 for (i = 0; i < msix->msix_alloc; i++)
2057 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2058 msix->msix_vectors[i].mv_irq);
2059 free(msix->msix_vectors, M_DEVBUF);
2060 msix->msix_alloc = 0;
2061 return (0);
2062 }
2063
2064 /*
2065 * Return the max supported MSI-X messages this device supports.
2066 * Basically, assuming the MD code can alloc messages, this function
2067 * should return the maximum value that pci_alloc_msix() can return.
2068 * Thus, it is subject to the tunables, etc.
2069 */
2070 int
2071 pci_msix_count_method(device_t dev, device_t child)
2072 {
2073 struct pci_devinfo *dinfo = device_get_ivars(child);
2074 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2075
2076 if (pci_do_msix && msix->msix_location != 0)
2077 return (msix->msix_msgnum);
2078 return (0);
2079 }
2080
2081 int
2082 pci_msix_pba_bar_method(device_t dev, device_t child)
2083 {
2084 struct pci_devinfo *dinfo = device_get_ivars(child);
2085 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2086
2087 if (pci_do_msix && msix->msix_location != 0)
2088 return (msix->msix_pba_bar);
2089 return (-1);
2090 }
2091
2092 int
2093 pci_msix_table_bar_method(device_t dev, device_t child)
2094 {
2095 struct pci_devinfo *dinfo = device_get_ivars(child);
2096 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2097
2098 if (pci_do_msix && msix->msix_location != 0)
2099 return (msix->msix_table_bar);
2100 return (-1);
2101 }
2102
2103 /*
2104 * HyperTransport MSI mapping control
2105 */
2106 void
2107 pci_ht_map_msi(device_t dev, uint64_t addr)
2108 {
2109 struct pci_devinfo *dinfo = device_get_ivars(dev);
2110 struct pcicfg_ht *ht = &dinfo->cfg.ht;
2111
2112 if (!ht->ht_msimap)
2113 return;
2114
2115 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
2116 ht->ht_msiaddr >> 20 == addr >> 20) {
2117 /* Enable MSI -> HT mapping. */
2118 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
2119 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2120 ht->ht_msictrl, 2);
2121 }
2122
2123 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
2124 /* Disable MSI -> HT mapping. */
2125 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
2126 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2127 ht->ht_msictrl, 2);
2128 }
2129 }
2130
2131 int
2132 pci_get_max_payload(device_t dev)
2133 {
2134 struct pci_devinfo *dinfo = device_get_ivars(dev);
2135 int cap;
2136 uint16_t val;
2137
2138 cap = dinfo->cfg.pcie.pcie_location;
2139 if (cap == 0)
2140 return (0);
2141 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2142 val &= PCIEM_CTL_MAX_PAYLOAD;
2143 val >>= 5;
2144 return (1 << (val + 7));
2145 }
2146
2147 int
2148 pci_get_max_read_req(device_t dev)
2149 {
2150 struct pci_devinfo *dinfo = device_get_ivars(dev);
2151 int cap;
2152 uint16_t val;
2153
2154 cap = dinfo->cfg.pcie.pcie_location;
2155 if (cap == 0)
2156 return (0);
2157 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2158 val &= PCIEM_CTL_MAX_READ_REQUEST;
2159 val >>= 12;
2160 return (1 << (val + 7));
2161 }
2162
2163 int
2164 pci_set_max_read_req(device_t dev, int size)
2165 {
2166 struct pci_devinfo *dinfo = device_get_ivars(dev);
2167 int cap;
2168 uint16_t val;
2169
2170 cap = dinfo->cfg.pcie.pcie_location;
2171 if (cap == 0)
2172 return (0);
2173 if (size < 128)
2174 size = 128;
2175 if (size > 4096)
2176 size = 4096;
2177 size = (1 << (fls(size) - 1));
2178 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2179 val &= ~PCIEM_CTL_MAX_READ_REQUEST;
2180 val |= (fls(size) - 8) << 12;
2181 pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2);
2182 return (size);
2183 }
2184
2185 uint32_t
2186 pcie_read_config(device_t dev, int reg, int width)
2187 {
2188 struct pci_devinfo *dinfo = device_get_ivars(dev);
2189 int cap;
2190
2191 cap = dinfo->cfg.pcie.pcie_location;
2192 if (cap == 0) {
2193 if (width == 2)
2194 return (0xffff);
2195 return (0xffffffff);
2196 }
2197
2198 return (pci_read_config(dev, cap + reg, width));
2199 }
2200
2201 void
2202 pcie_write_config(device_t dev, int reg, uint32_t value, int width)
2203 {
2204 struct pci_devinfo *dinfo = device_get_ivars(dev);
2205 int cap;
2206
2207 cap = dinfo->cfg.pcie.pcie_location;
2208 if (cap == 0)
2209 return;
2210 pci_write_config(dev, cap + reg, value, width);
2211 }
2212
2213 /*
2214 * Adjusts a PCI-e capability register by clearing the bits in mask
2215 * and setting the bits in (value & mask). Bits not set in mask are
2216 * not adjusted.
2217 *
2218 * Returns the old value on success or all ones on failure.
2219 */
2220 uint32_t
2221 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
2222 int width)
2223 {
2224 struct pci_devinfo *dinfo = device_get_ivars(dev);
2225 uint32_t old, new;
2226 int cap;
2227
2228 cap = dinfo->cfg.pcie.pcie_location;
2229 if (cap == 0) {
2230 if (width == 2)
2231 return (0xffff);
2232 return (0xffffffff);
2233 }
2234
2235 old = pci_read_config(dev, cap + reg, width);
2236 new = old & ~mask;
2237 new |= (value & mask);
2238 pci_write_config(dev, cap + reg, new, width);
2239 return (old);
2240 }
2241
2242 /*
2243 * Support for MSI message signalled interrupts.
2244 */
2245 void
2246 pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
2247 uint16_t data)
2248 {
2249 struct pci_devinfo *dinfo = device_get_ivars(child);
2250 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2251
2252 /* Write data and address values. */
2253 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
2254 address & 0xffffffff, 4);
2255 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2256 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
2257 address >> 32, 4);
2258 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
2259 data, 2);
2260 } else
2261 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
2262 2);
2263
2264 /* Enable MSI in the control register. */
2265 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
2266 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2267 msi->msi_ctrl, 2);
2268
2269 /* Enable MSI -> HT mapping. */
2270 pci_ht_map_msi(child, address);
2271 }
2272
2273 void
2274 pci_disable_msi_method(device_t dev, device_t child)
2275 {
2276 struct pci_devinfo *dinfo = device_get_ivars(child);
2277 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2278
2279 /* Disable MSI -> HT mapping. */
2280 pci_ht_map_msi(child, 0);
2281
2282 /* Disable MSI in the control register. */
2283 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
2284 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2285 msi->msi_ctrl, 2);
2286 }
2287
2288 /*
2289 * Restore MSI registers during resume. If MSI is enabled then
2290 * restore the data and address registers in addition to the control
2291 * register.
2292 */
2293 static void
2294 pci_resume_msi(device_t dev)
2295 {
2296 struct pci_devinfo *dinfo = device_get_ivars(dev);
2297 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2298 uint64_t address;
2299 uint16_t data;
2300
2301 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
2302 address = msi->msi_addr;
2303 data = msi->msi_data;
2304 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
2305 address & 0xffffffff, 4);
2306 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2307 pci_write_config(dev, msi->msi_location +
2308 PCIR_MSI_ADDR_HIGH, address >> 32, 4);
2309 pci_write_config(dev, msi->msi_location +
2310 PCIR_MSI_DATA_64BIT, data, 2);
2311 } else
2312 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
2313 data, 2);
2314 }
2315 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2316 2);
2317 }
2318
2319 static int
2320 pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
2321 {
2322 struct pci_devinfo *dinfo = device_get_ivars(dev);
2323 pcicfgregs *cfg = &dinfo->cfg;
2324 struct resource_list_entry *rle;
2325 struct msix_table_entry *mte;
2326 struct msix_vector *mv;
2327 uint64_t addr;
2328 uint32_t data;
2329 int error, i, j;
2330
2331 /*
2332 * Handle MSI first. We try to find this IRQ among our list
2333 * of MSI IRQs. If we find it, we request updated address and
2334 * data registers and apply the results.
2335 */
2336 if (cfg->msi.msi_alloc > 0) {
2337
2338 /* If we don't have any active handlers, nothing to do. */
2339 if (cfg->msi.msi_handlers == 0)
2340 return (0);
2341 for (i = 0; i < cfg->msi.msi_alloc; i++) {
2342 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
2343 i + 1);
2344 if (rle->start == irq) {
2345 error = PCIB_MAP_MSI(device_get_parent(bus),
2346 dev, irq, &addr, &data);
2347 if (error)
2348 return (error);
2349 pci_disable_msi(dev);
2350 dinfo->cfg.msi.msi_addr = addr;
2351 dinfo->cfg.msi.msi_data = data;
2352 pci_enable_msi(dev, addr, data);
2353 return (0);
2354 }
2355 }
2356 return (ENOENT);
2357 }
2358
2359 /*
2360 * For MSI-X, we check to see if we have this IRQ. If we do,
2361 * we request the updated mapping info. If that works, we go
2362 * through all the slots that use this IRQ and update them.
2363 */
2364 if (cfg->msix.msix_alloc > 0) {
2365 for (i = 0; i < cfg->msix.msix_alloc; i++) {
2366 mv = &cfg->msix.msix_vectors[i];
2367 if (mv->mv_irq == irq) {
2368 error = PCIB_MAP_MSI(device_get_parent(bus),
2369 dev, irq, &addr, &data);
2370 if (error)
2371 return (error);
2372 mv->mv_address = addr;
2373 mv->mv_data = data;
2374 for (j = 0; j < cfg->msix.msix_table_len; j++) {
2375 mte = &cfg->msix.msix_table[j];
2376 if (mte->mte_vector != i + 1)
2377 continue;
2378 if (mte->mte_handlers == 0)
2379 continue;
2380 pci_mask_msix(dev, j);
2381 pci_enable_msix(dev, j, addr, data);
2382 pci_unmask_msix(dev, j);
2383 }
2384 }
2385 }
2386 return (ENOENT);
2387 }
2388
2389 return (ENOENT);
2390 }
2391
2392 /*
2393 * Returns true if the specified device is blacklisted because MSI
2394 * doesn't work.
2395 */
2396 int
2397 pci_msi_device_blacklisted(device_t dev)
2398 {
2399
2400 if (!pci_honor_msi_blacklist)
2401 return (0);
2402
2403 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
2404 }
2405
2406 /*
2407 * Determine if MSI is blacklisted globally on this system. Currently,
2408 * we just check for blacklisted chipsets as represented by the
2409 * host-PCI bridge at device 0:0:0. In the future, it may become
2410 * necessary to check other system attributes, such as the kenv values
2411 * that give the motherboard manufacturer and model number.
2412 */
2413 static int
2414 pci_msi_blacklisted(void)
2415 {
2416 device_t dev;
2417
2418 if (!pci_honor_msi_blacklist)
2419 return (0);
2420
2421 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */
2422 if (!(pcie_chipset || pcix_chipset)) {
2423 if (vm_guest != VM_GUEST_NO) {
2424 /*
2425 * Whitelist older chipsets in virtual
2426 * machines known to support MSI.
2427 */
2428 dev = pci_find_bsf(0, 0, 0);
2429 if (dev != NULL)
2430 return (!pci_has_quirk(pci_get_devid(dev),
2431 PCI_QUIRK_ENABLE_MSI_VM));
2432 }
2433 return (1);
2434 }
2435
2436 dev = pci_find_bsf(0, 0, 0);
2437 if (dev != NULL)
2438 return (pci_msi_device_blacklisted(dev));
2439 return (0);
2440 }
2441
2442 /*
2443 * Returns true if the specified device is blacklisted because MSI-X
2444 * doesn't work. Note that this assumes that if MSI doesn't work,
2445 * MSI-X doesn't either.
2446 */
2447 int
2448 pci_msix_device_blacklisted(device_t dev)
2449 {
2450
2451 if (!pci_honor_msi_blacklist)
2452 return (0);
2453
2454 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
2455 return (1);
2456
2457 return (pci_msi_device_blacklisted(dev));
2458 }
2459
2460 /*
2461 * Determine if MSI-X is blacklisted globally on this system. If MSI
2462 * is blacklisted, assume that MSI-X is as well. Check for additional
2463 * chipsets where MSI works but MSI-X does not.
2464 */
2465 static int
2466 pci_msix_blacklisted(void)
2467 {
2468 device_t dev;
2469
2470 if (!pci_honor_msi_blacklist)
2471 return (0);
2472
2473 dev = pci_find_bsf(0, 0, 0);
2474 if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
2475 PCI_QUIRK_DISABLE_MSIX))
2476 return (1);
2477
2478 return (pci_msi_blacklisted());
2479 }
2480
2481 /*
2482 * Attempt to allocate *count MSI messages. The actual number allocated is
2483 * returned in *count. After this function returns, each message will be
2484 * available to the driver as SYS_RES_IRQ resources starting at a rid 1.
2485 */
2486 int
2487 pci_alloc_msi_method(device_t dev, device_t child, int *count)
2488 {
2489 struct pci_devinfo *dinfo = device_get_ivars(child);
2490 pcicfgregs *cfg = &dinfo->cfg;
2491 struct resource_list_entry *rle;
2492 int actual, error, i, irqs[32];
2493 uint16_t ctrl;
2494
2495 /* Don't let count == 0 get us into trouble. */
2496 if (*count == 0)
2497 return (EINVAL);
2498
2499 /* If rid 0 is allocated, then fail. */
2500 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
2501 if (rle != NULL && rle->res != NULL)
2502 return (ENXIO);
2503
2504 /* Already have allocated messages? */
2505 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
2506 return (ENXIO);
2507
2508 /* If MSI is blacklisted for this system, fail. */
2509 if (pci_msi_blacklisted())
2510 return (ENXIO);
2511
2512 /* MSI capability present? */
2513 if (cfg->msi.msi_location == 0 || !pci_do_msi)
2514 return (ENODEV);
2515
2516 if (bootverbose)
2517 device_printf(child,
2518 "attempting to allocate %d MSI vectors (%d supported)\n",
2519 *count, cfg->msi.msi_msgnum);
2520
2521 /* Don't ask for more than the device supports. */
2522 actual = min(*count, cfg->msi.msi_msgnum);
2523
2524 /* Don't ask for more than 32 messages. */
2525 actual = min(actual, 32);
2526
2527 /* MSI requires power of 2 number of messages. */
2528 if (!powerof2(actual))
2529 return (EINVAL);
2530
2531 for (;;) {
2532 /* Try to allocate N messages. */
2533 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
2534 actual, irqs);
2535 if (error == 0)
2536 break;
2537 if (actual == 1)
2538 return (error);
2539
2540 /* Try N / 2. */
2541 actual >>= 1;
2542 }
2543
2544 /*
2545 * We now have N actual messages mapped onto SYS_RES_IRQ
2546 * resources in the irqs[] array, so add new resources
2547 * starting at rid 1.
2548 */
2549 for (i = 0; i < actual; i++)
2550 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
2551 irqs[i], irqs[i], 1);
2552
2553 if (bootverbose) {
2554 if (actual == 1)
2555 device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
2556 else {
2557 int run;
2558
2559 /*
2560 * Be fancy and try to print contiguous runs
2561 * of IRQ values as ranges. 'run' is true if
2562 * we are in a range.
2563 */
2564 device_printf(child, "using IRQs %d", irqs[0]);
2565 run = 0;
2566 for (i = 1; i < actual; i++) {
2567
2568 /* Still in a run? */
2569 if (irqs[i] == irqs[i - 1] + 1) {
2570 run = 1;
2571 continue;
2572 }
2573
2574 /* Finish previous range. */
2575 if (run) {
2576 printf("-%d", irqs[i - 1]);
2577 run = 0;
2578 }
2579
2580 /* Start new range. */
2581 printf(",%d", irqs[i]);
2582 }
2583
2584 /* Unfinished range? */
2585 if (run)
2586 printf("-%d", irqs[actual - 1]);
2587 printf(" for MSI\n");
2588 }
2589 }
2590
2591 /* Update control register with actual count. */
2592 ctrl = cfg->msi.msi_ctrl;
2593 ctrl &= ~PCIM_MSICTRL_MME_MASK;
2594 ctrl |= (ffs(actual) - 1) << 4;
2595 cfg->msi.msi_ctrl = ctrl;
2596 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);
2597
2598 /* Update counts of alloc'd messages. */
2599 cfg->msi.msi_alloc = actual;
2600 cfg->msi.msi_handlers = 0;
2601 *count = actual;
2602 return (0);
2603 }
2604
2605 /* Release the MSI messages associated with this device. */
2606 int
2607 pci_release_msi_method(device_t dev, device_t child)
2608 {
2609 struct pci_devinfo *dinfo = device_get_ivars(child);
2610 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2611 struct resource_list_entry *rle;
2612 int error, i, irqs[32];
2613
2614 /* Try MSI-X first. */
2615 error = pci_release_msix(dev, child);
2616 if (error != ENODEV)
2617 return (error);
2618
2619 /* Do we have any messages to release? */
2620 if (msi->msi_alloc == 0)
2621 return (ENODEV);
2622 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
2623
2624 /* Make sure none of the resources are allocated. */
2625 if (msi->msi_handlers > 0)
2626 return (EBUSY);
2627 for (i = 0; i < msi->msi_alloc; i++) {
2628 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2629 KASSERT(rle != NULL, ("missing MSI resource"));
2630 if (rle->res != NULL)
2631 return (EBUSY);
2632 irqs[i] = rle->start;
2633 }
2634
2635 /* Update control register with 0 count. */
2636 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
2637 ("%s: MSI still enabled", __func__));
2638 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
2639 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2640 msi->msi_ctrl, 2);
2641
2642 /* Release the messages. */
2643 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
2644 for (i = 0; i < msi->msi_alloc; i++)
2645 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2646
2647 /* Update alloc count. */
2648 msi->msi_alloc = 0;
2649 msi->msi_addr = 0;
2650 msi->msi_data = 0;
2651 return (0);
2652 }
2653
2654 /*
2655 * Return the max supported MSI messages this device supports.
2656 * Basically, assuming the MD code can alloc messages, this function
2657 * should return the maximum value that pci_alloc_msi() can return.
2658 * Thus, it is subject to the tunables, etc.
2659 */
2660 int
2661 pci_msi_count_method(device_t dev, device_t child)
2662 {
2663 struct pci_devinfo *dinfo = device_get_ivars(child);
2664 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2665
2666 if (pci_do_msi && msi->msi_location != 0)
2667 return (msi->msi_msgnum);
2668 return (0);
2669 }
2670
2671 /* free pcicfgregs structure and all depending data structures */
2672
2673 int
2674 pci_freecfg(struct pci_devinfo *dinfo)
2675 {
2676 struct devlist *devlist_head;
2677 struct pci_map *pm, *next;
2678 int i;
2679
2680 devlist_head = &pci_devq;
2681
2682 if (dinfo->cfg.vpd.vpd_reg) {
2683 free(dinfo->cfg.vpd.vpd_ident, M_DEVBUF);
2684 for (i = 0; i < dinfo->cfg.vpd.vpd_rocnt; i++)
2685 free(dinfo->cfg.vpd.vpd_ros[i].value, M_DEVBUF);
2686 free(dinfo->cfg.vpd.vpd_ros, M_DEVBUF);
2687 for (i = 0; i < dinfo->cfg.vpd.vpd_wcnt; i++)
2688 free(dinfo->cfg.vpd.vpd_w[i].value, M_DEVBUF);
2689 free(dinfo->cfg.vpd.vpd_w, M_DEVBUF);
2690 }
2691 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
2692 free(pm, M_DEVBUF);
2693 }
2694 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
2695 free(dinfo, M_DEVBUF);
2696
2697 /* increment the generation count */
2698 pci_generation++;
2699
2700 /* we're losing one device */
2701 pci_numdevs--;
2702 return (0);
2703 }
2704
2705 /*
2706 * PCI power manangement
2707 */
2708 int
2709 pci_set_powerstate_method(device_t dev, device_t child, int state)
2710 {
2711 struct pci_devinfo *dinfo = device_get_ivars(child);
2712 pcicfgregs *cfg = &dinfo->cfg;
2713 uint16_t status;
2714 int oldstate, highest, delay;
2715
2716 if (cfg->pp.pp_cap == 0)
2717 return (EOPNOTSUPP);
2718
2719 /*
2720 * Optimize a no state change request away. While it would be OK to
2721 * write to the hardware in theory, some devices have shown odd
2722 * behavior when going from D3 -> D3.
2723 */
2724 oldstate = pci_get_powerstate(child);
2725 if (oldstate == state)
2726 return (0);
2727
2728 /*
2729 * The PCI power management specification states that after a state
2730 * transition between PCI power states, system software must
2731 * guarantee a minimal delay before the function accesses the device.
2732 * Compute the worst case delay that we need to guarantee before we
2733 * access the device. Many devices will be responsive much more
2734 * quickly than this delay, but there are some that don't respond
2735 * instantly to state changes. Transitions to/from D3 state require
2736 * 10ms, while D2 requires 200us, and D0/1 require none. The delay
2737 * is done below with DELAY rather than a sleeper function because
2738 * this function can be called from contexts where we cannot sleep.
2739 */
2740 highest = (oldstate > state) ? oldstate : state;
2741 if (highest == PCI_POWERSTATE_D3)
2742 delay = 10000;
2743 else if (highest == PCI_POWERSTATE_D2)
2744 delay = 200;
2745 else
2746 delay = 0;
2747 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2)
2748 & ~PCIM_PSTAT_DMASK;
2749 switch (state) {
2750 case PCI_POWERSTATE_D0:
2751 status |= PCIM_PSTAT_D0;
2752 break;
2753 case PCI_POWERSTATE_D1:
2754 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
2755 return (EOPNOTSUPP);
2756 status |= PCIM_PSTAT_D1;
2757 break;
2758 case PCI_POWERSTATE_D2:
2759 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
2760 return (EOPNOTSUPP);
2761 status |= PCIM_PSTAT_D2;
2762 break;
2763 case PCI_POWERSTATE_D3:
2764 status |= PCIM_PSTAT_D3;
2765 break;
2766 default:
2767 return (EINVAL);
2768 }
2769
2770 if (bootverbose)
2771 pci_printf(cfg, "Transition from D%d to D%d\n", oldstate,
2772 state);
2773
2774 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2);
2775 if (delay)
2776 DELAY(delay);
2777 return (0);
2778 }
2779
2780 int
2781 pci_get_powerstate_method(device_t dev, device_t child)
2782 {
2783 struct pci_devinfo *dinfo = device_get_ivars(child);
2784 pcicfgregs *cfg = &dinfo->cfg;
2785 uint16_t status;
2786 int result;
2787
2788 if (cfg->pp.pp_cap != 0) {
2789 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2);
2790 switch (status & PCIM_PSTAT_DMASK) {
2791 case PCIM_PSTAT_D0:
2792 result = PCI_POWERSTATE_D0;
2793 break;
2794 case PCIM_PSTAT_D1:
2795 result = PCI_POWERSTATE_D1;
2796 break;
2797 case PCIM_PSTAT_D2:
2798 result = PCI_POWERSTATE_D2;
2799 break;
2800 case PCIM_PSTAT_D3:
2801 result = PCI_POWERSTATE_D3;
2802 break;
2803 default:
2804 result = PCI_POWERSTATE_UNKNOWN;
2805 break;
2806 }
2807 } else {
2808 /* No support, device is always at D0 */
2809 result = PCI_POWERSTATE_D0;
2810 }
2811 return (result);
2812 }
2813
2814 /*
2815 * Some convenience functions for PCI device drivers.
2816 */
2817
2818 static __inline void
2819 pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
2820 {
2821 uint16_t command;
2822
2823 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2824 command |= bit;
2825 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2826 }
2827
2828 static __inline void
2829 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
2830 {
2831 uint16_t command;
2832
2833 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2834 command &= ~bit;
2835 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2836 }
2837
2838 int
2839 pci_enable_busmaster_method(device_t dev, device_t child)
2840 {
2841 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2842 return (0);
2843 }
2844
2845 int
2846 pci_disable_busmaster_method(device_t dev, device_t child)
2847 {
2848 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2849 return (0);
2850 }
2851
2852 int
2853 pci_enable_io_method(device_t dev, device_t child, int space)
2854 {
2855 uint16_t bit;
2856
2857 switch(space) {
2858 case SYS_RES_IOPORT:
2859 bit = PCIM_CMD_PORTEN;
2860 break;
2861 case SYS_RES_MEMORY:
2862 bit = PCIM_CMD_MEMEN;
2863 break;
2864 default:
2865 return (EINVAL);
2866 }
2867 pci_set_command_bit(dev, child, bit);
2868 return (0);
2869 }
2870
2871 int
2872 pci_disable_io_method(device_t dev, device_t child, int space)
2873 {
2874 uint16_t bit;
2875
2876 switch(space) {
2877 case SYS_RES_IOPORT:
2878 bit = PCIM_CMD_PORTEN;
2879 break;
2880 case SYS_RES_MEMORY:
2881 bit = PCIM_CMD_MEMEN;
2882 break;
2883 default:
2884 return (EINVAL);
2885 }
2886 pci_clear_command_bit(dev, child, bit);
2887 return (0);
2888 }
2889
2890 /*
2891 * New style pci driver. Parent device is either a pci-host-bridge or a
2892 * pci-pci-bridge. Both kinds are represented by instances of pcib.
2893 */
2894
2895 void
2896 pci_print_verbose(struct pci_devinfo *dinfo)
2897 {
2898
2899 if (bootverbose) {
2900 pcicfgregs *cfg = &dinfo->cfg;
2901
2902 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
2903 cfg->vendor, cfg->device, cfg->revid);
2904 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
2905 cfg->domain, cfg->bus, cfg->slot, cfg->func);
2906 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
2907 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
2908 cfg->mfdev);
2909 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
2910 cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
2911 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
2912 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
2913 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
2914 if (cfg->intpin > 0)
2915 printf("\tintpin=%c, irq=%d\n",
2916 cfg->intpin +'a' -1, cfg->intline);
2917 if (cfg->pp.pp_cap) {
2918 uint16_t status;
2919
2920 status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2);
2921 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
2922 cfg->pp.pp_cap & PCIM_PCAP_SPEC,
2923 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
2924 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
2925 status & PCIM_PSTAT_DMASK);
2926 }
2927 if (cfg->msi.msi_location) {
2928 int ctrl;
2929
2930 ctrl = cfg->msi.msi_ctrl;
2931 printf("\tMSI supports %d message%s%s%s\n",
2932 cfg->msi.msi_msgnum,
2933 (cfg->msi.msi_msgnum == 1) ? "" : "s",
2934 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
2935 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
2936 }
2937 if (cfg->msix.msix_location) {
2938 printf("\tMSI-X supports %d message%s ",
2939 cfg->msix.msix_msgnum,
2940 (cfg->msix.msix_msgnum == 1) ? "" : "s");
2941 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
2942 printf("in map 0x%x\n",
2943 cfg->msix.msix_table_bar);
2944 else
2945 printf("in maps 0x%x and 0x%x\n",
2946 cfg->msix.msix_table_bar,
2947 cfg->msix.msix_pba_bar);
2948 }
2949 }
2950 }
2951
2952 static int
2953 pci_porten(device_t dev)
2954 {
2955 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
2956 }
2957
2958 static int
2959 pci_memen(device_t dev)
2960 {
2961 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
2962 }
2963
2964 void
2965 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp,
2966 int *bar64)
2967 {
2968 struct pci_devinfo *dinfo;
2969 pci_addr_t map, testval;
2970 int ln2range;
2971 uint16_t cmd;
2972
2973 /*
2974 * The device ROM BAR is special. It is always a 32-bit
2975 * memory BAR. Bit 0 is special and should not be set when
2976 * sizing the BAR.
2977 */
2978 dinfo = device_get_ivars(dev);
2979 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
2980 map = pci_read_config(dev, reg, 4);
2981 pci_write_config(dev, reg, 0xfffffffe, 4);
2982 testval = pci_read_config(dev, reg, 4);
2983 pci_write_config(dev, reg, map, 4);
2984 *mapp = map;
2985 *testvalp = testval;
2986 if (bar64 != NULL)
2987 *bar64 = 0;
2988 return;
2989 }
2990
2991 map = pci_read_config(dev, reg, 4);
2992 ln2range = pci_maprange(map);
2993 if (ln2range == 64)
2994 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
2995
2996 /*
2997 * Disable decoding via the command register before
2998 * determining the BAR's length since we will be placing it in
2999 * a weird state.
3000 */
3001 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3002 pci_write_config(dev, PCIR_COMMAND,
3003 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
3004
3005 /*
3006 * Determine the BAR's length by writing all 1's. The bottom
3007 * log_2(size) bits of the BAR will stick as 0 when we read
3008 * the value back.
3009 */
3010 pci_write_config(dev, reg, 0xffffffff, 4);
3011 testval = pci_read_config(dev, reg, 4);
3012 if (ln2range == 64) {
3013 pci_write_config(dev, reg + 4, 0xffffffff, 4);
3014 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3015 }
3016
3017 /*
3018 * Restore the original value of the BAR. We may have reprogrammed
3019 * the BAR of the low-level console device and when booting verbose,
3020 * we need the console device addressable.
3021 */
3022 pci_write_config(dev, reg, map, 4);
3023 if (ln2range == 64)
3024 pci_write_config(dev, reg + 4, map >> 32, 4);
3025 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3026
3027 *mapp = map;
3028 *testvalp = testval;
3029 if (bar64 != NULL)
3030 *bar64 = (ln2range == 64);
3031 }
3032
3033 static void
3034 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
3035 {
3036 struct pci_devinfo *dinfo;
3037 int ln2range;
3038
3039 /* The device ROM BAR is always a 32-bit memory BAR. */
3040 dinfo = device_get_ivars(dev);
3041 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3042 ln2range = 32;
3043 else
3044 ln2range = pci_maprange(pm->pm_value);
3045 pci_write_config(dev, pm->pm_reg, base, 4);
3046 if (ln2range == 64)
3047 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
3048 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
3049 if (ln2range == 64)
3050 pm->pm_value |= (pci_addr_t)pci_read_config(dev,
3051 pm->pm_reg + 4, 4) << 32;
3052 }
3053
3054 struct pci_map *
3055 pci_find_bar(device_t dev, int reg)
3056 {
3057 struct pci_devinfo *dinfo;
3058 struct pci_map *pm;
3059
3060 dinfo = device_get_ivars(dev);
3061 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3062 if (pm->pm_reg == reg)
3063 return (pm);
3064 }
3065 return (NULL);
3066 }
3067
3068 int
3069 pci_bar_enabled(device_t dev, struct pci_map *pm)
3070 {
3071 struct pci_devinfo *dinfo;
3072 uint16_t cmd;
3073
3074 dinfo = device_get_ivars(dev);
3075 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
3076 !(pm->pm_value & PCIM_BIOS_ENABLE))
3077 return (0);
3078 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3079 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value))
3080 return ((cmd & PCIM_CMD_MEMEN) != 0);
3081 else
3082 return ((cmd & PCIM_CMD_PORTEN) != 0);
3083 }
3084
3085 struct pci_map *
3086 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
3087 {
3088 struct pci_devinfo *dinfo;
3089 struct pci_map *pm, *prev;
3090
3091 dinfo = device_get_ivars(dev);
3092 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO);
3093 pm->pm_reg = reg;
3094 pm->pm_value = value;
3095 pm->pm_size = size;
3096 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
3097 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
3098 reg));
3099 if (STAILQ_NEXT(prev, pm_link) == NULL ||
3100 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
3101 break;
3102 }
3103 if (prev != NULL)
3104 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
3105 else
3106 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
3107 return (pm);
3108 }
3109
3110 static void
3111 pci_restore_bars(device_t dev)
3112 {
3113 struct pci_devinfo *dinfo;
3114 struct pci_map *pm;
3115 int ln2range;
3116
3117 dinfo = device_get_ivars(dev);
3118 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3119 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3120 ln2range = 32;
3121 else
3122 ln2range = pci_maprange(pm->pm_value);
3123 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
3124 if (ln2range == 64)
3125 pci_write_config(dev, pm->pm_reg + 4,
3126 pm->pm_value >> 32, 4);
3127 }
3128 }
3129
3130 /*
3131 * Add a resource based on a pci map register. Return 1 if the map
3132 * register is a 32bit map register or 2 if it is a 64bit register.
3133 */
3134 static int
3135 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
3136 int force, int prefetch)
3137 {
3138 struct pci_map *pm;
3139 pci_addr_t base, map, testval;
3140 pci_addr_t start, end, count;
3141 int barlen, basezero, flags, maprange, mapsize, type;
3142 uint16_t cmd;
3143 struct resource *res;
3144
3145 /*
3146 * The BAR may already exist if the device is a CardBus card
3147 * whose CIS is stored in this BAR.
3148 */
3149 pm = pci_find_bar(dev, reg);
3150 if (pm != NULL) {
3151 maprange = pci_maprange(pm->pm_value);
3152 barlen = maprange == 64 ? 2 : 1;
3153 return (barlen);
3154 }
3155
3156 pci_read_bar(dev, reg, &map, &testval, NULL);
3157 if (PCI_BAR_MEM(map)) {
3158 type = SYS_RES_MEMORY;
3159 if (map & PCIM_BAR_MEM_PREFETCH)
3160 prefetch = 1;
3161 } else
3162 type = SYS_RES_IOPORT;
3163 mapsize = pci_mapsize(testval);
3164 base = pci_mapbase(map);
3165 #ifdef __PCI_BAR_ZERO_VALID
3166 basezero = 0;
3167 #else
3168 basezero = base == 0;
3169 #endif
3170 maprange = pci_maprange(map);
3171 barlen = maprange == 64 ? 2 : 1;
3172
3173 /*
3174 * For I/O registers, if bottom bit is set, and the next bit up
3175 * isn't clear, we know we have a BAR that doesn't conform to the
3176 * spec, so ignore it. Also, sanity check the size of the data
3177 * areas to the type of memory involved. Memory must be at least
3178 * 16 bytes in size, while I/O ranges must be at least 4.
3179 */
3180 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
3181 return (barlen);
3182 if ((type == SYS_RES_MEMORY && mapsize < 4) ||
3183 (type == SYS_RES_IOPORT && mapsize < 2))
3184 return (barlen);
3185
3186 /* Save a record of this BAR. */
3187 pm = pci_add_bar(dev, reg, map, mapsize);
3188 if (bootverbose) {
3189 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
3190 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
3191 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3192 printf(", port disabled\n");
3193 else if (type == SYS_RES_MEMORY && !pci_memen(dev))
3194 printf(", memory disabled\n");
3195 else
3196 printf(", enabled\n");
3197 }
3198
3199 /*
3200 * If base is 0, then we have problems if this architecture does
3201 * not allow that. It is best to ignore such entries for the
3202 * moment. These will be allocated later if the driver specifically
3203 * requests them. However, some removable busses look better when
3204 * all resources are allocated, so allow '' to be overriden.
3205 *
3206 * Similarly treat maps whose values is the same as the test value
3207 * read back. These maps have had all f's written to them by the
3208 * BIOS in an attempt to disable the resources.
3209 */
3210 if (!force && (basezero || map == testval))
3211 return (barlen);
3212 if ((u_long)base != base) {
3213 device_printf(bus,
3214 "pci%d:%d:%d:%d bar %#x too many address bits",
3215 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
3216 pci_get_function(dev), reg);
3217 return (barlen);
3218 }
3219
3220 /*
3221 * This code theoretically does the right thing, but has
3222 * undesirable side effects in some cases where peripherals
3223 * respond oddly to having these bits enabled. Let the user
3224 * be able to turn them off (since pci_enable_io_modes is 1 by
3225 * default).
3226 */
3227 if (pci_enable_io_modes) {
3228 /* Turn on resources that have been left off by a lazy BIOS */
3229 if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
3230 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3231 cmd |= PCIM_CMD_PORTEN;
3232 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3233 }
3234 if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
3235 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3236 cmd |= PCIM_CMD_MEMEN;
3237 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3238 }
3239 } else {
3240 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3241 return (barlen);
3242 if (type == SYS_RES_MEMORY && !pci_memen(dev))
3243 return (barlen);
3244 }
3245
3246 count = (pci_addr_t)1 << mapsize;
3247 flags = RF_ALIGNMENT_LOG2(mapsize);
3248 if (prefetch)
3249 flags |= RF_PREFETCHABLE;
3250 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) {
3251 start = 0; /* Let the parent decide. */
3252 end = ~0;
3253 } else {
3254 start = base;
3255 end = base + count - 1;
3256 }
3257 resource_list_add(rl, type, reg, start, end, count);
3258
3259 /*
3260 * Try to allocate the resource for this BAR from our parent
3261 * so that this resource range is already reserved. The
3262 * driver for this device will later inherit this resource in
3263 * pci_alloc_resource().
3264 */
3265 res = resource_list_reserve(rl, bus, dev, type, ®, start, end, count,
3266 flags);
3267 if (pci_do_realloc_bars && res == NULL && (start != 0 || end != ~0)) {
3268 /*
3269 * If the allocation fails, try to allocate a resource for
3270 * this BAR using any available range. The firmware felt
3271 * it was important enough to assign a resource, so don't
3272 * disable decoding if we can help it.
3273 */
3274 resource_list_delete(rl, type, reg);
3275 resource_list_add(rl, type, reg, 0, ~0, count);
3276 res = resource_list_reserve(rl, bus, dev, type, ®, 0, ~0,
3277 count, flags);
3278 }
3279 if (res == NULL) {
3280 /*
3281 * If the allocation fails, delete the resource list entry
3282 * and disable decoding for this device.
3283 *
3284 * If the driver requests this resource in the future,
3285 * pci_reserve_map() will try to allocate a fresh
3286 * resource range.
3287 */
3288 resource_list_delete(rl, type, reg);
3289 pci_disable_io(dev, type);
3290 if (bootverbose)
3291 device_printf(bus,
3292 "pci%d:%d:%d:%d bar %#x failed to allocate\n",
3293 pci_get_domain(dev), pci_get_bus(dev),
3294 pci_get_slot(dev), pci_get_function(dev), reg);
3295 } else {
3296 start = rman_get_start(res);
3297 pci_write_bar(dev, pm, start);
3298 }
3299 return (barlen);
3300 }
3301
3302 /*
3303 * For ATA devices we need to decide early what addressing mode to use.
3304 * Legacy demands that the primary and secondary ATA ports sits on the
3305 * same addresses that old ISA hardware did. This dictates that we use
3306 * those addresses and ignore the BAR's if we cannot set PCI native
3307 * addressing mode.
3308 */
3309 static void
3310 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
3311 uint32_t prefetchmask)
3312 {
3313 int rid, type, progif;
3314 #if 0
3315 /* if this device supports PCI native addressing use it */
3316 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3317 if ((progif & 0x8a) == 0x8a) {
3318 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
3319 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
3320 printf("Trying ATA native PCI addressing mode\n");
3321 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1);
3322 }
3323 }
3324 #endif
3325 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3326 type = SYS_RES_IOPORT;
3327 if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
3328 pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
3329 prefetchmask & (1 << 0));
3330 pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
3331 prefetchmask & (1 << 1));
3332 } else {
3333 rid = PCIR_BAR(0);
3334 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
3335 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0,
3336 0x1f7, 8, 0);
3337 rid = PCIR_BAR(1);
3338 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
3339 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6,
3340 0x3f6, 1, 0);
3341 }
3342 if (progif & PCIP_STORAGE_IDE_MODESEC) {
3343 pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
3344 prefetchmask & (1 << 2));
3345 pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
3346 prefetchmask & (1 << 3));
3347 } else {
3348 rid = PCIR_BAR(2);
3349 resource_list_add(rl, type, rid, 0x170, 0x177, 8);
3350 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170,
3351 0x177, 8, 0);
3352 rid = PCIR_BAR(3);
3353 resource_list_add(rl, type, rid, 0x376, 0x376, 1);
3354 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376,
3355 0x376, 1, 0);
3356 }
3357 pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
3358 prefetchmask & (1 << 4));
3359 pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
3360 prefetchmask & (1 << 5));
3361 }
3362
3363 static void
3364 pci_assign_interrupt(device_t bus, device_t dev, int force_route)
3365 {
3366 struct pci_devinfo *dinfo = device_get_ivars(dev);
3367 pcicfgregs *cfg = &dinfo->cfg;
3368 char tunable_name[64];
3369 int irq;
3370
3371 /* Has to have an intpin to have an interrupt. */
3372 if (cfg->intpin == 0)
3373 return;
3374
3375 /* Let the user override the IRQ with a tunable. */
3376 irq = PCI_INVALID_IRQ;
3377 snprintf(tunable_name, sizeof(tunable_name),
3378 "hw.pci%d.%d.%d.INT%c.irq",
3379 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
3380 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
3381 irq = PCI_INVALID_IRQ;
3382
3383 /*
3384 * If we didn't get an IRQ via the tunable, then we either use the
3385 * IRQ value in the intline register or we ask the bus to route an
3386 * interrupt for us. If force_route is true, then we only use the
3387 * value in the intline register if the bus was unable to assign an
3388 * IRQ.
3389 */
3390 if (!PCI_INTERRUPT_VALID(irq)) {
3391 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route)
3392 irq = PCI_ASSIGN_INTERRUPT(bus, dev);
3393 if (!PCI_INTERRUPT_VALID(irq))
3394 irq = cfg->intline;
3395 }
3396
3397 /* If after all that we don't have an IRQ, just bail. */
3398 if (!PCI_INTERRUPT_VALID(irq))
3399 return;
3400
3401 /* Update the config register if it changed. */
3402 if (irq != cfg->intline) {
3403 cfg->intline = irq;
3404 pci_write_config(dev, PCIR_INTLINE, irq, 1);
3405 }
3406
3407 /* Add this IRQ as rid 0 interrupt resource. */
3408 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
3409 }
3410
3411 /* Perform early OHCI takeover from SMM. */
3412 static void
3413 ohci_early_takeover(device_t self)
3414 {
3415 struct resource *res;
3416 uint32_t ctl;
3417 int rid;
3418 int i;
3419
3420 rid = PCIR_BAR(0);
3421 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3422 if (res == NULL)
3423 return;
3424
3425 ctl = bus_read_4(res, OHCI_CONTROL);
3426 if (ctl & OHCI_IR) {
3427 if (bootverbose)
3428 printf("ohci early: "
3429 "SMM active, request owner change\n");
3430 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
3431 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
3432 DELAY(1000);
3433 ctl = bus_read_4(res, OHCI_CONTROL);
3434 }
3435 if (ctl & OHCI_IR) {
3436 if (bootverbose)
3437 printf("ohci early: "
3438 "SMM does not respond, resetting\n");
3439 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
3440 }
3441 /* Disable interrupts */
3442 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
3443 }
3444
3445 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3446 }
3447
3448 /* Perform early UHCI takeover from SMM. */
3449 static void
3450 uhci_early_takeover(device_t self)
3451 {
3452 struct resource *res;
3453 int rid;
3454
3455 /*
3456 * Set the PIRQD enable bit and switch off all the others. We don't
3457 * want legacy support to interfere with us XXX Does this also mean
3458 * that the BIOS won't touch the keyboard anymore if it is connected
3459 * to the ports of the root hub?
3460 */
3461 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
3462
3463 /* Disable interrupts */
3464 rid = PCI_UHCI_BASE_REG;
3465 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
3466 if (res != NULL) {
3467 bus_write_2(res, UHCI_INTR, 0);
3468 bus_release_resource(self, SYS_RES_IOPORT, rid, res);
3469 }
3470 }
3471
3472 /* Perform early EHCI takeover from SMM. */
3473 static void
3474 ehci_early_takeover(device_t self)
3475 {
3476 struct resource *res;
3477 uint32_t cparams;
3478 uint32_t eec;
3479 uint8_t eecp;
3480 uint8_t bios_sem;
3481 uint8_t offs;
3482 int rid;
3483 int i;
3484
3485 rid = PCIR_BAR(0);
3486 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3487 if (res == NULL)
3488 return;
3489
3490 cparams = bus_read_4(res, EHCI_HCCPARAMS);
3491
3492 /* Synchronise with the BIOS if it owns the controller. */
3493 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
3494 eecp = EHCI_EECP_NEXT(eec)) {
3495 eec = pci_read_config(self, eecp, 4);
3496 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
3497 continue;
3498 }
3499 bios_sem = pci_read_config(self, eecp +
3500 EHCI_LEGSUP_BIOS_SEM, 1);
3501 if (bios_sem == 0) {
3502 continue;
3503 }
3504 if (bootverbose)
3505 printf("ehci early: "
3506 "SMM active, request owner change\n");
3507
3508 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);
3509
3510 for (i = 0; (i < 100) && (bios_sem != 0); i++) {
3511 DELAY(1000);
3512 bios_sem = pci_read_config(self, eecp +
3513 EHCI_LEGSUP_BIOS_SEM, 1);
3514 }
3515
3516 if (bios_sem != 0) {
3517 if (bootverbose)
3518 printf("ehci early: "
3519 "SMM does not respond\n");
3520 }
3521 /* Disable interrupts */
3522 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
3523 bus_write_4(res, offs + EHCI_USBINTR, 0);
3524 }
3525 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3526 }
3527
3528 /* Perform early XHCI takeover from SMM. */
3529 static void
3530 xhci_early_takeover(device_t self)
3531 {
3532 struct resource *res;
3533 uint32_t cparams;
3534 uint32_t eec;
3535 uint8_t eecp;
3536 uint8_t bios_sem;
3537 uint8_t offs;
3538 int rid;
3539 int i;
3540
3541 rid = PCIR_BAR(0);
3542 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3543 if (res == NULL)
3544 return;
3545
3546 cparams = bus_read_4(res, XHCI_HCSPARAMS0);
3547
3548 eec = -1;
3549
3550 /* Synchronise with the BIOS if it owns the controller. */
3551 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
3552 eecp += XHCI_XECP_NEXT(eec) << 2) {
3553 eec = bus_read_4(res, eecp);
3554
3555 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
3556 continue;
3557
3558 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
3559 if (bios_sem == 0)
3560 continue;
3561
3562 if (bootverbose)
3563 printf("xhci early: "
3564 "SMM active, request owner change\n");
3565
3566 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);
3567
3568 /* wait a maximum of 5 second */
3569
3570 for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
3571 DELAY(1000);
3572 bios_sem = bus_read_1(res, eecp +
3573 XHCI_XECP_BIOS_SEM);
3574 }
3575
3576 if (bios_sem != 0) {
3577 if (bootverbose)
3578 printf("xhci early: "
3579 "SMM does not respond\n");
3580 }
3581
3582 /* Disable interrupts */
3583 offs = bus_read_1(res, XHCI_CAPLENGTH);
3584 bus_write_4(res, offs + XHCI_USBCMD, 0);
3585 bus_read_4(res, offs + XHCI_USBSTS);
3586 }
3587 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3588 }
3589
3590 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
3591 static void
3592 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
3593 struct resource_list *rl)
3594 {
3595 struct resource *res;
3596 char *cp;
3597 rman_res_t start, end, count;
3598 int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;
3599
3600 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3601 case PCIM_HDRTYPE_BRIDGE:
3602 sec_reg = PCIR_SECBUS_1;
3603 sub_reg = PCIR_SUBBUS_1;
3604 break;
3605 case PCIM_HDRTYPE_CARDBUS:
3606 sec_reg = PCIR_SECBUS_2;
3607 sub_reg = PCIR_SUBBUS_2;
3608 break;
3609 default:
3610 return;
3611 }
3612
3613 /*
3614 * If the existing bus range is valid, attempt to reserve it
3615 * from our parent. If this fails for any reason, clear the
3616 * secbus and subbus registers.
3617 *
3618 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
3619 * This would at least preserve the existing sec_bus if it is
3620 * valid.
3621 */
3622 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
3623 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);
3624
3625 /* Quirk handling. */
3626 switch (pci_get_devid(dev)) {
3627 case 0x12258086: /* Intel 82454KX/GX (Orion) */
3628 sup_bus = pci_read_config(dev, 0x41, 1);
3629 if (sup_bus != 0xff) {
3630 sec_bus = sup_bus + 1;
3631 sub_bus = sup_bus + 1;
3632 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
3633 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3634 }
3635 break;
3636
3637 case 0x00dd10de:
3638 /* Compaq R3000 BIOS sets wrong subordinate bus number. */
3639 if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
3640 break;
3641 if (strncmp(cp, "Compal", 6) != 0) {
3642 freeenv(cp);
3643 break;
3644 }
3645 freeenv(cp);
3646 if ((cp = kern_getenv("smbios.planar.product")) == NULL)
3647 break;
3648 if (strncmp(cp, "08A0", 4) != 0) {
3649 freeenv(cp);
3650 break;
3651 }
3652 freeenv(cp);
3653 if (sub_bus < 0xa) {
3654 sub_bus = 0xa;
3655 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3656 }
3657 break;
3658 }
3659
3660 if (bootverbose)
3661 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
3662 if (sec_bus > 0 && sub_bus >= sec_bus) {
3663 start = sec_bus;
3664 end = sub_bus;
3665 count = end - start + 1;
3666
3667 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);
3668
3669 /*
3670 * If requested, clear secondary bus registers in
3671 * bridge devices to force a complete renumbering
3672 * rather than reserving the existing range. However,
3673 * preserve the existing size.
3674 */
3675 if (pci_clear_buses)
3676 goto clear;
3677
3678 rid = 0;
3679 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid,
3680 start, end, count, 0);
3681 if (res != NULL)
3682 return;
3683
3684 if (bootverbose)
3685 device_printf(bus,
3686 "pci%d:%d:%d:%d secbus failed to allocate\n",
3687 pci_get_domain(dev), pci_get_bus(dev),
3688 pci_get_slot(dev), pci_get_function(dev));
3689 }
3690
3691 clear:
3692 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
3693 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
3694 }
3695
3696 static struct resource *
3697 pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start,
3698 rman_res_t end, rman_res_t count, u_int flags)
3699 {
3700 struct pci_devinfo *dinfo;
3701 pcicfgregs *cfg;
3702 struct resource_list *rl;
3703 struct resource *res;
3704 int sec_reg, sub_reg;
3705
3706 dinfo = device_get_ivars(child);
3707 cfg = &dinfo->cfg;
3708 rl = &dinfo->resources;
3709 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3710 case PCIM_HDRTYPE_BRIDGE:
3711 sec_reg = PCIR_SECBUS_1;
3712 sub_reg = PCIR_SUBBUS_1;
3713 break;
3714 case PCIM_HDRTYPE_CARDBUS:
3715 sec_reg = PCIR_SECBUS_2;
3716 sub_reg = PCIR_SUBBUS_2;
3717 break;
3718 default:
3719 return (NULL);
3720 }
3721
3722 if (*rid != 0)
3723 return (NULL);
3724
3725 if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL)
3726 resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count);
3727 if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) {
3728 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
3729 start, end, count, flags & ~RF_ACTIVE);
3730 if (res == NULL) {
3731 resource_list_delete(rl, PCI_RES_BUS, *rid);
3732 device_printf(child, "allocating %ju bus%s failed\n",
3733 count, count == 1 ? "" : "es");
3734 return (NULL);
3735 }
3736 if (bootverbose)
3737 device_printf(child,
3738 "Lazy allocation of %ju bus%s at %ju\n", count,
3739 count == 1 ? "" : "es", rman_get_start(res));
3740 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
3741 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
3742 }
3743 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
3744 end, count, flags));
3745 }
3746 #endif
3747
3748 static int
3749 pci_ea_bei_to_rid(device_t dev, int bei)
3750 {
3751 #ifdef PCI_IOV
3752 struct pci_devinfo *dinfo;
3753 int iov_pos;
3754 struct pcicfg_iov *iov;
3755
3756 dinfo = device_get_ivars(dev);
3757 iov = dinfo->cfg.iov;
3758 if (iov != NULL)
3759 iov_pos = iov->iov_pos;
3760 else
3761 iov_pos = 0;
3762 #endif
3763
3764 /* Check if matches BAR */
3765 if ((bei >= PCIM_EA_BEI_BAR_0) &&
3766 (bei <= PCIM_EA_BEI_BAR_5))
3767 return (PCIR_BAR(bei));
3768
3769 /* Check ROM */
3770 if (bei == PCIM_EA_BEI_ROM)
3771 return (PCIR_BIOS);
3772
3773 #ifdef PCI_IOV
3774 /* Check if matches VF_BAR */
3775 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
3776 (bei <= PCIM_EA_BEI_VF_BAR_5))
3777 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
3778 iov_pos);
3779 #endif
3780
3781 return (-1);
3782 }
3783
3784 int
3785 pci_ea_is_enabled(device_t dev, int rid)
3786 {
3787 struct pci_ea_entry *ea;
3788 struct pci_devinfo *dinfo;
3789
3790 dinfo = device_get_ivars(dev);
3791
3792 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3793 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
3794 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
3795 }
3796
3797 return (0);
3798 }
3799
3800 void
3801 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
3802 {
3803 struct pci_ea_entry *ea;
3804 struct pci_devinfo *dinfo;
3805 pci_addr_t start, end, count;
3806 struct resource_list *rl;
3807 int type, flags, rid;
3808 struct resource *res;
3809 uint32_t tmp;
3810 #ifdef PCI_IOV
3811 struct pcicfg_iov *iov;
3812 #endif
3813
3814 dinfo = device_get_ivars(dev);
3815 rl = &dinfo->resources;
3816 flags = 0;
3817
3818 #ifdef PCI_IOV
3819 iov = dinfo->cfg.iov;
3820 #endif
3821
3822 if (dinfo->cfg.ea.ea_location == 0)
3823 return;
3824
3825 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3826
3827 /*
3828 * TODO: Ignore EA-BAR if is not enabled.
3829 * Currently the EA implementation supports
3830 * only situation, where EA structure contains
3831 * predefined entries. In case they are not enabled
3832 * leave them unallocated and proceed with
3833 * a legacy-BAR mechanism.
3834 */
3835 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
3836 continue;
3837
3838 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
3839 case PCIM_EA_P_MEM_PREFETCH:
3840 case PCIM_EA_P_VF_MEM_PREFETCH:
3841 flags = RF_PREFETCHABLE;
3842 /* FALLTHROUGH */
3843 case PCIM_EA_P_VF_MEM:
3844 case PCIM_EA_P_MEM:
3845 type = SYS_RES_MEMORY;
3846 break;
3847 case PCIM_EA_P_IO:
3848 type = SYS_RES_IOPORT;
3849 break;
3850 default:
3851 continue;
3852 }
3853
3854 if (alloc_iov != 0) {
3855 #ifdef PCI_IOV
3856 /* Allocating IOV, confirm BEI matches */
3857 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) ||
3858 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
3859 continue;
3860 #else
3861 continue;
3862 #endif
3863 } else {
3864 /* Allocating BAR, confirm BEI matches */
3865 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) ||
3866 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
3867 (ea->eae_bei != PCIM_EA_BEI_ROM))
3868 continue;
3869 }
3870
3871 rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
3872 if (rid < 0)
3873 continue;
3874
3875 /* Skip resources already allocated by EA */
3876 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) ||
3877 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
3878 continue;
3879
3880 start = ea->eae_base;
3881 count = ea->eae_max_offset + 1;
3882 #ifdef PCI_IOV
3883 if (iov != NULL)
3884 count = count * iov->iov_num_vfs;
3885 #endif
3886 end = start + count - 1;
3887 if (count == 0)
3888 continue;
3889
3890 resource_list_add(rl, type, rid, start, end, count);
3891 res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count,
3892 flags);
3893 if (res == NULL) {
3894 resource_list_delete(rl, type, rid);
3895
3896 /*
3897 * Failed to allocate using EA, disable entry.
3898 * Another attempt to allocation will be performed
3899 * further, but this time using legacy BAR registers
3900 */
3901 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
3902 tmp &= ~PCIM_EA_ENABLE;
3903 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);
3904
3905 /*
3906 * Disabling entry might fail in case it is hardwired.
3907 * Read flags again to match current status.
3908 */
3909 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);
3910
3911 continue;
3912 }
3913
3914 /* As per specification, fill BAR with zeros */
3915 pci_write_config(dev, rid, 0, 4);
3916 }
3917 }
3918
3919 void
3920 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
3921 {
3922 struct pci_devinfo *dinfo;
3923 pcicfgregs *cfg;
3924 struct resource_list *rl;
3925 const struct pci_quirk *q;
3926 uint32_t devid;
3927 int i;
3928
3929 dinfo = device_get_ivars(dev);
3930 cfg = &dinfo->cfg;
3931 rl = &dinfo->resources;
3932 devid = (cfg->device << 16) | cfg->vendor;
3933
3934 /* Allocate resources using Enhanced Allocation */
3935 pci_add_resources_ea(bus, dev, 0);
3936
3937 /* ATA devices needs special map treatment */
3938 if ((pci_get_class(dev) == PCIC_STORAGE) &&
3939 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
3940 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) ||
3941 (!pci_read_config(dev, PCIR_BAR(0), 4) &&
3942 !pci_read_config(dev, PCIR_BAR(2), 4))) )
3943 pci_ata_maps(bus, dev, rl, force, prefetchmask);
3944 else
3945 for (i = 0; i < cfg->nummaps;) {
3946 /* Skip resources already managed by EA */
3947 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) ||
3948 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) ||
3949 pci_ea_is_enabled(dev, PCIR_BAR(i))) {
3950 i++;
3951 continue;
3952 }
3953
3954 /*
3955 * Skip quirked resources.
3956 */
3957 for (q = &pci_quirks[0]; q->devid != 0; q++)
3958 if (q->devid == devid &&
3959 q->type == PCI_QUIRK_UNMAP_REG &&
3960 q->arg1 == PCIR_BAR(i))
3961 break;
3962 if (q->devid != 0) {
3963 i++;
3964 continue;
3965 }
3966 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
3967 prefetchmask & (1 << i));
3968 }
3969
3970 /*
3971 * Add additional, quirked resources.
3972 */
3973 for (q = &pci_quirks[0]; q->devid != 0; q++)
3974 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
3975 pci_add_map(bus, dev, q->arg1, rl, force, 0);
3976
3977 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) {
3978 #ifdef __PCI_REROUTE_INTERRUPT
3979 /*
3980 * Try to re-route interrupts. Sometimes the BIOS or
3981 * firmware may leave bogus values in these registers.
3982 * If the re-route fails, then just stick with what we
3983 * have.
3984 */
3985 pci_assign_interrupt(bus, dev, 1);
3986 #else
3987 pci_assign_interrupt(bus, dev, 0);
3988 #endif
3989 }
3990
3991 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
3992 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
3993 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
3994 xhci_early_takeover(dev);
3995 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
3996 ehci_early_takeover(dev);
3997 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
3998 ohci_early_takeover(dev);
3999 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
4000 uhci_early_takeover(dev);
4001 }
4002
4003 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
4004 /*
4005 * Reserve resources for secondary bus ranges behind bridge
4006 * devices.
4007 */
4008 pci_reserve_secbus(bus, dev, cfg, rl);
4009 #endif
4010 }
4011
4012 static struct pci_devinfo *
4013 pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
4014 int slot, int func)
4015 {
4016 struct pci_devinfo *dinfo;
4017
4018 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
4019 if (dinfo != NULL)
4020 pci_add_child(dev, dinfo);
4021
4022 return (dinfo);
4023 }
4024
4025 void
4026 pci_add_children(device_t dev, int domain, int busno)
4027 {
4028 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4029 device_t pcib = device_get_parent(dev);
4030 struct pci_devinfo *dinfo;
4031 int maxslots;
4032 int s, f, pcifunchigh;
4033 uint8_t hdrtype;
4034 int first_func;
4035
4036 /*
4037 * Try to detect a device at slot 0, function 0. If it exists, try to
4038 * enable ARI. We must enable ARI before detecting the rest of the
4039 * functions on this bus as ARI changes the set of slots and functions
4040 * that are legal on this bus.
4041 */
4042 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
4043 if (dinfo != NULL && pci_enable_ari)
4044 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
4045
4046 /*
4047 * Start looking for new devices on slot 0 at function 1 because we
4048 * just identified the device at slot 0, function 0.
4049 */
4050 first_func = 1;
4051
4052 maxslots = PCIB_MAXSLOTS(pcib);
4053 for (s = 0; s <= maxslots; s++, first_func = 0) {
4054 pcifunchigh = 0;
4055 f = 0;
4056 DELAY(1);
4057 hdrtype = REG(PCIR_HDRTYPE, 1);
4058 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4059 continue;
4060 if (hdrtype & PCIM_MFDEV)
4061 pcifunchigh = PCIB_MAXFUNCS(pcib);
4062 for (f = first_func; f <= pcifunchigh; f++)
4063 pci_identify_function(pcib, dev, domain, busno, s, f);
4064 }
4065 #undef REG
4066 }
4067
4068 int
4069 pci_rescan_method(device_t dev)
4070 {
4071 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4072 device_t pcib = device_get_parent(dev);
4073 struct pci_softc *sc;
4074 device_t child, *devlist, *unchanged;
4075 int devcount, error, i, j, maxslots, oldcount;
4076 int busno, domain, s, f, pcifunchigh;
4077 uint8_t hdrtype;
4078
4079 /* No need to check for ARI on a rescan. */
4080 error = device_get_children(dev, &devlist, &devcount);
4081 if (error)
4082 return (error);
4083 if (devcount != 0) {
4084 unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
4085 M_NOWAIT | M_ZERO);
4086 if (unchanged == NULL) {
4087 free(devlist, M_TEMP);
4088 return (ENOMEM);
4089 }
4090 } else
4091 unchanged = NULL;
4092
4093 sc = device_get_softc(dev);
4094 domain = pcib_get_domain(dev);
4095 busno = pcib_get_bus(dev);
4096 maxslots = PCIB_MAXSLOTS(pcib);
4097 for (s = 0; s <= maxslots; s++) {
4098 /* If function 0 is not present, skip to the next slot. */
4099 f = 0;
4100 if (REG(PCIR_VENDOR, 2) == 0xffff)
4101 continue;
4102 pcifunchigh = 0;
4103 hdrtype = REG(PCIR_HDRTYPE, 1);
4104 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4105 continue;
4106 if (hdrtype & PCIM_MFDEV)
4107 pcifunchigh = PCIB_MAXFUNCS(pcib);
4108 for (f = 0; f <= pcifunchigh; f++) {
4109 if (REG(PCIR_VENDOR, 2) == 0xffff)
4110 continue;
4111
4112 /*
4113 * Found a valid function. Check if a
4114 * device_t for this device already exists.
4115 */
4116 for (i = 0; i < devcount; i++) {
4117 child = devlist[i];
4118 if (child == NULL)
4119 continue;
4120 if (pci_get_slot(child) == s &&
4121 pci_get_function(child) == f) {
4122 unchanged[i] = child;
4123 goto next_func;
4124 }
4125 }
4126
4127 pci_identify_function(pcib, dev, domain, busno, s, f);
4128 next_func:;
4129 }
4130 }
4131
4132 /* Remove devices that are no longer present. */
4133 for (i = 0; i < devcount; i++) {
4134 if (unchanged[i] != NULL)
4135 continue;
4136 device_delete_child(dev, devlist[i]);
4137 }
4138
4139 free(devlist, M_TEMP);
4140 oldcount = devcount;
4141
4142 /* Try to attach the devices just added. */
4143 error = device_get_children(dev, &devlist, &devcount);
4144 if (error) {
4145 free(unchanged, M_TEMP);
4146 return (error);
4147 }
4148
4149 for (i = 0; i < devcount; i++) {
4150 for (j = 0; j < oldcount; j++) {
4151 if (devlist[i] == unchanged[j])
4152 goto next_device;
4153 }
4154
4155 device_probe_and_attach(devlist[i]);
4156 next_device:;
4157 }
4158
4159 free(unchanged, M_TEMP);
4160 free(devlist, M_TEMP);
4161 return (0);
4162 #undef REG
4163 }
4164
4165 #ifdef PCI_IOV
4166 device_t
4167 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
4168 uint16_t did)
4169 {
4170 struct pci_devinfo *pf_dinfo, *vf_dinfo;
4171 device_t pcib;
4172 int busno, slot, func;
4173
4174 pf_dinfo = device_get_ivars(pf);
4175
4176 pcib = device_get_parent(bus);
4177
4178 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);
4179
4180 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
4181 slot, func, vid, did);
4182
4183 vf_dinfo->cfg.flags |= PCICFG_VF;
4184 pci_add_child(bus, vf_dinfo);
4185
4186 return (vf_dinfo->cfg.dev);
4187 }
4188
4189 device_t
4190 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
4191 uint16_t vid, uint16_t did)
4192 {
4193
4194 return (pci_add_iov_child(bus, pf, rid, vid, did));
4195 }
4196 #endif
4197
4198 void
4199 pci_add_child(device_t bus, struct pci_devinfo *dinfo)
4200 {
4201 dinfo->cfg.dev = device_add_child(bus, NULL, -1);
4202 device_set_ivars(dinfo->cfg.dev, dinfo);
4203 resource_list_init(&dinfo->resources);
4204 pci_cfg_save(dinfo->cfg.dev, dinfo, 0);
4205 pci_cfg_restore(dinfo->cfg.dev, dinfo);
4206 pci_print_verbose(dinfo);
4207 pci_add_resources(bus, dinfo->cfg.dev, 0, 0);
4208 pci_child_added(dinfo->cfg.dev);
4209 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
4210 }
4211
4212 void
4213 pci_child_added_method(device_t dev, device_t child)
4214 {
4215
4216 }
4217
4218 static int
4219 pci_probe(device_t dev)
4220 {
4221
4222 device_set_desc(dev, "PCI bus");
4223
4224 /* Allow other subclasses to override this driver. */
4225 return (BUS_PROBE_GENERIC);
4226 }
4227
4228 int
4229 pci_attach_common(device_t dev)
4230 {
4231 struct pci_softc *sc;
4232 int busno, domain;
4233 #ifdef PCI_DMA_BOUNDARY
4234 int error, tag_valid;
4235 #endif
4236 #ifdef PCI_RES_BUS
4237 int rid;
4238 #endif
4239
4240 sc = device_get_softc(dev);
4241 domain = pcib_get_domain(dev);
4242 busno = pcib_get_bus(dev);
4243 #ifdef PCI_RES_BUS
4244 rid = 0;
4245 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
4246 1, 0);
4247 if (sc->sc_bus == NULL) {
4248 device_printf(dev, "failed to allocate bus number\n");
4249 return (ENXIO);
4250 }
4251 #endif
4252 if (bootverbose)
4253 device_printf(dev, "domain=%d, physical bus=%d\n",
4254 domain, busno);
4255 #ifdef PCI_DMA_BOUNDARY
4256 tag_valid = 0;
4257 if (device_get_devclass(device_get_parent(device_get_parent(dev))) !=
4258 devclass_find("pci")) {
4259 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1,
4260 PCI_DMA_BOUNDARY, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4261 NULL, NULL, BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED,
4262 BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->sc_dma_tag);
4263 if (error)
4264 device_printf(dev, "Failed to create DMA tag: %d\n",
4265 error);
4266 else
4267 tag_valid = 1;
4268 }
4269 if (!tag_valid)
4270 #endif
4271 sc->sc_dma_tag = bus_get_dma_tag(dev);
4272 return (0);
4273 }
4274
4275 static int
4276 pci_attach(device_t dev)
4277 {
4278 int busno, domain, error;
4279
4280 error = pci_attach_common(dev);
4281 if (error)
4282 return (error);
4283
4284 /*
4285 * Since there can be multiple independently numbered PCI
4286 * busses on systems with multiple PCI domains, we can't use
4287 * the unit number to decide which bus we are probing. We ask
4288 * the parent pcib what our domain and bus numbers are.
4289 */
4290 domain = pcib_get_domain(dev);
4291 busno = pcib_get_bus(dev);
4292 pci_add_children(dev, domain, busno);
4293 return (bus_generic_attach(dev));
4294 }
4295
4296 static int
4297 pci_detach(device_t dev)
4298 {
4299 #ifdef PCI_RES_BUS
4300 struct pci_softc *sc;
4301 #endif
4302 int error;
4303
4304 error = bus_generic_detach(dev);
4305 if (error)
4306 return (error);
4307 #ifdef PCI_RES_BUS
4308 sc = device_get_softc(dev);
4309 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
4310 if (error)
4311 return (error);
4312 #endif
4313 return (device_delete_children(dev));
4314 }
4315
4316 static void
4317 pci_set_power_child(device_t dev, device_t child, int state)
4318 {
4319 device_t pcib;
4320 int dstate;
4321
4322 /*
4323 * Set the device to the given state. If the firmware suggests
4324 * a different power state, use it instead. If power management
4325 * is not present, the firmware is responsible for managing
4326 * device power. Skip children who aren't attached since they
4327 * are handled separately.
4328 */
4329 pcib = device_get_parent(dev);
4330 dstate = state;
4331 if (device_is_attached(child) &&
4332 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
4333 pci_set_powerstate(child, dstate);
4334 }
4335
4336 int
4337 pci_suspend_child(device_t dev, device_t child)
4338 {
4339 struct pci_devinfo *dinfo;
4340 int error;
4341
4342 dinfo = device_get_ivars(child);
4343
4344 /*
4345 * Save the PCI configuration space for the child and set the
4346 * device in the appropriate power state for this sleep state.
4347 */
4348 pci_cfg_save(child, dinfo, 0);
4349
4350 /* Suspend devices before potentially powering them down. */
4351 error = bus_generic_suspend_child(dev, child);
4352
4353 if (error)
4354 return (error);
4355
4356 if (pci_do_power_suspend)
4357 pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
4358
4359 return (0);
4360 }
4361
4362 int
4363 pci_resume_child(device_t dev, device_t child)
4364 {
4365 struct pci_devinfo *dinfo;
4366
4367 if (pci_do_power_resume)
4368 pci_set_power_child(dev, child, PCI_POWERSTATE_D0);
4369
4370 dinfo = device_get_ivars(child);
4371 pci_cfg_restore(child, dinfo);
4372 if (!device_is_attached(child))
4373 pci_cfg_save(child, dinfo, 1);
4374
4375 bus_generic_resume_child(dev, child);
4376
4377 return (0);
4378 }
4379
4380 int
4381 pci_resume(device_t dev)
4382 {
4383 device_t child, *devlist;
4384 int error, i, numdevs;
4385
4386 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
4387 return (error);
4388
4389 /*
4390 * Resume critical devices first, then everything else later.
4391 */
4392 for (i = 0; i < numdevs; i++) {
4393 child = devlist[i];
4394 switch (pci_get_class(child)) {
4395 case PCIC_DISPLAY:
4396 case PCIC_MEMORY:
4397 case PCIC_BRIDGE:
4398 case PCIC_BASEPERIPH:
4399 BUS_RESUME_CHILD(dev, child);
4400 break;
4401 }
4402 }
4403 for (i = 0; i < numdevs; i++) {
4404 child = devlist[i];
4405 switch (pci_get_class(child)) {
4406 case PCIC_DISPLAY:
4407 case PCIC_MEMORY:
4408 case PCIC_BRIDGE:
4409 case PCIC_BASEPERIPH:
4410 break;
4411 default:
4412 BUS_RESUME_CHILD(dev, child);
4413 }
4414 }
4415 free(devlist, M_TEMP);
4416 return (0);
4417 }
4418
4419 static void
4420 pci_load_vendor_data(void)
4421 {
4422 caddr_t data;
4423 void *ptr;
4424 size_t sz;
4425
4426 data = preload_search_by_type("pci_vendor_data");
4427 if (data != NULL) {
4428 ptr = preload_fetch_addr(data);
4429 sz = preload_fetch_size(data);
4430 if (ptr != NULL && sz != 0) {
4431 pci_vendordata = ptr;
4432 pci_vendordata_size = sz;
4433 /* terminate the database */
4434 pci_vendordata[pci_vendordata_size] = '\n';
4435 }
4436 }
4437 }
4438
4439 void
4440 pci_driver_added(device_t dev, driver_t *driver)
4441 {
4442 int numdevs;
4443 device_t *devlist;
4444 device_t child;
4445 struct pci_devinfo *dinfo;
4446 int i;
4447
4448 if (bootverbose)
4449 device_printf(dev, "driver added\n");
4450 DEVICE_IDENTIFY(driver, dev);
4451 if (device_get_children(dev, &devlist, &numdevs) != 0)
4452 return;
4453 for (i = 0; i < numdevs; i++) {
4454 child = devlist[i];
4455 if (device_get_state(child) != DS_NOTPRESENT)
4456 continue;
4457 dinfo = device_get_ivars(child);
4458 pci_print_verbose(dinfo);
4459 if (bootverbose)
4460 pci_printf(&dinfo->cfg, "reprobing on driver added\n");
4461 pci_cfg_restore(child, dinfo);
4462 if (device_probe_and_attach(child) != 0)
4463 pci_child_detached(dev, child);
4464 }
4465 free(devlist, M_TEMP);
4466 }
4467
4468 int
4469 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
4470 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep)
4471 {
4472 struct pci_devinfo *dinfo;
4473 struct msix_table_entry *mte;
4474 struct msix_vector *mv;
4475 uint64_t addr;
4476 uint32_t data;
4477 void *cookie;
4478 int error, rid;
4479
4480 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
4481 arg, &cookie);
4482 if (error)
4483 return (error);
4484
4485 /* If this is not a direct child, just bail out. */
4486 if (device_get_parent(child) != dev) {
4487 *cookiep = cookie;
4488 return(0);
4489 }
4490
4491 rid = rman_get_rid(irq);
4492 if (rid == 0) {
4493 /* Make sure that INTx is enabled */
4494 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4495 } else {
4496 /*
4497 * Check to see if the interrupt is MSI or MSI-X.
4498 * Ask our parent to map the MSI and give
4499 * us the address and data register values.
4500 * If we fail for some reason, teardown the
4501 * interrupt handler.
4502 */
4503 dinfo = device_get_ivars(child);
4504 if (dinfo->cfg.msi.msi_alloc > 0) {
4505 if (dinfo->cfg.msi.msi_addr == 0) {
4506 KASSERT(dinfo->cfg.msi.msi_handlers == 0,
4507 ("MSI has handlers, but vectors not mapped"));
4508 error = PCIB_MAP_MSI(device_get_parent(dev),
4509 child, rman_get_start(irq), &addr, &data);
4510 if (error)
4511 goto bad;
4512 dinfo->cfg.msi.msi_addr = addr;
4513 dinfo->cfg.msi.msi_data = data;
4514 }
4515 if (dinfo->cfg.msi.msi_handlers == 0)
4516 pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
4517 dinfo->cfg.msi.msi_data);
4518 dinfo->cfg.msi.msi_handlers++;
4519 } else {
4520 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4521 ("No MSI or MSI-X interrupts allocated"));
4522 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4523 ("MSI-X index too high"));
4524 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4525 KASSERT(mte->mte_vector != 0, ("no message vector"));
4526 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
4527 KASSERT(mv->mv_irq == rman_get_start(irq),
4528 ("IRQ mismatch"));
4529 if (mv->mv_address == 0) {
4530 KASSERT(mte->mte_handlers == 0,
4531 ("MSI-X table entry has handlers, but vector not mapped"));
4532 error = PCIB_MAP_MSI(device_get_parent(dev),
4533 child, rman_get_start(irq), &addr, &data);
4534 if (error)
4535 goto bad;
4536 mv->mv_address = addr;
4537 mv->mv_data = data;
4538 }
4539
4540 /*
4541 * The MSIX table entry must be made valid by
4542 * incrementing the mte_handlers before
4543 * calling pci_enable_msix() and
4544 * pci_resume_msix(). Else the MSIX rewrite
4545 * table quirk will not work as expected.
4546 */
4547 mte->mte_handlers++;
4548 if (mte->mte_handlers == 1) {
4549 pci_enable_msix(child, rid - 1, mv->mv_address,
4550 mv->mv_data);
4551 pci_unmask_msix(child, rid - 1);
4552 }
4553 }
4554
4555 /*
4556 * Make sure that INTx is disabled if we are using MSI/MSI-X,
4557 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
4558 * in which case we "enable" INTx so MSI/MSI-X actually works.
4559 */
4560 if (!pci_has_quirk(pci_get_devid(child),
4561 PCI_QUIRK_MSI_INTX_BUG))
4562 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4563 else
4564 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4565 bad:
4566 if (error) {
4567 (void)bus_generic_teardown_intr(dev, child, irq,
4568 cookie);
4569 return (error);
4570 }
4571 }
4572 *cookiep = cookie;
4573 return (0);
4574 }
4575
4576 int
4577 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
4578 void *cookie)
4579 {
4580 struct msix_table_entry *mte;
4581 struct resource_list_entry *rle;
4582 struct pci_devinfo *dinfo;
4583 int error, rid;
4584
4585 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
4586 return (EINVAL);
4587
4588 /* If this isn't a direct child, just bail out */
4589 if (device_get_parent(child) != dev)
4590 return(bus_generic_teardown_intr(dev, child, irq, cookie));
4591
4592 rid = rman_get_rid(irq);
4593 if (rid == 0) {
4594 /* Mask INTx */
4595 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4596 } else {
4597 /*
4598 * Check to see if the interrupt is MSI or MSI-X. If so,
4599 * decrement the appropriate handlers count and mask the
4600 * MSI-X message, or disable MSI messages if the count
4601 * drops to 0.
4602 */
4603 dinfo = device_get_ivars(child);
4604 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
4605 if (rle->res != irq)
4606 return (EINVAL);
4607 if (dinfo->cfg.msi.msi_alloc > 0) {
4608 KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
4609 ("MSI-X index too high"));
4610 if (dinfo->cfg.msi.msi_handlers == 0)
4611 return (EINVAL);
4612 dinfo->cfg.msi.msi_handlers--;
4613 if (dinfo->cfg.msi.msi_handlers == 0)
4614 pci_disable_msi(child);
4615 } else {
4616 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4617 ("No MSI or MSI-X interrupts allocated"));
4618 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4619 ("MSI-X index too high"));
4620 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4621 if (mte->mte_handlers == 0)
4622 return (EINVAL);
4623 mte->mte_handlers--;
4624 if (mte->mte_handlers == 0)
4625 pci_mask_msix(child, rid - 1);
4626 }
4627 }
4628 error = bus_generic_teardown_intr(dev, child, irq, cookie);
4629 if (rid > 0)
4630 KASSERT(error == 0,
4631 ("%s: generic teardown failed for MSI/MSI-X", __func__));
4632 return (error);
4633 }
4634
4635 int
4636 pci_print_child(device_t dev, device_t child)
4637 {
4638 struct pci_devinfo *dinfo;
4639 struct resource_list *rl;
4640 int retval = 0;
4641
4642 dinfo = device_get_ivars(child);
4643 rl = &dinfo->resources;
4644
4645 retval += bus_print_child_header(dev, child);
4646
4647 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
4648 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
4649 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
4650 if (device_get_flags(dev))
4651 retval += printf(" flags %#x", device_get_flags(dev));
4652
4653 retval += printf(" at device %d.%d", pci_get_slot(child),
4654 pci_get_function(child));
4655
4656 retval += bus_print_child_domain(dev, child);
4657 retval += bus_print_child_footer(dev, child);
4658
4659 return (retval);
4660 }
4661
4662 static const struct
4663 {
4664 int class;
4665 int subclass;
4666 int report; /* 0 = bootverbose, 1 = always */
4667 const char *desc;
4668 } pci_nomatch_tab[] = {
4669 {PCIC_OLD, -1, 1, "old"},
4670 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
4671 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
4672 {PCIC_STORAGE, -1, 1, "mass storage"},
4673 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
4674 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
4675 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
4676 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
4677 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
4678 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
4679 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
4680 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
4681 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
4682 {PCIC_NETWORK, -1, 1, "network"},
4683 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
4684 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
4685 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
4686 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
4687 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
4688 {PCIC_DISPLAY, -1, 1, "display"},
4689 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
4690 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
4691 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
4692 {PCIC_MULTIMEDIA, -1, 1, "multimedia"},
4693 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
4694 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
4695 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
4696 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
4697 {PCIC_MEMORY, -1, 1, "memory"},
4698 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
4699 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
4700 {PCIC_BRIDGE, -1, 1, "bridge"},
4701 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
4702 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
4703 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
4704 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
4705 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
4706 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
4707 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
4708 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
4709 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
4710 {PCIC_SIMPLECOMM, -1, 1, "simple comms"},
4711 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
4712 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
4713 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
4714 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
4715 {PCIC_BASEPERIPH, -1, 0, "base peripheral"},
4716 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
4717 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
4718 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
4719 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
4720 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
4721 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
4722 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
4723 {PCIC_INPUTDEV, -1, 1, "input device"},
4724 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
4725 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
4726 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
4727 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
4728 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
4729 {PCIC_DOCKING, -1, 1, "docking station"},
4730 {PCIC_PROCESSOR, -1, 1, "processor"},
4731 {PCIC_SERIALBUS, -1, 1, "serial bus"},
4732 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
4733 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
4734 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
4735 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
4736 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
4737 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
4738 {PCIC_WIRELESS, -1, 1, "wireless controller"},
4739 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
4740 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
4741 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
4742 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
4743 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
4744 {PCIC_SATCOM, -1, 1, "satellite communication"},
4745 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
4746 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
4747 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
4748 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
4749 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
4750 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
4751 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
4752 {PCIC_DASP, -1, 0, "dasp"},
4753 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
4754 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
4755 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
4756 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
4757 {0, 0, 0, NULL}
4758 };
4759
4760 void
4761 pci_probe_nomatch(device_t dev, device_t child)
4762 {
4763 int i, report;
4764 const char *cp, *scp;
4765 char *device;
4766
4767 /*
4768 * Look for a listing for this device in a loaded device database.
4769 */
4770 report = 1;
4771 if ((device = pci_describe_device(child)) != NULL) {
4772 device_printf(dev, "<%s>", device);
4773 free(device, M_DEVBUF);
4774 } else {
4775 /*
4776 * Scan the class/subclass descriptions for a general
4777 * description.
4778 */
4779 cp = "unknown";
4780 scp = NULL;
4781 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
4782 if (pci_nomatch_tab[i].class == pci_get_class(child)) {
4783 if (pci_nomatch_tab[i].subclass == -1) {
4784 cp = pci_nomatch_tab[i].desc;
4785 report = pci_nomatch_tab[i].report;
4786 } else if (pci_nomatch_tab[i].subclass ==
4787 pci_get_subclass(child)) {
4788 scp = pci_nomatch_tab[i].desc;
4789 report = pci_nomatch_tab[i].report;
4790 }
4791 }
4792 }
4793 if (report || bootverbose) {
4794 device_printf(dev, "<%s%s%s>",
4795 cp ? cp : "",
4796 ((cp != NULL) && (scp != NULL)) ? ", " : "",
4797 scp ? scp : "");
4798 }
4799 }
4800 if (report || bootverbose) {
4801 printf(" at device %d.%d (no driver attached)\n",
4802 pci_get_slot(child), pci_get_function(child));
4803 }
4804 pci_cfg_save(child, device_get_ivars(child), 1);
4805 }
4806
4807 void
4808 pci_child_detached(device_t dev, device_t child)
4809 {
4810 struct pci_devinfo *dinfo;
4811 struct resource_list *rl;
4812
4813 dinfo = device_get_ivars(child);
4814 rl = &dinfo->resources;
4815
4816 /*
4817 * Have to deallocate IRQs before releasing any MSI messages and
4818 * have to release MSI messages before deallocating any memory
4819 * BARs.
4820 */
4821 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
4822 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
4823 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) {
4824 pci_printf(&dinfo->cfg, "Device leaked MSI vectors\n");
4825 (void)pci_release_msi(child);
4826 }
4827 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
4828 pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
4829 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
4830 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
4831 #ifdef PCI_RES_BUS
4832 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
4833 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
4834 #endif
4835
4836 pci_cfg_save(child, dinfo, 1);
4837 }
4838
4839 /*
4840 * Parse the PCI device database, if loaded, and return a pointer to a
4841 * description of the device.
4842 *
4843 * The database is flat text formatted as follows:
4844 *
4845 * Any line not in a valid format is ignored.
4846 * Lines are terminated with newline '\n' characters.
4847 *
4848 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
4849 * the vendor name.
4850 *
4851 * A DEVICE line is entered immediately below the corresponding VENDOR ID.
4852 * - devices cannot be listed without a corresponding VENDOR line.
4853 * A DEVICE line consists of a TAB, the 4 digit (hex) device code,
4854 * another TAB, then the device name.
4855 */
4856
4857 /*
4858 * Assuming (ptr) points to the beginning of a line in the database,
4859 * return the vendor or device and description of the next entry.
4860 * The value of (vendor) or (device) inappropriate for the entry type
4861 * is set to -1. Returns nonzero at the end of the database.
4862 *
4863 * Note that this is slightly unrobust in the face of corrupt data;
4864 * we attempt to safeguard against this by spamming the end of the
4865 * database with a newline when we initialise.
4866 */
4867 static int
4868 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc)
4869 {
4870 char *cp = *ptr;
4871 int left;
4872
4873 *device = -1;
4874 *vendor = -1;
4875 **desc = '\0';
4876 for (;;) {
4877 left = pci_vendordata_size - (cp - pci_vendordata);
4878 if (left <= 0) {
4879 *ptr = cp;
4880 return(1);
4881 }
4882
4883 /* vendor entry? */
4884 if (*cp != '\t' &&
4885 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
4886 break;
4887 /* device entry? */
4888 if (*cp == '\t' &&
4889 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
4890 break;
4891
4892 /* skip to next line */
4893 while (*cp != '\n' && left > 0) {
4894 cp++;
4895 left--;
4896 }
4897 if (*cp == '\n') {
4898 cp++;
4899 left--;
4900 }
4901 }
4902 /* skip to next line */
4903 while (*cp != '\n' && left > 0) {
4904 cp++;
4905 left--;
4906 }
4907 if (*cp == '\n' && left > 0)
4908 cp++;
4909 *ptr = cp;
4910 return(0);
4911 }
4912
4913 static char *
4914 pci_describe_device(device_t dev)
4915 {
4916 int vendor, device;
4917 char *desc, *vp, *dp, *line;
4918
4919 desc = vp = dp = NULL;
4920
4921 /*
4922 * If we have no vendor data, we can't do anything.
4923 */
4924 if (pci_vendordata == NULL)
4925 goto out;
4926
4927 /*
4928 * Scan the vendor data looking for this device
4929 */
4930 line = pci_vendordata;
4931 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
4932 goto out;
4933 for (;;) {
4934 if (pci_describe_parse_line(&line, &vendor, &device, &vp))
4935 goto out;
4936 if (vendor == pci_get_vendor(dev))
4937 break;
4938 }
4939 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
4940 goto out;
4941 for (;;) {
4942 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
4943 *dp = 0;
4944 break;
4945 }
4946 if (vendor != -1) {
4947 *dp = 0;
4948 break;
4949 }
4950 if (device == pci_get_device(dev))
4951 break;
4952 }
4953 if (dp[0] == '\0')
4954 snprintf(dp, 80, "0x%x", pci_get_device(dev));
4955 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
4956 NULL)
4957 sprintf(desc, "%s, %s", vp, dp);
4958 out:
4959 if (vp != NULL)
4960 free(vp, M_DEVBUF);
4961 if (dp != NULL)
4962 free(dp, M_DEVBUF);
4963 return(desc);
4964 }
4965
4966 int
4967 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
4968 {
4969 struct pci_devinfo *dinfo;
4970 pcicfgregs *cfg;
4971
4972 dinfo = device_get_ivars(child);
4973 cfg = &dinfo->cfg;
4974
4975 switch (which) {
4976 case PCI_IVAR_ETHADDR:
4977 /*
4978 * The generic accessor doesn't deal with failure, so
4979 * we set the return value, then return an error.
4980 */
4981 *((uint8_t **) result) = NULL;
4982 return (EINVAL);
4983 case PCI_IVAR_SUBVENDOR:
4984 *result = cfg->subvendor;
4985 break;
4986 case PCI_IVAR_SUBDEVICE:
4987 *result = cfg->subdevice;
4988 break;
4989 case PCI_IVAR_VENDOR:
4990 *result = cfg->vendor;
4991 break;
4992 case PCI_IVAR_DEVICE:
4993 *result = cfg->device;
4994 break;
4995 case PCI_IVAR_DEVID:
4996 *result = (cfg->device << 16) | cfg->vendor;
4997 break;
4998 case PCI_IVAR_CLASS:
4999 *result = cfg->baseclass;
5000 break;
5001 case PCI_IVAR_SUBCLASS:
5002 *result = cfg->subclass;
5003 break;
5004 case PCI_IVAR_PROGIF:
5005 *result = cfg->progif;
5006 break;
5007 case PCI_IVAR_REVID:
5008 *result = cfg->revid;
5009 break;
5010 case PCI_IVAR_INTPIN:
5011 *result = cfg->intpin;
5012 break;
5013 case PCI_IVAR_IRQ:
5014 *result = cfg->intline;
5015 break;
5016 case PCI_IVAR_DOMAIN:
5017 *result = cfg->domain;
5018 break;
5019 case PCI_IVAR_BUS:
5020 *result = cfg->bus;
5021 break;
5022 case PCI_IVAR_SLOT:
5023 *result = cfg->slot;
5024 break;
5025 case PCI_IVAR_FUNCTION:
5026 *result = cfg->func;
5027 break;
5028 case PCI_IVAR_CMDREG:
5029 *result = cfg->cmdreg;
5030 break;
5031 case PCI_IVAR_CACHELNSZ:
5032 *result = cfg->cachelnsz;
5033 break;
5034 case PCI_IVAR_MINGNT:
5035 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5036 *result = -1;
5037 return (EINVAL);
5038 }
5039 *result = cfg->mingnt;
5040 break;
5041 case PCI_IVAR_MAXLAT:
5042 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5043 *result = -1;
5044 return (EINVAL);
5045 }
5046 *result = cfg->maxlat;
5047 break;
5048 case PCI_IVAR_LATTIMER:
5049 *result = cfg->lattimer;
5050 break;
5051 default:
5052 return (ENOENT);
5053 }
5054 return (0);
5055 }
5056
5057 int
5058 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
5059 {
5060 struct pci_devinfo *dinfo;
5061
5062 dinfo = device_get_ivars(child);
5063
5064 switch (which) {
5065 case PCI_IVAR_INTPIN:
5066 dinfo->cfg.intpin = value;
5067 return (0);
5068 case PCI_IVAR_ETHADDR:
5069 case PCI_IVAR_SUBVENDOR:
5070 case PCI_IVAR_SUBDEVICE:
5071 case PCI_IVAR_VENDOR:
5072 case PCI_IVAR_DEVICE:
5073 case PCI_IVAR_DEVID:
5074 case PCI_IVAR_CLASS:
5075 case PCI_IVAR_SUBCLASS:
5076 case PCI_IVAR_PROGIF:
5077 case PCI_IVAR_REVID:
5078 case PCI_IVAR_IRQ:
5079 case PCI_IVAR_DOMAIN:
5080 case PCI_IVAR_BUS:
5081 case PCI_IVAR_SLOT:
5082 case PCI_IVAR_FUNCTION:
5083 return (EINVAL); /* disallow for now */
5084
5085 default:
5086 return (ENOENT);
5087 }
5088 }
5089
5090 #include "opt_ddb.h"
5091 #ifdef DDB
5092 #include <ddb/ddb.h>
5093 #include <sys/cons.h>
5094
5095 /*
5096 * List resources based on pci map registers, used for within ddb
5097 */
5098
5099 DB_SHOW_COMMAND(pciregs, db_pci_dump)
5100 {
5101 struct pci_devinfo *dinfo;
5102 struct devlist *devlist_head;
5103 struct pci_conf *p;
5104 const char *name;
5105 int i, error, none_count;
5106
5107 none_count = 0;
5108 /* get the head of the device queue */
5109 devlist_head = &pci_devq;
5110
5111 /*
5112 * Go through the list of devices and print out devices
5113 */
5114 for (error = 0, i = 0,
5115 dinfo = STAILQ_FIRST(devlist_head);
5116 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
5117 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {
5118
5119 /* Populate pd_name and pd_unit */
5120 name = NULL;
5121 if (dinfo->cfg.dev)
5122 name = device_get_name(dinfo->cfg.dev);
5123
5124 p = &dinfo->conf;
5125 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
5126 "chip=0x%08x rev=0x%02x hdr=0x%02x\n",
5127 (name && *name) ? name : "none",
5128 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
5129 none_count++,
5130 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
5131 p->pc_sel.pc_func, (p->pc_class << 16) |
5132 (p->pc_subclass << 8) | p->pc_progif,
5133 (p->pc_subdevice << 16) | p->pc_subvendor,
5134 (p->pc_device << 16) | p->pc_vendor,
5135 p->pc_revid, p->pc_hdr);
5136 }
5137 }
5138 #endif /* DDB */
5139
5140 static struct resource *
5141 pci_reserve_map(device_t dev, device_t child, int type, int *rid,
5142 rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
5143 u_int flags)
5144 {
5145 struct pci_devinfo *dinfo = device_get_ivars(child);
5146 struct resource_list *rl = &dinfo->resources;
5147 struct resource *res;
5148 struct pci_map *pm;
5149 uint16_t cmd;
5150 pci_addr_t map, testval;
5151 int mapsize;
5152
5153 res = NULL;
5154
5155 /* If rid is managed by EA, ignore it */
5156 if (pci_ea_is_enabled(child, *rid))
5157 goto out;
5158
5159 pm = pci_find_bar(child, *rid);
5160 if (pm != NULL) {
5161 /* This is a BAR that we failed to allocate earlier. */
5162 mapsize = pm->pm_size;
5163 map = pm->pm_value;
5164 } else {
5165 /*
5166 * Weed out the bogons, and figure out how large the
5167 * BAR/map is. BARs that read back 0 here are bogus
5168 * and unimplemented. Note: atapci in legacy mode are
5169 * special and handled elsewhere in the code. If you
5170 * have a atapci device in legacy mode and it fails
5171 * here, that other code is broken.
5172 */
5173 pci_read_bar(child, *rid, &map, &testval, NULL);
5174
5175 /*
5176 * Determine the size of the BAR and ignore BARs with a size
5177 * of 0. Device ROM BARs use a different mask value.
5178 */
5179 if (PCIR_IS_BIOS(&dinfo->cfg, *rid))
5180 mapsize = pci_romsize(testval);
5181 else
5182 mapsize = pci_mapsize(testval);
5183 if (mapsize == 0)
5184 goto out;
5185 pm = pci_add_bar(child, *rid, map, mapsize);
5186 }
5187
5188 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, *rid)) {
5189 if (type != SYS_RES_MEMORY) {
5190 if (bootverbose)
5191 device_printf(dev,
5192 "child %s requested type %d for rid %#x,"
5193 " but the BAR says it is an memio\n",
5194 device_get_nameunit(child), type, *rid);
5195 goto out;
5196 }
5197 } else {
5198 if (type != SYS_RES_IOPORT) {
5199 if (bootverbose)
5200 device_printf(dev,
5201 "child %s requested type %d for rid %#x,"
5202 " but the BAR says it is an ioport\n",
5203 device_get_nameunit(child), type, *rid);
5204 goto out;
5205 }
5206 }
5207
5208 /*
5209 * For real BARs, we need to override the size that
5210 * the driver requests, because that's what the BAR
5211 * actually uses and we would otherwise have a
5212 * situation where we might allocate the excess to
5213 * another driver, which won't work.
5214 */
5215 count = ((pci_addr_t)1 << mapsize) * num;
5216 if (RF_ALIGNMENT(flags) < mapsize)
5217 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize);
5218 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
5219 flags |= RF_PREFETCHABLE;
5220
5221 /*
5222 * Allocate enough resource, and then write back the
5223 * appropriate BAR for that resource.
5224 */
5225 resource_list_add(rl, type, *rid, start, end, count);
5226 res = resource_list_reserve(rl, dev, child, type, rid, start, end,
5227 count, flags & ~RF_ACTIVE);
5228 if (res == NULL) {
5229 resource_list_delete(rl, type, *rid);
5230 device_printf(child,
5231 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
5232 count, *rid, type, start, end);
5233 goto out;
5234 }
5235 if (bootverbose)
5236 device_printf(child,
5237 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
5238 count, *rid, type, rman_get_start(res));
5239
5240 /* Disable decoding via the CMD register before updating the BAR */
5241 cmd = pci_read_config(child, PCIR_COMMAND, 2);
5242 pci_write_config(child, PCIR_COMMAND,
5243 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
5244
5245 map = rman_get_start(res);
5246 pci_write_bar(child, pm, map);
5247
5248 /* Restore the original value of the CMD register */
5249 pci_write_config(child, PCIR_COMMAND, cmd, 2);
5250 out:
5251 return (res);
5252 }
5253
5254 struct resource *
5255 pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid,
5256 rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
5257 u_int flags)
5258 {
5259 struct pci_devinfo *dinfo;
5260 struct resource_list *rl;
5261 struct resource_list_entry *rle;
5262 struct resource *res;
5263 pcicfgregs *cfg;
5264
5265 /*
5266 * Perform lazy resource allocation
5267 */
5268 dinfo = device_get_ivars(child);
5269 rl = &dinfo->resources;
5270 cfg = &dinfo->cfg;
5271 switch (type) {
5272 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
5273 case PCI_RES_BUS:
5274 return (pci_alloc_secbus(dev, child, rid, start, end, count,
5275 flags));
5276 #endif
5277 case SYS_RES_IRQ:
5278 /*
5279 * Can't alloc legacy interrupt once MSI messages have
5280 * been allocated.
5281 */
5282 if (*rid == 0 && (cfg->msi.msi_alloc > 0 ||
5283 cfg->msix.msix_alloc > 0))
5284 return (NULL);
5285
5286 /*
5287 * If the child device doesn't have an interrupt
5288 * routed and is deserving of an interrupt, try to
5289 * assign it one.
5290 */
5291 if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
5292 (cfg->intpin != 0))
5293 pci_assign_interrupt(dev, child, 0);
5294 break;
5295 case SYS_RES_IOPORT:
5296 case SYS_RES_MEMORY:
5297 #ifdef NEW_PCIB
5298 /*
5299 * PCI-PCI bridge I/O window resources are not BARs.
5300 * For those allocations just pass the request up the
5301 * tree.
5302 */
5303 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
5304 switch (*rid) {
5305 case PCIR_IOBASEL_1:
5306 case PCIR_MEMBASE_1:
5307 case PCIR_PMBASEL_1:
5308 /*
5309 * XXX: Should we bother creating a resource
5310 * list entry?
5311 */
5312 return (bus_generic_alloc_resource(dev, child,
5313 type, rid, start, end, count, flags));
5314 }
5315 }
5316 #endif
5317 /* Reserve resources for this BAR if needed. */
5318 rle = resource_list_find(rl, type, *rid);
5319 if (rle == NULL) {
5320 res = pci_reserve_map(dev, child, type, rid, start, end,
5321 count, num, flags);
5322 if (res == NULL)
5323 return (NULL);
5324 }
5325 }
5326 return (resource_list_alloc(rl, dev, child, type, rid,
5327 start, end, count, flags));
5328 }
5329
5330 struct resource *
5331 pci_alloc_resource(device_t dev, device_t child, int type, int *rid,
5332 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
5333 {
5334 #ifdef PCI_IOV
5335 struct pci_devinfo *dinfo;
5336 #endif
5337
5338 if (device_get_parent(child) != dev)
5339 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
5340 type, rid, start, end, count, flags));
5341
5342 #ifdef PCI_IOV
5343 dinfo = device_get_ivars(child);
5344 if (dinfo->cfg.flags & PCICFG_VF) {
5345 switch (type) {
5346 /* VFs can't have I/O BARs. */
5347 case SYS_RES_IOPORT:
5348 return (NULL);
5349 case SYS_RES_MEMORY:
5350 return (pci_vf_alloc_mem_resource(dev, child, rid,
5351 start, end, count, flags));
5352 }
5353
5354 /* Fall through for other types of resource allocations. */
5355 }
5356 #endif
5357
5358 return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
5359 count, 1, flags));
5360 }
5361
5362 int
5363 pci_release_resource(device_t dev, device_t child, int type, int rid,
5364 struct resource *r)
5365 {
5366 struct pci_devinfo *dinfo;
5367 struct resource_list *rl;
5368 pcicfgregs *cfg;
5369
5370 if (device_get_parent(child) != dev)
5371 return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child,
5372 type, rid, r));
5373
5374 dinfo = device_get_ivars(child);
5375 cfg = &dinfo->cfg;
5376
5377 #ifdef PCI_IOV
5378 if (dinfo->cfg.flags & PCICFG_VF) {
5379 switch (type) {
5380 /* VFs can't have I/O BARs. */
5381 case SYS_RES_IOPORT:
5382 return (EDOOFUS);
5383 case SYS_RES_MEMORY:
5384 return (pci_vf_release_mem_resource(dev, child, rid,
5385 r));
5386 }
5387
5388 /* Fall through for other types of resource allocations. */
5389 }
5390 #endif
5391
5392 #ifdef NEW_PCIB
5393 /*
5394 * PCI-PCI bridge I/O window resources are not BARs. For
5395 * those allocations just pass the request up the tree.
5396 */
5397 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
5398 (type == SYS_RES_IOPORT || type == SYS_RES_MEMORY)) {
5399 switch (rid) {
5400 case PCIR_IOBASEL_1:
5401 case PCIR_MEMBASE_1:
5402 case PCIR_PMBASEL_1:
5403 return (bus_generic_release_resource(dev, child, type,
5404 rid, r));
5405 }
5406 }
5407 #endif
5408
5409 rl = &dinfo->resources;
5410 return (resource_list_release(rl, dev, child, type, rid, r));
5411 }
5412
5413 int
5414 pci_activate_resource(device_t dev, device_t child, int type, int rid,
5415 struct resource *r)
5416 {
5417 struct pci_devinfo *dinfo;
5418 int error;
5419
5420 error = bus_generic_activate_resource(dev, child, type, rid, r);
5421 if (error)
5422 return (error);
5423
5424 /* Enable decoding in the command register when activating BARs. */
5425 if (device_get_parent(child) == dev) {
5426 /* Device ROMs need their decoding explicitly enabled. */
5427 dinfo = device_get_ivars(child);
5428 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5429 pci_write_bar(child, pci_find_bar(child, rid),
5430 rman_get_start(r) | PCIM_BIOS_ENABLE);
5431 switch (type) {
5432 case SYS_RES_IOPORT:
5433 case SYS_RES_MEMORY:
5434 error = PCI_ENABLE_IO(dev, child, type);
5435 break;
5436 }
5437 }
5438 return (error);
5439 }
5440
5441 int
5442 pci_deactivate_resource(device_t dev, device_t child, int type,
5443 int rid, struct resource *r)
5444 {
5445 struct pci_devinfo *dinfo;
5446 int error;
5447
5448 error = bus_generic_deactivate_resource(dev, child, type, rid, r);
5449 if (error)
5450 return (error);
5451
5452 /* Disable decoding for device ROMs. */
5453 if (device_get_parent(child) == dev) {
5454 dinfo = device_get_ivars(child);
5455 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5456 pci_write_bar(child, pci_find_bar(child, rid),
5457 rman_get_start(r));
5458 }
5459 return (0);
5460 }
5461
5462 void
5463 pci_child_deleted(device_t dev, device_t child)
5464 {
5465 struct resource_list_entry *rle;
5466 struct resource_list *rl;
5467 struct pci_devinfo *dinfo;
5468
5469 dinfo = device_get_ivars(child);
5470 rl = &dinfo->resources;
5471
5472 EVENTHANDLER_INVOKE(pci_delete_device, child);
5473
5474 /* Turn off access to resources we're about to free */
5475 if (bus_child_present(child) != 0) {
5476 pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
5477 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
5478
5479 pci_disable_busmaster(child);
5480 }
5481
5482 /* Free all allocated resources */
5483 STAILQ_FOREACH(rle, rl, link) {
5484 if (rle->res) {
5485 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5486 resource_list_busy(rl, rle->type, rle->rid)) {
5487 pci_printf(&dinfo->cfg,
5488 "Resource still owned, oops. "
5489 "(type=%d, rid=%d, addr=%lx)\n",
5490 rle->type, rle->rid,
5491 rman_get_start(rle->res));
5492 bus_release_resource(child, rle->type, rle->rid,
5493 rle->res);
5494 }
5495 resource_list_unreserve(rl, dev, child, rle->type,
5496 rle->rid);
5497 }
5498 }
5499 resource_list_free(rl);
5500
5501 pci_freecfg(dinfo);
5502 }
5503
5504 void
5505 pci_delete_resource(device_t dev, device_t child, int type, int rid)
5506 {
5507 struct pci_devinfo *dinfo;
5508 struct resource_list *rl;
5509 struct resource_list_entry *rle;
5510
5511 if (device_get_parent(child) != dev)
5512 return;
5513
5514 dinfo = device_get_ivars(child);
5515 rl = &dinfo->resources;
5516 rle = resource_list_find(rl, type, rid);
5517 if (rle == NULL)
5518 return;
5519
5520 if (rle->res) {
5521 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5522 resource_list_busy(rl, type, rid)) {
5523 device_printf(dev, "delete_resource: "
5524 "Resource still owned by child, oops. "
5525 "(type=%d, rid=%d, addr=%jx)\n",
5526 type, rid, rman_get_start(rle->res));
5527 return;
5528 }
5529 resource_list_unreserve(rl, dev, child, type, rid);
5530 }
5531 resource_list_delete(rl, type, rid);
5532 }
5533
5534 struct resource_list *
5535 pci_get_resource_list (device_t dev, device_t child)
5536 {
5537 struct pci_devinfo *dinfo = device_get_ivars(child);
5538
5539 return (&dinfo->resources);
5540 }
5541
5542 bus_dma_tag_t
5543 pci_get_dma_tag(device_t bus, device_t dev)
5544 {
5545 struct pci_softc *sc = device_get_softc(bus);
5546
5547 return (sc->sc_dma_tag);
5548 }
5549
5550 uint32_t
5551 pci_read_config_method(device_t dev, device_t child, int reg, int width)
5552 {
5553 struct pci_devinfo *dinfo = device_get_ivars(child);
5554 pcicfgregs *cfg = &dinfo->cfg;
5555
5556 #ifdef PCI_IOV
5557 /*
5558 * SR-IOV VFs don't implement the VID or DID registers, so we have to
5559 * emulate them here.
5560 */
5561 if (cfg->flags & PCICFG_VF) {
5562 if (reg == PCIR_VENDOR) {
5563 switch (width) {
5564 case 4:
5565 return (cfg->device << 16 | cfg->vendor);
5566 case 2:
5567 return (cfg->vendor);
5568 case 1:
5569 return (cfg->vendor & 0xff);
5570 default:
5571 return (0xffffffff);
5572 }
5573 } else if (reg == PCIR_DEVICE) {
5574 switch (width) {
5575 /* Note that an unaligned 4-byte read is an error. */
5576 case 2:
5577 return (cfg->device);
5578 case 1:
5579 return (cfg->device & 0xff);
5580 default:
5581 return (0xffffffff);
5582 }
5583 }
5584 }
5585 #endif
5586
5587 return (PCIB_READ_CONFIG(device_get_parent(dev),
5588 cfg->bus, cfg->slot, cfg->func, reg, width));
5589 }
5590
5591 void
5592 pci_write_config_method(device_t dev, device_t child, int reg,
5593 uint32_t val, int width)
5594 {
5595 struct pci_devinfo *dinfo = device_get_ivars(child);
5596 pcicfgregs *cfg = &dinfo->cfg;
5597
5598 PCIB_WRITE_CONFIG(device_get_parent(dev),
5599 cfg->bus, cfg->slot, cfg->func, reg, val, width);
5600 }
5601
5602 int
5603 pci_child_location_str_method(device_t dev, device_t child, char *buf,
5604 size_t buflen)
5605 {
5606
5607 snprintf(buf, buflen, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
5608 pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
5609 pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
5610 return (0);
5611 }
5612
5613 int
5614 pci_child_pnpinfo_str_method(device_t dev, device_t child, char *buf,
5615 size_t buflen)
5616 {
5617 struct pci_devinfo *dinfo;
5618 pcicfgregs *cfg;
5619
5620 dinfo = device_get_ivars(child);
5621 cfg = &dinfo->cfg;
5622 snprintf(buf, buflen, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
5623 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
5624 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
5625 cfg->progif);
5626 return (0);
5627 }
5628
5629 int
5630 pci_assign_interrupt_method(device_t dev, device_t child)
5631 {
5632 struct pci_devinfo *dinfo = device_get_ivars(child);
5633 pcicfgregs *cfg = &dinfo->cfg;
5634
5635 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
5636 cfg->intpin));
5637 }
5638
5639 static void
5640 pci_lookup(void *arg, const char *name, device_t *dev)
5641 {
5642 long val;
5643 char *end;
5644 int domain, bus, slot, func;
5645
5646 if (*dev != NULL)
5647 return;
5648
5649 /*
5650 * Accept pciconf-style selectors of either pciD:B:S:F or
5651 * pciB:S:F. In the latter case, the domain is assumed to
5652 * be zero.
5653 */
5654 if (strncmp(name, "pci", 3) != 0)
5655 return;
5656 val = strtol(name + 3, &end, 10);
5657 if (val < 0 || val > INT_MAX || *end != ':')
5658 return;
5659 domain = val;
5660 val = strtol(end + 1, &end, 10);
5661 if (val < 0 || val > INT_MAX || *end != ':')
5662 return;
5663 bus = val;
5664 val = strtol(end + 1, &end, 10);
5665 if (val < 0 || val > INT_MAX)
5666 return;
5667 slot = val;
5668 if (*end == ':') {
5669 val = strtol(end + 1, &end, 10);
5670 if (val < 0 || val > INT_MAX || *end != '\0')
5671 return;
5672 func = val;
5673 } else if (*end == '\0') {
5674 func = slot;
5675 slot = bus;
5676 bus = domain;
5677 domain = 0;
5678 } else
5679 return;
5680
5681 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX ||
5682 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX))
5683 return;
5684
5685 *dev = pci_find_dbsf(domain, bus, slot, func);
5686 }
5687
5688 static int
5689 pci_modevent(module_t mod, int what, void *arg)
5690 {
5691 static struct cdev *pci_cdev;
5692 static eventhandler_tag tag;
5693
5694 switch (what) {
5695 case MOD_LOAD:
5696 STAILQ_INIT(&pci_devq);
5697 pci_generation = 0;
5698 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
5699 "pci");
5700 pci_load_vendor_data();
5701 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
5702 1000);
5703 break;
5704
5705 case MOD_UNLOAD:
5706 if (tag != NULL)
5707 EVENTHANDLER_DEREGISTER(dev_lookup, tag);
5708 destroy_dev(pci_cdev);
5709 break;
5710 }
5711
5712 return (0);
5713 }
5714
5715 static void
5716 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
5717 {
5718 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
5719 struct pcicfg_pcie *cfg;
5720 int version, pos;
5721
5722 cfg = &dinfo->cfg.pcie;
5723 pos = cfg->pcie_location;
5724
5725 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
5726
5727 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);
5728
5729 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
5730 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
5731 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
5732 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);
5733
5734 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
5735 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
5736 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
5737 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);
5738
5739 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
5740 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
5741 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);
5742
5743 if (version > 1) {
5744 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
5745 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
5746 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
5747 }
5748 #undef WREG
5749 }
5750
5751 static void
5752 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
5753 {
5754 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
5755 dinfo->cfg.pcix.pcix_command, 2);
5756 }
5757
5758 void
5759 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
5760 {
5761
5762 /*
5763 * Restore the device to full power mode. We must do this
5764 * before we restore the registers because moving from D3 to
5765 * D0 will cause the chip's BARs and some other registers to
5766 * be reset to some unknown power on reset values. Cut down
5767 * the noise on boot by doing nothing if we are already in
5768 * state D0.
5769 */
5770 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
5771 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
5772 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);
5773 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
5774 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
5775 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
5776 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
5777 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
5778 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
5779 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
5780 case PCIM_HDRTYPE_NORMAL:
5781 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
5782 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
5783 break;
5784 case PCIM_HDRTYPE_BRIDGE:
5785 pci_write_config(dev, PCIR_SECLAT_1,
5786 dinfo->cfg.bridge.br_seclat, 1);
5787 pci_write_config(dev, PCIR_SUBBUS_1,
5788 dinfo->cfg.bridge.br_subbus, 1);
5789 pci_write_config(dev, PCIR_SECBUS_1,
5790 dinfo->cfg.bridge.br_secbus, 1);
5791 pci_write_config(dev, PCIR_PRIBUS_1,
5792 dinfo->cfg.bridge.br_pribus, 1);
5793 pci_write_config(dev, PCIR_BRIDGECTL_1,
5794 dinfo->cfg.bridge.br_control, 2);
5795 break;
5796 case PCIM_HDRTYPE_CARDBUS:
5797 pci_write_config(dev, PCIR_SECLAT_2,
5798 dinfo->cfg.bridge.br_seclat, 1);
5799 pci_write_config(dev, PCIR_SUBBUS_2,
5800 dinfo->cfg.bridge.br_subbus, 1);
5801 pci_write_config(dev, PCIR_SECBUS_2,
5802 dinfo->cfg.bridge.br_secbus, 1);
5803 pci_write_config(dev, PCIR_PRIBUS_2,
5804 dinfo->cfg.bridge.br_pribus, 1);
5805 pci_write_config(dev, PCIR_BRIDGECTL_2,
5806 dinfo->cfg.bridge.br_control, 2);
5807 break;
5808 }
5809 pci_restore_bars(dev);
5810
5811 /*
5812 * Restore extended capabilities for PCI-Express and PCI-X
5813 */
5814 if (dinfo->cfg.pcie.pcie_location != 0)
5815 pci_cfg_restore_pcie(dev, dinfo);
5816 if (dinfo->cfg.pcix.pcix_location != 0)
5817 pci_cfg_restore_pcix(dev, dinfo);
5818
5819 /* Restore MSI and MSI-X configurations if they are present. */
5820 if (dinfo->cfg.msi.msi_location != 0)
5821 pci_resume_msi(dev);
5822 if (dinfo->cfg.msix.msix_location != 0)
5823 pci_resume_msix(dev);
5824
5825 #ifdef PCI_IOV
5826 if (dinfo->cfg.iov != NULL)
5827 pci_iov_cfg_restore(dev, dinfo);
5828 #endif
5829 }
5830
5831 static void
5832 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
5833 {
5834 #define RREG(n) pci_read_config(dev, pos + (n), 2)
5835 struct pcicfg_pcie *cfg;
5836 int version, pos;
5837
5838 cfg = &dinfo->cfg.pcie;
5839 pos = cfg->pcie_location;
5840
5841 cfg->pcie_flags = RREG(PCIER_FLAGS);
5842
5843 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
5844
5845 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);
5846
5847 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
5848 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
5849 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
5850 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);
5851
5852 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
5853 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
5854 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
5855 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);
5856
5857 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
5858 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
5859 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);
5860
5861 if (version > 1) {
5862 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
5863 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
5864 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
5865 }
5866 #undef RREG
5867 }
5868
5869 static void
5870 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
5871 {
5872 dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
5873 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
5874 }
5875
5876 void
5877 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
5878 {
5879 uint32_t cls;
5880 int ps;
5881
5882 /*
5883 * Some drivers apparently write to these registers w/o updating our
5884 * cached copy. No harm happens if we update the copy, so do so here
5885 * so we can restore them. The COMMAND register is modified by the
5886 * bus w/o updating the cache. This should represent the normally
5887 * writable portion of the 'defined' part of type 0/1/2 headers.
5888 */
5889 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
5890 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
5891 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
5892 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
5893 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
5894 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
5895 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
5896 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
5897 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
5898 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
5899 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
5900 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
5901 case PCIM_HDRTYPE_NORMAL:
5902 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
5903 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
5904 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
5905 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
5906 break;
5907 case PCIM_HDRTYPE_BRIDGE:
5908 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
5909 PCIR_SECLAT_1, 1);
5910 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
5911 PCIR_SUBBUS_1, 1);
5912 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
5913 PCIR_SECBUS_1, 1);
5914 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
5915 PCIR_PRIBUS_1, 1);
5916 dinfo->cfg.bridge.br_control = pci_read_config(dev,
5917 PCIR_BRIDGECTL_1, 2);
5918 break;
5919 case PCIM_HDRTYPE_CARDBUS:
5920 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
5921 PCIR_SECLAT_2, 1);
5922 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
5923 PCIR_SUBBUS_2, 1);
5924 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
5925 PCIR_SECBUS_2, 1);
5926 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
5927 PCIR_PRIBUS_2, 1);
5928 dinfo->cfg.bridge.br_control = pci_read_config(dev,
5929 PCIR_BRIDGECTL_2, 2);
5930 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
5931 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
5932 break;
5933 }
5934
5935 if (dinfo->cfg.pcie.pcie_location != 0)
5936 pci_cfg_save_pcie(dev, dinfo);
5937
5938 if (dinfo->cfg.pcix.pcix_location != 0)
5939 pci_cfg_save_pcix(dev, dinfo);
5940
5941 #ifdef PCI_IOV
5942 if (dinfo->cfg.iov != NULL)
5943 pci_iov_cfg_save(dev, dinfo);
5944 #endif
5945
5946 /*
5947 * don't set the state for display devices, base peripherals and
5948 * memory devices since bad things happen when they are powered down.
5949 * We should (a) have drivers that can easily detach and (b) use
5950 * generic drivers for these devices so that some device actually
5951 * attaches. We need to make sure that when we implement (a) we don't
5952 * power the device down on a reattach.
5953 */
5954 cls = pci_get_class(dev);
5955 if (!setstate)
5956 return;
5957 switch (pci_do_power_nodriver)
5958 {
5959 case 0: /* NO powerdown at all */
5960 return;
5961 case 1: /* Conservative about what to power down */
5962 if (cls == PCIC_STORAGE)
5963 return;
5964 /*FALLTHROUGH*/
5965 case 2: /* Aggressive about what to power down */
5966 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY ||
5967 cls == PCIC_BASEPERIPH)
5968 return;
5969 /*FALLTHROUGH*/
5970 case 3: /* Power down everything */
5971 break;
5972 }
5973 /*
5974 * PCI spec says we can only go into D3 state from D0 state.
5975 * Transition from D[12] into D0 before going to D3 state.
5976 */
5977 ps = pci_get_powerstate(dev);
5978 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
5979 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
5980 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
5981 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
5982 }
5983
5984 /* Wrapper APIs suitable for device driver use. */
5985 void
5986 pci_save_state(device_t dev)
5987 {
5988 struct pci_devinfo *dinfo;
5989
5990 dinfo = device_get_ivars(dev);
5991 pci_cfg_save(dev, dinfo, 0);
5992 }
5993
5994 void
5995 pci_restore_state(device_t dev)
5996 {
5997 struct pci_devinfo *dinfo;
5998
5999 dinfo = device_get_ivars(dev);
6000 pci_cfg_restore(dev, dinfo);
6001 }
6002
6003 static int
6004 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
6005 uintptr_t *id)
6006 {
6007
6008 return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
6009 }
6010
6011 /* Find the upstream port of a given PCI device in a root complex. */
6012 device_t
6013 pci_find_pcie_root_port(device_t dev)
6014 {
6015 struct pci_devinfo *dinfo;
6016 devclass_t pci_class;
6017 device_t pcib, bus;
6018
6019 pci_class = devclass_find("pci");
6020 KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class,
6021 ("%s: non-pci device %s", __func__, device_get_nameunit(dev)));
6022
6023 /*
6024 * Walk the bridge hierarchy until we find a PCI-e root
6025 * port or a non-PCI device.
6026 */
6027 for (;;) {
6028 bus = device_get_parent(dev);
6029 KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
6030 device_get_nameunit(dev)));
6031
6032 pcib = device_get_parent(bus);
6033 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
6034 device_get_nameunit(bus)));
6035
6036 /*
6037 * pcib's parent must be a PCI bus for this to be a
6038 * PCI-PCI bridge.
6039 */
6040 if (device_get_devclass(device_get_parent(pcib)) != pci_class)
6041 return (NULL);
6042
6043 dinfo = device_get_ivars(pcib);
6044 if (dinfo->cfg.pcie.pcie_location != 0 &&
6045 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
6046 return (pcib);
6047
6048 dev = pcib;
6049 }
6050 }
6051
6052 /*
6053 * Wait for pending transactions to complete on a PCI-express function.
6054 *
6055 * The maximum delay is specified in milliseconds in max_delay. Note
6056 * that this function may sleep.
6057 *
6058 * Returns true if the function is idle and false if the timeout is
6059 * exceeded. If dev is not a PCI-express function, this returns true.
6060 */
6061 bool
6062 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
6063 {
6064 struct pci_devinfo *dinfo = device_get_ivars(dev);
6065 uint16_t sta;
6066 int cap;
6067
6068 cap = dinfo->cfg.pcie.pcie_location;
6069 if (cap == 0)
6070 return (true);
6071
6072 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6073 while (sta & PCIEM_STA_TRANSACTION_PND) {
6074 if (max_delay == 0)
6075 return (false);
6076
6077 /* Poll once every 100 milliseconds up to the timeout. */
6078 if (max_delay > 100) {
6079 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
6080 max_delay -= 100;
6081 } else {
6082 pause_sbt("pcietp", max_delay * SBT_1MS, 0,
6083 C_HARDCLOCK);
6084 max_delay = 0;
6085 }
6086 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6087 }
6088
6089 return (true);
6090 }
6091
6092 /*
6093 * Determine the maximum Completion Timeout in microseconds.
6094 *
6095 * For non-PCI-express functions this returns 0.
6096 */
6097 int
6098 pcie_get_max_completion_timeout(device_t dev)
6099 {
6100 struct pci_devinfo *dinfo = device_get_ivars(dev);
6101 int cap;
6102
6103 cap = dinfo->cfg.pcie.pcie_location;
6104 if (cap == 0)
6105 return (0);
6106
6107 /*
6108 * Functions using the 1.x spec use the default timeout range of
6109 * 50 microseconds to 50 milliseconds. Functions that do not
6110 * support programmable timeouts also use this range.
6111 */
6112 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 ||
6113 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
6114 PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
6115 return (50 * 1000);
6116
6117 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
6118 PCIEM_CTL2_COMP_TIMO_VAL) {
6119 case PCIEM_CTL2_COMP_TIMO_100US:
6120 return (100);
6121 case PCIEM_CTL2_COMP_TIMO_10MS:
6122 return (10 * 1000);
6123 case PCIEM_CTL2_COMP_TIMO_55MS:
6124 return (55 * 1000);
6125 case PCIEM_CTL2_COMP_TIMO_210MS:
6126 return (210 * 1000);
6127 case PCIEM_CTL2_COMP_TIMO_900MS:
6128 return (900 * 1000);
6129 case PCIEM_CTL2_COMP_TIMO_3500MS:
6130 return (3500 * 1000);
6131 case PCIEM_CTL2_COMP_TIMO_13S:
6132 return (13 * 1000 * 1000);
6133 case PCIEM_CTL2_COMP_TIMO_64S:
6134 return (64 * 1000 * 1000);
6135 default:
6136 return (50 * 1000);
6137 }
6138 }
6139
6140 /*
6141 * Perform a Function Level Reset (FLR) on a device.
6142 *
6143 * This function first waits for any pending transactions to complete
6144 * within the timeout specified by max_delay. If transactions are
6145 * still pending, the function will return false without attempting a
6146 * reset.
6147 *
6148 * If dev is not a PCI-express function or does not support FLR, this
6149 * function returns false.
6150 *
6151 * Note that no registers are saved or restored. The caller is
6152 * responsible for saving and restoring any registers including
6153 * PCI-standard registers via pci_save_state() and
6154 * pci_restore_state().
6155 */
6156 bool
6157 pcie_flr(device_t dev, u_int max_delay, bool force)
6158 {
6159 struct pci_devinfo *dinfo = device_get_ivars(dev);
6160 uint16_t cmd, ctl;
6161 int compl_delay;
6162 int cap;
6163
6164 cap = dinfo->cfg.pcie.pcie_location;
6165 if (cap == 0)
6166 return (false);
6167
6168 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR))
6169 return (false);
6170
6171 /*
6172 * Disable busmastering to prevent generation of new
6173 * transactions while waiting for the device to go idle. If
6174 * the idle timeout fails, the command register is restored
6175 * which will re-enable busmastering.
6176 */
6177 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
6178 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
6179 if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
6180 if (!force) {
6181 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
6182 return (false);
6183 }
6184 pci_printf(&dinfo->cfg,
6185 "Resetting with transactions pending after %d ms\n",
6186 max_delay);
6187
6188 /*
6189 * Extend the post-FLR delay to cover the maximum
6190 * Completion Timeout delay of anything in flight
6191 * during the FLR delay. Enforce a minimum delay of
6192 * at least 10ms.
6193 */
6194 compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
6195 if (compl_delay < 10)
6196 compl_delay = 10;
6197 } else
6198 compl_delay = 0;
6199
6200 /* Initiate the reset. */
6201 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
6202 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl |
6203 PCIEM_CTL_INITIATE_FLR, 2);
6204
6205 /* Wait for 100ms. */
6206 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);
6207
6208 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
6209 PCIEM_STA_TRANSACTION_PND)
6210 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
6211 return (true);
6212 }
Cache object: 3eef79f4289c913e8a9bcd8cdcc464b6
|