FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/msi.c
1 /*-
2 * Copyright (c) 2006 Yahoo!, Inc.
3 * All rights reserved.
4 * Written by: John Baldwin <jhb@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the author nor the names of any co-contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /*
32 * Support for PCI Message Signalled Interrupts (MSI). MSI interrupts on
33 * x86 are basically APIC messages that the northbridge delivers directly
34 * to the local APICs as if they had come from an I/O APIC.
35 */
36
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39
40 #include <sys/param.h>
41 #include <sys/bus.h>
42 #include <sys/kernel.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mutex.h>
46 #include <sys/sx.h>
47 #include <sys/systm.h>
48 #include <machine/apicreg.h>
49 #include <machine/md_var.h>
50 #include <machine/frame.h>
51 #include <machine/intr_machdep.h>
52 #include <machine/apicvar.h>
53 #include <dev/pci/pcivar.h>
54
55 /* Fields in address for Intel MSI messages. */
56 #define MSI_INTEL_ADDR_DEST 0x000ff000
57 #define MSI_INTEL_ADDR_RH 0x00000008
58 # define MSI_INTEL_ADDR_RH_ON 0x00000008
59 # define MSI_INTEL_ADDR_RH_OFF 0x00000000
60 #define MSI_INTEL_ADDR_DM 0x00000004
61 # define MSI_INTEL_ADDR_DM_PHYSICAL 0x00000000
62 # define MSI_INTEL_ADDR_DM_LOGICAL 0x00000004
63
64 /* Fields in data for Intel MSI messages. */
65 #define MSI_INTEL_DATA_TRGRMOD IOART_TRGRMOD /* Trigger mode. */
66 # define MSI_INTEL_DATA_TRGREDG IOART_TRGREDG
67 # define MSI_INTEL_DATA_TRGRLVL IOART_TRGRLVL
68 #define MSI_INTEL_DATA_LEVEL 0x00004000 /* Polarity. */
69 # define MSI_INTEL_DATA_DEASSERT 0x00000000
70 # define MSI_INTEL_DATA_ASSERT 0x00004000
71 #define MSI_INTEL_DATA_DELMOD IOART_DELMOD /* Delivery mode. */
72 # define MSI_INTEL_DATA_DELFIXED IOART_DELFIXED
73 # define MSI_INTEL_DATA_DELLOPRI IOART_DELLOPRI
74 # define MSI_INTEL_DATA_DELSMI IOART_DELSMI
75 # define MSI_INTEL_DATA_DELNMI IOART_DELNMI
76 # define MSI_INTEL_DATA_DELINIT IOART_DELINIT
77 # define MSI_INTEL_DATA_DELEXINT IOART_DELEXINT
78 #define MSI_INTEL_DATA_INTVEC IOART_INTVEC /* Interrupt vector. */
79
80 /*
81 * Build Intel MSI message and data values from a source. AMD64 systems
82 * seem to be compatible, so we use the same function for both.
83 */
84 #define INTEL_ADDR(msi) \
85 (MSI_INTEL_ADDR_BASE | (msi)->msi_cpu << 12 | \
86 MSI_INTEL_ADDR_RH_OFF | MSI_INTEL_ADDR_DM_PHYSICAL)
87 #define INTEL_DATA(msi) \
88 (MSI_INTEL_DATA_TRGREDG | MSI_INTEL_DATA_DELFIXED | (msi)->msi_vector)
89
90 static MALLOC_DEFINE(M_MSI, "msi", "PCI MSI");
91
92 /*
93 * MSI sources are bunched into groups. This is because MSI forces
94 * all of the messages to share the address and data registers and
95 * thus certain properties (such as the local APIC ID target on x86).
96 * Each group has a 'first' source that contains information global to
97 * the group. These fields are marked with (g) below.
98 *
99 * Note that local APIC ID is kind of special. Each message will be
100 * assigned an ID by the system; however, a group will use the ID from
101 * the first message.
102 *
103 * For MSI-X, each message is isolated.
104 */
105 struct msi_intsrc {
106 struct intsrc msi_intsrc;
107 device_t msi_dev; /* Owning device. (g) */
108 struct msi_intsrc *msi_first; /* First source in group. */
109 u_int msi_irq; /* IRQ cookie. */
110 u_int msi_msix; /* MSI-X message. */
111 u_int msi_vector:8; /* IDT vector. */
112 u_int msi_cpu:8; /* Local APIC ID. (g) */
113 u_int msi_count:8; /* Messages in this group. (g) */
114 };
115
116 static void msi_create_source(void);
117 static void msi_enable_source(struct intsrc *isrc);
118 static void msi_disable_source(struct intsrc *isrc, int eoi);
119 static void msi_eoi_source(struct intsrc *isrc);
120 static void msi_enable_intr(struct intsrc *isrc);
121 static int msi_vector(struct intsrc *isrc);
122 static int msi_source_pending(struct intsrc *isrc);
123 static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
124 enum intr_polarity pol);
125 static void msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
126
127 struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
128 msi_enable_intr, msi_vector, msi_source_pending,
129 NULL, NULL, msi_config_intr, msi_assign_cpu };
130
131 static int msi_enabled;
132 static int msi_last_irq;
133 static struct mtx msi_lock;
134
135 static void
136 msi_enable_source(struct intsrc *isrc)
137 {
138 }
139
140 static void
141 msi_disable_source(struct intsrc *isrc, int eoi)
142 {
143
144 if (eoi == PIC_EOI)
145 lapic_eoi();
146 }
147
148 static void
149 msi_eoi_source(struct intsrc *isrc)
150 {
151
152 lapic_eoi();
153 }
154
155 static void
156 msi_enable_intr(struct intsrc *isrc)
157 {
158 struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
159
160 apic_enable_vector(msi->msi_vector);
161 }
162
163 static int
164 msi_vector(struct intsrc *isrc)
165 {
166 struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
167
168 return (msi->msi_irq);
169 }
170
171 static int
172 msi_source_pending(struct intsrc *isrc)
173 {
174
175 return (0);
176 }
177
178 static int
179 msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
180 enum intr_polarity pol)
181 {
182
183 return (ENODEV);
184 }
185
186 static void
187 msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
188 {
189 struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
190
191 msi->msi_cpu = apic_id;
192 if (bootverbose)
193 printf("msi: Assigning %s IRQ %d to local APIC %u\n",
194 msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
195 msi->msi_cpu);
196 if (isrc->is_enabled)
197 pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
198 }
199
200 void
201 msi_init(void)
202 {
203
204 /* Check if we have a supported CPU. */
205 if (!(strcmp(cpu_vendor, "GenuineIntel") == 0 ||
206 strcmp(cpu_vendor, "AuthenticAMD") == 0))
207 return;
208
209 msi_enabled = 1;
210 intr_register_pic(&msi_pic);
211 mtx_init(&msi_lock, "msi", NULL, MTX_DEF);
212 }
213
214 void
215 msi_create_source(void)
216 {
217 struct msi_intsrc *msi;
218 u_int irq;
219
220 mtx_lock(&msi_lock);
221 if (msi_last_irq >= NUM_MSI_INTS) {
222 mtx_unlock(&msi_lock);
223 return;
224 }
225 irq = msi_last_irq + FIRST_MSI_INT;
226 msi_last_irq++;
227 mtx_unlock(&msi_lock);
228
229 msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
230 msi->msi_intsrc.is_pic = &msi_pic;
231 msi->msi_irq = irq;
232 intr_register_source(&msi->msi_intsrc);
233 nexus_add_irq(irq);
234 }
235
236 /*
237 * Try to allocate 'count' interrupt sources with contiguous IDT values. If
238 * we allocate any new sources, then their IRQ values will be at the end of
239 * the irqs[] array, with *newirq being the index of the first new IRQ value
240 * and *newcount being the number of new IRQ values added.
241 */
242 int
243 msi_alloc(device_t dev, int count, int maxcount, int *irqs)
244 {
245 struct msi_intsrc *msi, *fsrc;
246 int cnt, i, vector;
247
248 if (!msi_enabled)
249 return (ENXIO);
250
251 again:
252 mtx_lock(&msi_lock);
253
254 /* Try to find 'count' free IRQs. */
255 cnt = 0;
256 for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
257 msi = (struct msi_intsrc *)intr_lookup_source(i);
258
259 /* End of allocated sources, so break. */
260 if (msi == NULL)
261 break;
262
263 /* If this is a free one, save its IRQ in the array. */
264 if (msi->msi_dev == NULL) {
265 irqs[cnt] = i;
266 cnt++;
267 if (cnt == count)
268 break;
269 }
270 }
271
272 /* Do we need to create some new sources? */
273 if (cnt < count) {
274 /* If we would exceed the max, give up. */
275 if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
276 mtx_unlock(&msi_lock);
277 return (ENXIO);
278 }
279 mtx_unlock(&msi_lock);
280
281 /* We need count - cnt more sources. */
282 while (cnt < count) {
283 msi_create_source();
284 cnt++;
285 }
286 goto again;
287 }
288
289 /* Ok, we now have the IRQs allocated. */
290 KASSERT(cnt == count, ("count mismatch"));
291
292 /* Allocate 'count' IDT vectors. */
293 vector = apic_alloc_vectors(irqs, count, maxcount);
294 if (vector == 0) {
295 mtx_unlock(&msi_lock);
296 return (ENOSPC);
297 }
298
299 /* Assign IDT vectors and make these messages owned by 'dev'. */
300 fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
301 for (i = 0; i < count; i++) {
302 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
303 msi->msi_dev = dev;
304 msi->msi_cpu = PCPU_GET(apic_id);
305 msi->msi_vector = vector + i;
306 if (bootverbose)
307 printf("msi: routing MSI IRQ %d to vector %u\n",
308 msi->msi_irq, msi->msi_vector);
309 msi->msi_first = fsrc;
310
311 /* XXX: Somewhat gross. */
312 msi->msi_intsrc.is_enabled = 0;
313 }
314 fsrc->msi_count = count;
315 mtx_unlock(&msi_lock);
316
317 return (0);
318 }
319
320 int
321 msi_release(int *irqs, int count)
322 {
323 struct msi_intsrc *msi, *first;
324 int i;
325
326 mtx_lock(&msi_lock);
327 first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
328 if (first == NULL) {
329 mtx_unlock(&msi_lock);
330 return (ENOENT);
331 }
332
333 /* Make sure this isn't an MSI-X message. */
334 if (first->msi_msix) {
335 mtx_unlock(&msi_lock);
336 return (EINVAL);
337 }
338
339 /* Make sure this message is allocated to a group. */
340 if (first->msi_first == NULL) {
341 mtx_unlock(&msi_lock);
342 return (ENXIO);
343 }
344
345 /*
346 * Make sure this is the start of a group and that we are releasing
347 * the entire group.
348 */
349 if (first->msi_first != first || first->msi_count != count) {
350 mtx_unlock(&msi_lock);
351 return (EINVAL);
352 }
353 KASSERT(first->msi_dev != NULL, ("unowned group"));
354
355 /* Clear all the extra messages in the group. */
356 for (i = 1; i < count; i++) {
357 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
358 KASSERT(msi->msi_first == first, ("message not in group"));
359 KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
360 msi->msi_first = NULL;
361 msi->msi_dev = NULL;
362 apic_free_vector(msi->msi_vector, msi->msi_irq);
363 msi->msi_vector = 0;
364 }
365
366 /* Clear out the first message. */
367 first->msi_first = NULL;
368 first->msi_dev = NULL;
369 apic_free_vector(first->msi_vector, first->msi_irq);
370 first->msi_vector = 0;
371 first->msi_count = 0;
372
373 mtx_unlock(&msi_lock);
374 return (0);
375 }
376
377 int
378 msi_map(int irq, uint64_t *addr, uint32_t *data)
379 {
380 struct msi_intsrc *msi;
381
382 mtx_lock(&msi_lock);
383 msi = (struct msi_intsrc *)intr_lookup_source(irq);
384 if (msi == NULL) {
385 mtx_unlock(&msi_lock);
386 return (ENOENT);
387 }
388
389 /* Make sure this message is allocated to a device. */
390 if (msi->msi_dev == NULL) {
391 mtx_unlock(&msi_lock);
392 return (ENXIO);
393 }
394
395 /*
396 * If this message isn't an MSI-X message, make sure it's part
397 * of a group, and switch to the first message in the
398 * group.
399 */
400 if (!msi->msi_msix) {
401 if (msi->msi_first == NULL) {
402 mtx_unlock(&msi_lock);
403 return (ENXIO);
404 }
405 msi = msi->msi_first;
406 }
407
408 *addr = INTEL_ADDR(msi);
409 *data = INTEL_DATA(msi);
410 mtx_unlock(&msi_lock);
411 return (0);
412 }
413
414 int
415 msix_alloc(device_t dev, int *irq)
416 {
417 struct msi_intsrc *msi;
418 int i, vector;
419
420 if (!msi_enabled)
421 return (ENXIO);
422
423 again:
424 mtx_lock(&msi_lock);
425
426 /* Find a free IRQ. */
427 for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
428 msi = (struct msi_intsrc *)intr_lookup_source(i);
429
430 /* End of allocated sources, so break. */
431 if (msi == NULL)
432 break;
433
434 /* Stop at the first free source. */
435 if (msi->msi_dev == NULL)
436 break;
437 }
438
439 /* Do we need to create a new source? */
440 if (msi == NULL) {
441 /* If we would exceed the max, give up. */
442 if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
443 mtx_unlock(&msi_lock);
444 return (ENXIO);
445 }
446 mtx_unlock(&msi_lock);
447
448 /* Create a new source. */
449 msi_create_source();
450 goto again;
451 }
452
453 /* Allocate an IDT vector. */
454 vector = apic_alloc_vector(i);
455 if (bootverbose)
456 printf("msi: routing MSI-X IRQ %d to vector %u\n", msi->msi_irq,
457 vector);
458
459 /* Setup source. */
460 msi->msi_dev = dev;
461 msi->msi_vector = vector;
462 msi->msi_cpu = PCPU_GET(apic_id);
463 msi->msi_msix = 1;
464
465 /* XXX: Somewhat gross. */
466 msi->msi_intsrc.is_enabled = 0;
467 mtx_unlock(&msi_lock);
468
469 *irq = i;
470 return (0);
471 }
472
473 int
474 msix_release(int irq)
475 {
476 struct msi_intsrc *msi;
477
478 mtx_lock(&msi_lock);
479 msi = (struct msi_intsrc *)intr_lookup_source(irq);
480 if (msi == NULL) {
481 mtx_unlock(&msi_lock);
482 return (ENOENT);
483 }
484
485 /* Make sure this is an MSI-X message. */
486 if (!msi->msi_msix) {
487 mtx_unlock(&msi_lock);
488 return (EINVAL);
489 }
490
491 KASSERT(msi->msi_dev != NULL, ("unowned message"));
492
493 /* Clear out the message. */
494 msi->msi_dev = NULL;
495 apic_free_vector(msi->msi_vector, msi->msi_irq);
496 msi->msi_vector = 0;
497 msi->msi_msix = 0;
498
499 mtx_unlock(&msi_lock);
500 return (0);
501 }
Cache object: 2dfdbbbba969f8f5edfe0ff2e21751bd
|