FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_intr.c
1 /*-
2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29
30 #include "opt_ddb.h"
31
32 #include <sys/param.h>
33 #include <sys/bus.h>
34 #include <sys/conf.h>
35 #include <sys/cpuset.h>
36 #include <sys/rtprio.h>
37 #include <sys/systm.h>
38 #include <sys/interrupt.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/ktr.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mutex.h>
46 #include <sys/proc.h>
47 #include <sys/random.h>
48 #include <sys/resourcevar.h>
49 #include <sys/sched.h>
50 #include <sys/smp.h>
51 #include <sys/sysctl.h>
52 #include <sys/unistd.h>
53 #include <sys/vmmeter.h>
54 #include <machine/atomic.h>
55 #include <machine/cpu.h>
56 #include <machine/md_var.h>
57 #include <machine/stdarg.h>
58 #ifdef DDB
59 #include <ddb/ddb.h>
60 #include <ddb/db_sym.h>
61 #endif
62
63 /*
64 * Describe an interrupt thread. There is one of these per interrupt event.
65 */
66 struct intr_thread {
67 struct intr_event *it_event;
68 struct thread *it_thread; /* Kernel thread. */
69 int it_flags; /* (j) IT_* flags. */
70 int it_need; /* Needs service. */
71 };
72
73 /* Interrupt thread flags kept in it_flags */
74 #define IT_DEAD 0x000001 /* Thread is waiting to exit. */
75
76 struct intr_entropy {
77 struct thread *td;
78 uintptr_t event;
79 };
80
81 struct intr_event *clk_intr_event;
82 struct intr_event *tty_intr_event;
83 void *softclock_ih;
84 void *vm_ih;
85
86 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads");
87
88 static int intr_storm_threshold = 1000;
89 TUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold);
90 SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW,
91 &intr_storm_threshold, 0,
92 "Number of consecutive interrupts before storm protection is enabled");
93 static TAILQ_HEAD(, intr_event) event_list =
94 TAILQ_HEAD_INITIALIZER(event_list);
95 static struct mtx event_lock;
96 MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF);
97
98 static void intr_event_update(struct intr_event *ie);
99 #ifdef INTR_FILTER
100 static int intr_event_schedule_thread(struct intr_event *ie,
101 struct intr_thread *ithd);
102 static int intr_filter_loop(struct intr_event *ie,
103 struct trapframe *frame, struct intr_thread **ithd);
104 static struct intr_thread *ithread_create(const char *name,
105 struct intr_handler *ih);
106 #else
107 static int intr_event_schedule_thread(struct intr_event *ie);
108 static struct intr_thread *ithread_create(const char *name);
109 #endif
110 static void ithread_destroy(struct intr_thread *ithread);
111 static void ithread_execute_handlers(struct proc *p,
112 struct intr_event *ie);
113 #ifdef INTR_FILTER
114 static void priv_ithread_execute_handler(struct proc *p,
115 struct intr_handler *ih);
116 #endif
117 static void ithread_loop(void *);
118 static void ithread_update(struct intr_thread *ithd);
119 static void start_softintr(void *);
120
121 /* Map an interrupt type to an ithread priority. */
122 u_char
123 intr_priority(enum intr_type flags)
124 {
125 u_char pri;
126
127 flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET |
128 INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV);
129 switch (flags) {
130 case INTR_TYPE_TTY:
131 pri = PI_TTY;
132 break;
133 case INTR_TYPE_BIO:
134 pri = PI_DISK;
135 break;
136 case INTR_TYPE_NET:
137 pri = PI_NET;
138 break;
139 case INTR_TYPE_CAM:
140 pri = PI_DISK;
141 break;
142 case INTR_TYPE_AV:
143 pri = PI_AV;
144 break;
145 case INTR_TYPE_CLK:
146 pri = PI_REALTIME;
147 break;
148 case INTR_TYPE_MISC:
149 pri = PI_DULL; /* don't care */
150 break;
151 default:
152 /* We didn't specify an interrupt level. */
153 panic("intr_priority: no interrupt type in flags");
154 }
155
156 return pri;
157 }
158
159 /*
160 * Update an ithread based on the associated intr_event.
161 */
162 static void
163 ithread_update(struct intr_thread *ithd)
164 {
165 struct intr_event *ie;
166 struct thread *td;
167 u_char pri;
168
169 ie = ithd->it_event;
170 td = ithd->it_thread;
171
172 /* Determine the overall priority of this event. */
173 if (TAILQ_EMPTY(&ie->ie_handlers))
174 pri = PRI_MAX_ITHD;
175 else
176 pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri;
177
178 /* Update name and priority. */
179 strlcpy(td->td_proc->p_comm, ie->ie_fullname,
180 sizeof(td->td_proc->p_comm));
181 thread_lock(td);
182 sched_prio(td, pri);
183 thread_unlock(td);
184 }
185
186 /*
187 * Regenerate the full name of an interrupt event and update its priority.
188 */
189 static void
190 intr_event_update(struct intr_event *ie)
191 {
192 struct intr_handler *ih;
193 char *last;
194 int missed, space;
195
196 /* Start off with no entropy and just the name of the event. */
197 mtx_assert(&ie->ie_lock, MA_OWNED);
198 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
199 ie->ie_flags &= ~IE_ENTROPY;
200 missed = 0;
201 space = 1;
202
203 /* Run through all the handlers updating values. */
204 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
205 if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <
206 sizeof(ie->ie_fullname)) {
207 strcat(ie->ie_fullname, " ");
208 strcat(ie->ie_fullname, ih->ih_name);
209 space = 0;
210 } else
211 missed++;
212 if (ih->ih_flags & IH_ENTROPY)
213 ie->ie_flags |= IE_ENTROPY;
214 }
215
216 /*
217 * If the handler names were too long, add +'s to indicate missing
218 * names. If we run out of room and still have +'s to add, change
219 * the last character from a + to a *.
220 */
221 last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];
222 while (missed-- > 0) {
223 if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {
224 if (*last == '+') {
225 *last = '*';
226 break;
227 } else
228 *last = '+';
229 } else if (space) {
230 strcat(ie->ie_fullname, " +");
231 space = 0;
232 } else
233 strcat(ie->ie_fullname, "+");
234 }
235
236 /*
237 * If this event has an ithread, update it's priority and
238 * name.
239 */
240 if (ie->ie_thread != NULL)
241 ithread_update(ie->ie_thread);
242 CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);
243 }
244
245 int
246 intr_event_create(struct intr_event **event, void *source, int flags, int irq,
247 void (*pre_ithread)(void *), void (*post_ithread)(void *),
248 void (*post_filter)(void *), int (*assign_cpu)(void *, u_char),
249 const char *fmt, ...)
250 {
251 struct intr_event *ie;
252 va_list ap;
253
254 /* The only valid flag during creation is IE_SOFT. */
255 if ((flags & ~IE_SOFT) != 0)
256 return (EINVAL);
257 ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);
258 ie->ie_source = source;
259 ie->ie_pre_ithread = pre_ithread;
260 ie->ie_post_ithread = post_ithread;
261 ie->ie_post_filter = post_filter;
262 ie->ie_assign_cpu = assign_cpu;
263 ie->ie_flags = flags;
264 ie->ie_irq = irq;
265 ie->ie_cpu = NOCPU;
266 TAILQ_INIT(&ie->ie_handlers);
267 mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
268
269 va_start(ap, fmt);
270 vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);
271 va_end(ap);
272 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
273 mtx_lock(&event_lock);
274 TAILQ_INSERT_TAIL(&event_list, ie, ie_list);
275 mtx_unlock(&event_lock);
276 if (event != NULL)
277 *event = ie;
278 CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);
279 return (0);
280 }
281
282 /*
283 * Bind an interrupt event to the specified CPU. Note that not all
284 * platforms support binding an interrupt to a CPU. For those
285 * platforms this request will fail. For supported platforms, any
286 * associated ithreads as well as the primary interrupt context will
287 * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds
288 * the interrupt event.
289 */
290 int
291 intr_event_bind(struct intr_event *ie, u_char cpu)
292 {
293 cpuset_t mask;
294 lwpid_t id;
295 int error;
296
297 /* Need a CPU to bind to. */
298 if (cpu != NOCPU && CPU_ABSENT(cpu))
299 return (EINVAL);
300
301 if (ie->ie_assign_cpu == NULL)
302 return (EOPNOTSUPP);
303 /*
304 * If we have any ithreads try to set their mask first since this
305 * can fail.
306 */
307 mtx_lock(&ie->ie_lock);
308 if (ie->ie_thread != NULL) {
309 CPU_ZERO(&mask);
310 if (cpu == NOCPU)
311 CPU_COPY(cpuset_root, &mask);
312 else
313 CPU_SET(cpu, &mask);
314 id = ie->ie_thread->it_thread->td_tid;
315 mtx_unlock(&ie->ie_lock);
316 error = cpuset_setthread(id, &mask);
317 if (error)
318 return (error);
319 } else
320 mtx_unlock(&ie->ie_lock);
321 error = ie->ie_assign_cpu(ie->ie_source, cpu);
322 if (error)
323 return (error);
324 mtx_lock(&ie->ie_lock);
325 ie->ie_cpu = cpu;
326 mtx_unlock(&ie->ie_lock);
327
328 return (error);
329 }
330
331 static struct intr_event *
332 intr_lookup(int irq)
333 {
334 struct intr_event *ie;
335
336 mtx_lock(&event_lock);
337 TAILQ_FOREACH(ie, &event_list, ie_list)
338 if (ie->ie_irq == irq &&
339 (ie->ie_flags & IE_SOFT) == 0 &&
340 TAILQ_FIRST(&ie->ie_handlers) != NULL)
341 break;
342 mtx_unlock(&event_lock);
343 return (ie);
344 }
345
346 int
347 intr_setaffinity(int irq, void *m)
348 {
349 struct intr_event *ie;
350 cpuset_t *mask;
351 u_char cpu;
352 int n;
353
354 mask = m;
355 cpu = NOCPU;
356 /*
357 * If we're setting all cpus we can unbind. Otherwise make sure
358 * only one cpu is in the set.
359 */
360 if (CPU_CMP(cpuset_root, mask)) {
361 for (n = 0; n < CPU_SETSIZE; n++) {
362 if (!CPU_ISSET(n, mask))
363 continue;
364 if (cpu != NOCPU)
365 return (EINVAL);
366 cpu = (u_char)n;
367 }
368 }
369 ie = intr_lookup(irq);
370 if (ie == NULL)
371 return (ESRCH);
372 return (intr_event_bind(ie, cpu));
373 }
374
375 int
376 intr_getaffinity(int irq, void *m)
377 {
378 struct intr_event *ie;
379 cpuset_t *mask;
380
381 mask = m;
382 ie = intr_lookup(irq);
383 if (ie == NULL)
384 return (ESRCH);
385 CPU_ZERO(mask);
386 mtx_lock(&ie->ie_lock);
387 if (ie->ie_cpu == NOCPU)
388 CPU_COPY(cpuset_root, mask);
389 else
390 CPU_SET(ie->ie_cpu, mask);
391 mtx_unlock(&ie->ie_lock);
392 return (0);
393 }
394
395 int
396 intr_event_destroy(struct intr_event *ie)
397 {
398
399 mtx_lock(&event_lock);
400 mtx_lock(&ie->ie_lock);
401 if (!TAILQ_EMPTY(&ie->ie_handlers)) {
402 mtx_unlock(&ie->ie_lock);
403 mtx_unlock(&event_lock);
404 return (EBUSY);
405 }
406 TAILQ_REMOVE(&event_list, ie, ie_list);
407 #ifndef notyet
408 if (ie->ie_thread != NULL) {
409 ithread_destroy(ie->ie_thread);
410 ie->ie_thread = NULL;
411 }
412 #endif
413 mtx_unlock(&ie->ie_lock);
414 mtx_unlock(&event_lock);
415 mtx_destroy(&ie->ie_lock);
416 free(ie, M_ITHREAD);
417 return (0);
418 }
419
420 #ifndef INTR_FILTER
421 static struct intr_thread *
422 ithread_create(const char *name)
423 {
424 struct intr_thread *ithd;
425 struct thread *td;
426 struct proc *p;
427 int error;
428
429 ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
430
431 error = kthread_create(ithread_loop, ithd, &p, RFSTOPPED | RFHIGHPID,
432 0, "%s", name);
433 if (error)
434 panic("kthread_create() failed with %d", error);
435 td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */
436 thread_lock(td);
437 sched_class(td, PRI_ITHD);
438 TD_SET_IWAIT(td);
439 thread_unlock(td);
440 td->td_pflags |= TDP_ITHREAD;
441 ithd->it_thread = td;
442 CTR2(KTR_INTR, "%s: created %s", __func__, name);
443 return (ithd);
444 }
445 #else
446 static struct intr_thread *
447 ithread_create(const char *name, struct intr_handler *ih)
448 {
449 struct intr_thread *ithd;
450 struct thread *td;
451 struct proc *p;
452 int error;
453
454 ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
455
456 error = kthread_create(ithread_loop, ih, &p, RFSTOPPED | RFHIGHPID,
457 0, "%s", name);
458 if (error)
459 panic("kthread_create() failed with %d", error);
460 td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */
461 thread_lock(td);
462 sched_class(td, PRI_ITHD);
463 TD_SET_IWAIT(td);
464 thread_unlock(td);
465 td->td_pflags |= TDP_ITHREAD;
466 ithd->it_thread = td;
467 CTR2(KTR_INTR, "%s: created %s", __func__, name);
468 return (ithd);
469 }
470 #endif
471
472 static void
473 ithread_destroy(struct intr_thread *ithread)
474 {
475 struct thread *td;
476
477 CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name);
478 td = ithread->it_thread;
479 thread_lock(td);
480 ithread->it_flags |= IT_DEAD;
481 if (TD_AWAITING_INTR(td)) {
482 TD_CLR_IWAIT(td);
483 sched_add(td, SRQ_INTR);
484 }
485 thread_unlock(td);
486 }
487
488 #ifndef INTR_FILTER
489 int
490 intr_event_add_handler(struct intr_event *ie, const char *name,
491 driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
492 enum intr_type flags, void **cookiep)
493 {
494 struct intr_handler *ih, *temp_ih;
495 struct intr_thread *it;
496
497 if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
498 return (EINVAL);
499
500 /* Allocate and populate an interrupt handler structure. */
501 ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
502 ih->ih_filter = filter;
503 ih->ih_handler = handler;
504 ih->ih_argument = arg;
505 strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
506 ih->ih_event = ie;
507 ih->ih_pri = pri;
508 if (flags & INTR_EXCL)
509 ih->ih_flags = IH_EXCLUSIVE;
510 if (flags & INTR_MPSAFE)
511 ih->ih_flags |= IH_MPSAFE;
512 if (flags & INTR_ENTROPY)
513 ih->ih_flags |= IH_ENTROPY;
514
515 /* We can only have one exclusive handler in a event. */
516 mtx_lock(&ie->ie_lock);
517 if (!TAILQ_EMPTY(&ie->ie_handlers)) {
518 if ((flags & INTR_EXCL) ||
519 (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
520 mtx_unlock(&ie->ie_lock);
521 free(ih, M_ITHREAD);
522 return (EINVAL);
523 }
524 }
525
526 /* Add the new handler to the event in priority order. */
527 TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
528 if (temp_ih->ih_pri > ih->ih_pri)
529 break;
530 }
531 if (temp_ih == NULL)
532 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
533 else
534 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
535 intr_event_update(ie);
536
537 /* Create a thread if we need one. */
538 while (ie->ie_thread == NULL && handler != NULL) {
539 if (ie->ie_flags & IE_ADDING_THREAD)
540 msleep(ie, &ie->ie_lock, 0, "ithread", 0);
541 else {
542 ie->ie_flags |= IE_ADDING_THREAD;
543 mtx_unlock(&ie->ie_lock);
544 it = ithread_create("intr: newborn");
545 mtx_lock(&ie->ie_lock);
546 ie->ie_flags &= ~IE_ADDING_THREAD;
547 ie->ie_thread = it;
548 it->it_event = ie;
549 ithread_update(it);
550 wakeup(ie);
551 }
552 }
553 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
554 ie->ie_name);
555 mtx_unlock(&ie->ie_lock);
556
557 if (cookiep != NULL)
558 *cookiep = ih;
559 return (0);
560 }
561 #else
562 int
563 intr_event_add_handler(struct intr_event *ie, const char *name,
564 driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
565 enum intr_type flags, void **cookiep)
566 {
567 struct intr_handler *ih, *temp_ih;
568 struct intr_thread *it;
569
570 if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
571 return (EINVAL);
572
573 /* Allocate and populate an interrupt handler structure. */
574 ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
575 ih->ih_filter = filter;
576 ih->ih_handler = handler;
577 ih->ih_argument = arg;
578 strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
579 ih->ih_event = ie;
580 ih->ih_pri = pri;
581 if (flags & INTR_EXCL)
582 ih->ih_flags = IH_EXCLUSIVE;
583 if (flags & INTR_MPSAFE)
584 ih->ih_flags |= IH_MPSAFE;
585 if (flags & INTR_ENTROPY)
586 ih->ih_flags |= IH_ENTROPY;
587
588 /* We can only have one exclusive handler in a event. */
589 mtx_lock(&ie->ie_lock);
590 if (!TAILQ_EMPTY(&ie->ie_handlers)) {
591 if ((flags & INTR_EXCL) ||
592 (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
593 mtx_unlock(&ie->ie_lock);
594 free(ih, M_ITHREAD);
595 return (EINVAL);
596 }
597 }
598
599 /* Add the new handler to the event in priority order. */
600 TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
601 if (temp_ih->ih_pri > ih->ih_pri)
602 break;
603 }
604 if (temp_ih == NULL)
605 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
606 else
607 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
608 intr_event_update(ie);
609
610 /* For filtered handlers, create a private ithread to run on. */
611 if (filter != NULL && handler != NULL) {
612 mtx_unlock(&ie->ie_lock);
613 it = ithread_create("intr: newborn", ih);
614 mtx_lock(&ie->ie_lock);
615 it->it_event = ie;
616 ih->ih_thread = it;
617 ithread_update(it); // XXX - do we really need this?!?!?
618 } else { /* Create the global per-event thread if we need one. */
619 while (ie->ie_thread == NULL && handler != NULL) {
620 if (ie->ie_flags & IE_ADDING_THREAD)
621 msleep(ie, &ie->ie_lock, 0, "ithread", 0);
622 else {
623 ie->ie_flags |= IE_ADDING_THREAD;
624 mtx_unlock(&ie->ie_lock);
625 it = ithread_create("intr: newborn", ih);
626 mtx_lock(&ie->ie_lock);
627 ie->ie_flags &= ~IE_ADDING_THREAD;
628 ie->ie_thread = it;
629 it->it_event = ie;
630 ithread_update(it);
631 wakeup(ie);
632 }
633 }
634 }
635 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
636 ie->ie_name);
637 mtx_unlock(&ie->ie_lock);
638
639 if (cookiep != NULL)
640 *cookiep = ih;
641 return (0);
642 }
643 #endif
644
645 /*
646 * Append a description preceded by a ':' to the name of the specified
647 * interrupt handler.
648 */
649 int
650 intr_event_describe_handler(struct intr_event *ie, void *cookie,
651 const char *descr)
652 {
653 struct intr_handler *ih;
654 size_t space;
655 char *start;
656
657 mtx_lock(&ie->ie_lock);
658 #ifdef INVARIANTS
659 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
660 if (ih == cookie)
661 break;
662 }
663 if (ih == NULL) {
664 mtx_unlock(&ie->ie_lock);
665 panic("handler %p not found in interrupt event %p", cookie, ie);
666 }
667 #endif
668 ih = cookie;
669
670 /*
671 * Look for an existing description by checking for an
672 * existing ":". This assumes device names do not include
673 * colons. If one is found, prepare to insert the new
674 * description at that point. If one is not found, find the
675 * end of the name to use as the insertion point.
676 */
677 start = index(ih->ih_name, ':');
678 if (start == NULL)
679 start = index(ih->ih_name, 0);
680
681 /*
682 * See if there is enough remaining room in the string for the
683 * description + ":". The "- 1" leaves room for the trailing
684 * '\0'. The "+ 1" accounts for the colon.
685 */
686 space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1;
687 if (strlen(descr) + 1 > space) {
688 mtx_unlock(&ie->ie_lock);
689 return (ENOSPC);
690 }
691
692 /* Append a colon followed by the description. */
693 *start = ':';
694 strcpy(start + 1, descr);
695 intr_event_update(ie);
696 mtx_unlock(&ie->ie_lock);
697 return (0);
698 }
699
700 /*
701 * Return the ie_source field from the intr_event an intr_handler is
702 * associated with.
703 */
704 void *
705 intr_handler_source(void *cookie)
706 {
707 struct intr_handler *ih;
708 struct intr_event *ie;
709
710 ih = (struct intr_handler *)cookie;
711 if (ih == NULL)
712 return (NULL);
713 ie = ih->ih_event;
714 KASSERT(ie != NULL,
715 ("interrupt handler \"%s\" has a NULL interrupt event",
716 ih->ih_name));
717 return (ie->ie_source);
718 }
719
720 #ifndef INTR_FILTER
721 int
722 intr_event_remove_handler(void *cookie)
723 {
724 struct intr_handler *handler = (struct intr_handler *)cookie;
725 struct intr_event *ie;
726 #ifdef INVARIANTS
727 struct intr_handler *ih;
728 #endif
729 #ifdef notyet
730 int dead;
731 #endif
732
733 if (handler == NULL)
734 return (EINVAL);
735 ie = handler->ih_event;
736 KASSERT(ie != NULL,
737 ("interrupt handler \"%s\" has a NULL interrupt event",
738 handler->ih_name));
739 mtx_lock(&ie->ie_lock);
740 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
741 ie->ie_name);
742 #ifdef INVARIANTS
743 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
744 if (ih == handler)
745 goto ok;
746 mtx_unlock(&ie->ie_lock);
747 panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
748 ih->ih_name, ie->ie_name);
749 ok:
750 #endif
751 /*
752 * If there is no ithread, then just remove the handler and return.
753 * XXX: Note that an INTR_FAST handler might be running on another
754 * CPU!
755 */
756 if (ie->ie_thread == NULL) {
757 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
758 mtx_unlock(&ie->ie_lock);
759 free(handler, M_ITHREAD);
760 return (0);
761 }
762
763 /*
764 * If the interrupt thread is already running, then just mark this
765 * handler as being dead and let the ithread do the actual removal.
766 *
767 * During a cold boot while cold is set, msleep() does not sleep,
768 * so we have to remove the handler here rather than letting the
769 * thread do it.
770 */
771 thread_lock(ie->ie_thread->it_thread);
772 if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) {
773 handler->ih_flags |= IH_DEAD;
774
775 /*
776 * Ensure that the thread will process the handler list
777 * again and remove this handler if it has already passed
778 * it on the list.
779 */
780 ie->ie_thread->it_need = 1;
781 } else
782 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
783 thread_unlock(ie->ie_thread->it_thread);
784 while (handler->ih_flags & IH_DEAD)
785 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
786 intr_event_update(ie);
787 #ifdef notyet
788 /*
789 * XXX: This could be bad in the case of ppbus(8). Also, I think
790 * this could lead to races of stale data when servicing an
791 * interrupt.
792 */
793 dead = 1;
794 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
795 if (!(ih->ih_flags & IH_FAST)) {
796 dead = 0;
797 break;
798 }
799 }
800 if (dead) {
801 ithread_destroy(ie->ie_thread);
802 ie->ie_thread = NULL;
803 }
804 #endif
805 mtx_unlock(&ie->ie_lock);
806 free(handler, M_ITHREAD);
807 return (0);
808 }
809
810 static int
811 intr_event_schedule_thread(struct intr_event *ie)
812 {
813 struct intr_entropy entropy;
814 struct intr_thread *it;
815 struct thread *td;
816 struct thread *ctd;
817 struct proc *p;
818
819 /*
820 * If no ithread or no handlers, then we have a stray interrupt.
821 */
822 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) ||
823 ie->ie_thread == NULL)
824 return (EINVAL);
825
826 ctd = curthread;
827 it = ie->ie_thread;
828 td = it->it_thread;
829 p = td->td_proc;
830
831 /*
832 * If any of the handlers for this ithread claim to be good
833 * sources of entropy, then gather some.
834 */
835 if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
836 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
837 p->p_pid, p->p_comm);
838 entropy.event = (uintptr_t)ie;
839 entropy.td = ctd;
840 random_harvest(&entropy, sizeof(entropy), 2, 0,
841 RANDOM_INTERRUPT);
842 }
843
844 KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
845
846 /*
847 * Set it_need to tell the thread to keep running if it is already
848 * running. Then, lock the thread and see if we actually need to
849 * put it on the runqueue.
850 */
851 it->it_need = 1;
852 thread_lock(td);
853 if (TD_AWAITING_INTR(td)) {
854 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
855 p->p_comm);
856 TD_CLR_IWAIT(td);
857 sched_add(td, SRQ_INTR);
858 } else {
859 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
860 __func__, p->p_pid, p->p_comm, it->it_need, td->td_state);
861 }
862 thread_unlock(td);
863
864 return (0);
865 }
866 #else
867 int
868 intr_event_remove_handler(void *cookie)
869 {
870 struct intr_handler *handler = (struct intr_handler *)cookie;
871 struct intr_event *ie;
872 struct intr_thread *it;
873 #ifdef INVARIANTS
874 struct intr_handler *ih;
875 #endif
876 #ifdef notyet
877 int dead;
878 #endif
879
880 if (handler == NULL)
881 return (EINVAL);
882 ie = handler->ih_event;
883 KASSERT(ie != NULL,
884 ("interrupt handler \"%s\" has a NULL interrupt event",
885 handler->ih_name));
886 mtx_lock(&ie->ie_lock);
887 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
888 ie->ie_name);
889 #ifdef INVARIANTS
890 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
891 if (ih == handler)
892 goto ok;
893 mtx_unlock(&ie->ie_lock);
894 panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
895 ih->ih_name, ie->ie_name);
896 ok:
897 #endif
898 /*
899 * If there are no ithreads (per event and per handler), then
900 * just remove the handler and return.
901 * XXX: Note that an INTR_FAST handler might be running on another CPU!
902 */
903 if (ie->ie_thread == NULL && handler->ih_thread == NULL) {
904 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
905 mtx_unlock(&ie->ie_lock);
906 free(handler, M_ITHREAD);
907 return (0);
908 }
909
910 /* Private or global ithread? */
911 it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread;
912 /*
913 * If the interrupt thread is already running, then just mark this
914 * handler as being dead and let the ithread do the actual removal.
915 *
916 * During a cold boot while cold is set, msleep() does not sleep,
917 * so we have to remove the handler here rather than letting the
918 * thread do it.
919 */
920 thread_lock(it->it_thread);
921 if (!TD_AWAITING_INTR(it->it_thread) && !cold) {
922 handler->ih_flags |= IH_DEAD;
923
924 /*
925 * Ensure that the thread will process the handler list
926 * again and remove this handler if it has already passed
927 * it on the list.
928 */
929 it->it_need = 1;
930 } else
931 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
932 thread_unlock(it->it_thread);
933 while (handler->ih_flags & IH_DEAD)
934 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
935 /*
936 * At this point, the handler has been disconnected from the event,
937 * so we can kill the private ithread if any.
938 */
939 if (handler->ih_thread) {
940 ithread_destroy(handler->ih_thread);
941 handler->ih_thread = NULL;
942 }
943 intr_event_update(ie);
944 #ifdef notyet
945 /*
946 * XXX: This could be bad in the case of ppbus(8). Also, I think
947 * this could lead to races of stale data when servicing an
948 * interrupt.
949 */
950 dead = 1;
951 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
952 if (handler != NULL) {
953 dead = 0;
954 break;
955 }
956 }
957 if (dead) {
958 ithread_destroy(ie->ie_thread);
959 ie->ie_thread = NULL;
960 }
961 #endif
962 mtx_unlock(&ie->ie_lock);
963 free(handler, M_ITHREAD);
964 return (0);
965 }
966
967 static int
968 intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it)
969 {
970 struct intr_entropy entropy;
971 struct thread *td;
972 struct thread *ctd;
973 struct proc *p;
974
975 /*
976 * If no ithread or no handlers, then we have a stray interrupt.
977 */
978 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL)
979 return (EINVAL);
980
981 ctd = curthread;
982 td = it->it_thread;
983 p = td->td_proc;
984
985 /*
986 * If any of the handlers for this ithread claim to be good
987 * sources of entropy, then gather some.
988 */
989 if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
990 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
991 p->p_pid, p->p_comm);
992 entropy.event = (uintptr_t)ie;
993 entropy.td = ctd;
994 random_harvest(&entropy, sizeof(entropy), 2, 0,
995 RANDOM_INTERRUPT);
996 }
997
998 KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
999
1000 /*
1001 * Set it_need to tell the thread to keep running if it is already
1002 * running. Then, lock the thread and see if we actually need to
1003 * put it on the runqueue.
1004 */
1005 it->it_need = 1;
1006 thread_lock(td);
1007 if (TD_AWAITING_INTR(td)) {
1008 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
1009 p->p_comm);
1010 TD_CLR_IWAIT(td);
1011 sched_add(td, SRQ_INTR);
1012 } else {
1013 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
1014 __func__, p->p_pid, p->p_comm, it->it_need, td->td_state);
1015 }
1016 thread_unlock(td);
1017
1018 return (0);
1019 }
1020 #endif
1021
1022 /*
1023 * Add a software interrupt handler to a specified event. If a given event
1024 * is not specified, then a new event is created.
1025 */
1026 int
1027 swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
1028 void *arg, int pri, enum intr_type flags, void **cookiep)
1029 {
1030 struct intr_event *ie;
1031 int error;
1032
1033 if (flags & INTR_ENTROPY)
1034 return (EINVAL);
1035
1036 ie = (eventp != NULL) ? *eventp : NULL;
1037
1038 if (ie != NULL) {
1039 if (!(ie->ie_flags & IE_SOFT))
1040 return (EINVAL);
1041 } else {
1042 error = intr_event_create(&ie, NULL, IE_SOFT, 0,
1043 NULL, NULL, NULL, NULL, "swi%d:", pri);
1044 if (error)
1045 return (error);
1046 if (eventp != NULL)
1047 *eventp = ie;
1048 }
1049 return (intr_event_add_handler(ie, name, NULL, handler, arg,
1050 PI_SWI(pri), flags, cookiep));
1051 /* XXKSE.. think of a better way to get separate queues */
1052 }
1053
1054 /*
1055 * Schedule a software interrupt thread.
1056 */
1057 void
1058 swi_sched(void *cookie, int flags)
1059 {
1060 struct intr_handler *ih = (struct intr_handler *)cookie;
1061 struct intr_event *ie = ih->ih_event;
1062 int error;
1063
1064 CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
1065 ih->ih_need);
1066
1067 /*
1068 * Set ih_need for this handler so that if the ithread is already
1069 * running it will execute this handler on the next pass. Otherwise,
1070 * it will execute it the next time it runs.
1071 */
1072 atomic_store_rel_int(&ih->ih_need, 1);
1073
1074 if (!(flags & SWI_DELAY)) {
1075 PCPU_INC(cnt.v_soft);
1076 #ifdef INTR_FILTER
1077 error = intr_event_schedule_thread(ie, ie->ie_thread);
1078 #else
1079 error = intr_event_schedule_thread(ie);
1080 #endif
1081 KASSERT(error == 0, ("stray software interrupt"));
1082 }
1083 }
1084
1085 /*
1086 * Remove a software interrupt handler. Currently this code does not
1087 * remove the associated interrupt event if it becomes empty. Calling code
1088 * may do so manually via intr_event_destroy(), but that's not really
1089 * an optimal interface.
1090 */
1091 int
1092 swi_remove(void *cookie)
1093 {
1094
1095 return (intr_event_remove_handler(cookie));
1096 }
1097
1098 #ifdef INTR_FILTER
1099 static void
1100 priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih)
1101 {
1102 struct intr_event *ie;
1103
1104 ie = ih->ih_event;
1105 /*
1106 * If this handler is marked for death, remove it from
1107 * the list of handlers and wake up the sleeper.
1108 */
1109 if (ih->ih_flags & IH_DEAD) {
1110 mtx_lock(&ie->ie_lock);
1111 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1112 ih->ih_flags &= ~IH_DEAD;
1113 wakeup(ih);
1114 mtx_unlock(&ie->ie_lock);
1115 return;
1116 }
1117
1118 /* Execute this handler. */
1119 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
1120 __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument,
1121 ih->ih_name, ih->ih_flags);
1122
1123 if (!(ih->ih_flags & IH_MPSAFE))
1124 mtx_lock(&Giant);
1125 ih->ih_handler(ih->ih_argument);
1126 if (!(ih->ih_flags & IH_MPSAFE))
1127 mtx_unlock(&Giant);
1128 }
1129 #endif
1130
1131 static void
1132 ithread_execute_handlers(struct proc *p, struct intr_event *ie)
1133 {
1134 struct intr_handler *ih, *ihn;
1135
1136 /* Interrupt handlers should not sleep. */
1137 if (!(ie->ie_flags & IE_SOFT))
1138 THREAD_NO_SLEEPING();
1139 TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {
1140
1141 /*
1142 * If this handler is marked for death, remove it from
1143 * the list of handlers and wake up the sleeper.
1144 */
1145 if (ih->ih_flags & IH_DEAD) {
1146 mtx_lock(&ie->ie_lock);
1147 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1148 ih->ih_flags &= ~IH_DEAD;
1149 wakeup(ih);
1150 mtx_unlock(&ie->ie_lock);
1151 continue;
1152 }
1153
1154 /* Skip filter only handlers */
1155 if (ih->ih_handler == NULL)
1156 continue;
1157
1158 /*
1159 * For software interrupt threads, we only execute
1160 * handlers that have their need flag set. Hardware
1161 * interrupt threads always invoke all of their handlers.
1162 */
1163 if (ie->ie_flags & IE_SOFT) {
1164 if (!ih->ih_need)
1165 continue;
1166 else
1167 atomic_store_rel_int(&ih->ih_need, 0);
1168 }
1169
1170 /* Execute this handler. */
1171 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
1172 __func__, p->p_pid, (void *)ih->ih_handler,
1173 ih->ih_argument, ih->ih_name, ih->ih_flags);
1174
1175 if (!(ih->ih_flags & IH_MPSAFE))
1176 mtx_lock(&Giant);
1177 ih->ih_handler(ih->ih_argument);
1178 if (!(ih->ih_flags & IH_MPSAFE))
1179 mtx_unlock(&Giant);
1180 }
1181 if (!(ie->ie_flags & IE_SOFT))
1182 THREAD_SLEEPING_OK();
1183
1184 /*
1185 * Interrupt storm handling:
1186 *
1187 * If this interrupt source is currently storming, then throttle
1188 * it to only fire the handler once per clock tick.
1189 *
1190 * If this interrupt source is not currently storming, but the
1191 * number of back to back interrupts exceeds the storm threshold,
1192 * then enter storming mode.
1193 */
1194 if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold &&
1195 !(ie->ie_flags & IE_SOFT)) {
1196 /* Report the message only once every second. */
1197 if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {
1198 printf(
1199 "interrupt storm detected on \"%s\"; throttling interrupt source\n",
1200 ie->ie_name);
1201 }
1202 pause("istorm", 1);
1203 } else
1204 ie->ie_count++;
1205
1206 /*
1207 * Now that all the handlers have had a chance to run, reenable
1208 * the interrupt source.
1209 */
1210 if (ie->ie_post_ithread != NULL)
1211 ie->ie_post_ithread(ie->ie_source);
1212 }
1213
1214 #ifndef INTR_FILTER
1215 /*
1216 * This is the main code for interrupt threads.
1217 */
1218 static void
1219 ithread_loop(void *arg)
1220 {
1221 struct intr_thread *ithd;
1222 struct intr_event *ie;
1223 struct thread *td;
1224 struct proc *p;
1225
1226 td = curthread;
1227 p = td->td_proc;
1228 ithd = (struct intr_thread *)arg;
1229 KASSERT(ithd->it_thread == td,
1230 ("%s: ithread and proc linkage out of sync", __func__));
1231 ie = ithd->it_event;
1232 ie->ie_count = 0;
1233
1234 /*
1235 * As long as we have interrupts outstanding, go through the
1236 * list of handlers, giving each one a go at it.
1237 */
1238 for (;;) {
1239 /*
1240 * If we are an orphaned thread, then just die.
1241 */
1242 if (ithd->it_flags & IT_DEAD) {
1243 CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
1244 p->p_pid, p->p_comm);
1245 free(ithd, M_ITHREAD);
1246 kthread_exit(0);
1247 }
1248
1249 /*
1250 * Service interrupts. If another interrupt arrives while
1251 * we are running, it will set it_need to note that we
1252 * should make another pass.
1253 */
1254 while (ithd->it_need) {
1255 /*
1256 * This might need a full read and write barrier
1257 * to make sure that this write posts before any
1258 * of the memory or device accesses in the
1259 * handlers.
1260 */
1261 atomic_store_rel_int(&ithd->it_need, 0);
1262 ithread_execute_handlers(p, ie);
1263 }
1264 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
1265 mtx_assert(&Giant, MA_NOTOWNED);
1266
1267 /*
1268 * Processed all our interrupts. Now get the sched
1269 * lock. This may take a while and it_need may get
1270 * set again, so we have to check it again.
1271 */
1272 thread_lock(td);
1273 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) {
1274 TD_SET_IWAIT(td);
1275 ie->ie_count = 0;
1276 mi_switch(SW_VOL, NULL);
1277 }
1278 thread_unlock(td);
1279 }
1280 }
1281
1282 /*
1283 * Main interrupt handling body.
1284 *
1285 * Input:
1286 * o ie: the event connected to this interrupt.
1287 * o frame: some archs (i.e. i386) pass a frame to some.
1288 * handlers as their main argument.
1289 * Return value:
1290 * o 0: everything ok.
1291 * o EINVAL: stray interrupt.
1292 */
1293 int
1294 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
1295 {
1296 struct intr_handler *ih;
1297 struct thread *td;
1298 int error, ret, thread;
1299
1300 td = curthread;
1301
1302 /* An interrupt with no event or handlers is a stray interrupt. */
1303 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
1304 return (EINVAL);
1305
1306 /*
1307 * Execute fast interrupt handlers directly.
1308 * To support clock handlers, if a handler registers
1309 * with a NULL argument, then we pass it a pointer to
1310 * a trapframe as its argument.
1311 */
1312 td->td_intr_nesting_level++;
1313 thread = 0;
1314 ret = 0;
1315 critical_enter();
1316 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1317 if (ih->ih_filter == NULL) {
1318 thread = 1;
1319 continue;
1320 }
1321 CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__,
1322 ih->ih_filter, ih->ih_argument == NULL ? frame :
1323 ih->ih_argument, ih->ih_name);
1324 if (ih->ih_argument == NULL)
1325 ret = ih->ih_filter(frame);
1326 else
1327 ret = ih->ih_filter(ih->ih_argument);
1328 KASSERT(ret == FILTER_STRAY ||
1329 ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
1330 (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
1331 ("%s: incorrect return value %#x from %s", __func__, ret,
1332 ih->ih_name));
1333
1334 /*
1335 * Wrapper handler special handling:
1336 *
1337 * in some particular cases (like pccard and pccbb),
1338 * the _real_ device handler is wrapped in a couple of
1339 * functions - a filter wrapper and an ithread wrapper.
1340 * In this case (and just in this case), the filter wrapper
1341 * could ask the system to schedule the ithread and mask
1342 * the interrupt source if the wrapped handler is composed
1343 * of just an ithread handler.
1344 *
1345 * TODO: write a generic wrapper to avoid people rolling
1346 * their own
1347 */
1348 if (!thread) {
1349 if (ret == FILTER_SCHEDULE_THREAD)
1350 thread = 1;
1351 }
1352 }
1353
1354 if (thread) {
1355 if (ie->ie_pre_ithread != NULL)
1356 ie->ie_pre_ithread(ie->ie_source);
1357 } else {
1358 if (ie->ie_post_filter != NULL)
1359 ie->ie_post_filter(ie->ie_source);
1360 }
1361
1362 /* Schedule the ithread if needed. */
1363 if (thread) {
1364 error = intr_event_schedule_thread(ie);
1365 KASSERT(error == 0, ("bad stray interrupt"));
1366 }
1367 critical_exit();
1368 td->td_intr_nesting_level--;
1369 return (0);
1370 }
1371 #else
1372 /*
1373 * This is the main code for interrupt threads.
1374 */
1375 static void
1376 ithread_loop(void *arg)
1377 {
1378 struct intr_thread *ithd;
1379 struct intr_handler *ih;
1380 struct intr_event *ie;
1381 struct thread *td;
1382 struct proc *p;
1383 int priv;
1384
1385 td = curthread;
1386 p = td->td_proc;
1387 ih = (struct intr_handler *)arg;
1388 priv = (ih->ih_thread != NULL) ? 1 : 0;
1389 ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread;
1390 KASSERT(ithd->it_thread == td,
1391 ("%s: ithread and proc linkage out of sync", __func__));
1392 ie = ithd->it_event;
1393 ie->ie_count = 0;
1394
1395 /*
1396 * As long as we have interrupts outstanding, go through the
1397 * list of handlers, giving each one a go at it.
1398 */
1399 for (;;) {
1400 /*
1401 * If we are an orphaned thread, then just die.
1402 */
1403 if (ithd->it_flags & IT_DEAD) {
1404 CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
1405 p->p_pid, p->p_comm);
1406 free(ithd, M_ITHREAD);
1407 kthread_exit(0);
1408 }
1409
1410 /*
1411 * Service interrupts. If another interrupt arrives while
1412 * we are running, it will set it_need to note that we
1413 * should make another pass.
1414 */
1415 while (ithd->it_need) {
1416 /*
1417 * This might need a full read and write barrier
1418 * to make sure that this write posts before any
1419 * of the memory or device accesses in the
1420 * handlers.
1421 */
1422 atomic_store_rel_int(&ithd->it_need, 0);
1423 if (priv)
1424 priv_ithread_execute_handler(p, ih);
1425 else
1426 ithread_execute_handlers(p, ie);
1427 }
1428 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
1429 mtx_assert(&Giant, MA_NOTOWNED);
1430
1431 /*
1432 * Processed all our interrupts. Now get the sched
1433 * lock. This may take a while and it_need may get
1434 * set again, so we have to check it again.
1435 */
1436 thread_lock(td);
1437 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) {
1438 TD_SET_IWAIT(td);
1439 ie->ie_count = 0;
1440 mi_switch(SW_VOL, NULL);
1441 }
1442 thread_unlock(td);
1443 }
1444 }
1445
1446 /*
1447 * Main loop for interrupt filter.
1448 *
1449 * Some architectures (i386, amd64 and arm) require the optional frame
1450 * parameter, and use it as the main argument for fast handler execution
1451 * when ih_argument == NULL.
1452 *
1453 * Return value:
1454 * o FILTER_STRAY: No filter recognized the event, and no
1455 * filter-less handler is registered on this
1456 * line.
1457 * o FILTER_HANDLED: A filter claimed the event and served it.
1458 * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at
1459 * least one filter-less handler on this line.
1460 * o FILTER_HANDLED |
1461 * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for
1462 * scheduling the per-handler ithread.
1463 *
1464 * In case an ithread has to be scheduled, in *ithd there will be a
1465 * pointer to a struct intr_thread containing the thread to be
1466 * scheduled.
1467 */
1468
1469 static int
1470 intr_filter_loop(struct intr_event *ie, struct trapframe *frame,
1471 struct intr_thread **ithd)
1472 {
1473 struct intr_handler *ih;
1474 void *arg;
1475 int ret, thread_only;
1476
1477 ret = 0;
1478 thread_only = 0;
1479 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1480 /*
1481 * Execute fast interrupt handlers directly.
1482 * To support clock handlers, if a handler registers
1483 * with a NULL argument, then we pass it a pointer to
1484 * a trapframe as its argument.
1485 */
1486 arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument);
1487
1488 CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__,
1489 ih->ih_filter, ih->ih_handler, arg, ih->ih_name);
1490
1491 if (ih->ih_filter != NULL)
1492 ret = ih->ih_filter(arg);
1493 else {
1494 thread_only = 1;
1495 continue;
1496 }
1497 KASSERT(ret == FILTER_STRAY ||
1498 ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
1499 (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
1500 ("%s: incorrect return value %#x from %s", __func__, ret,
1501 ih->ih_name));
1502 if (ret & FILTER_STRAY)
1503 continue;
1504 else {
1505 *ithd = ih->ih_thread;
1506 return (ret);
1507 }
1508 }
1509
1510 /*
1511 * No filters handled the interrupt and we have at least
1512 * one handler without a filter. In this case, we schedule
1513 * all of the filter-less handlers to run in the ithread.
1514 */
1515 if (thread_only) {
1516 *ithd = ie->ie_thread;
1517 return (FILTER_SCHEDULE_THREAD);
1518 }
1519 return (FILTER_STRAY);
1520 }
1521
1522 /*
1523 * Main interrupt handling body.
1524 *
1525 * Input:
1526 * o ie: the event connected to this interrupt.
1527 * o frame: some archs (i.e. i386) pass a frame to some.
1528 * handlers as their main argument.
1529 * Return value:
1530 * o 0: everything ok.
1531 * o EINVAL: stray interrupt.
1532 */
1533 int
1534 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
1535 {
1536 struct intr_thread *ithd;
1537 struct thread *td;
1538 int thread;
1539
1540 ithd = NULL;
1541 td = curthread;
1542
1543 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
1544 return (EINVAL);
1545
1546 td->td_intr_nesting_level++;
1547 thread = 0;
1548 critical_enter();
1549 thread = intr_filter_loop(ie, frame, &ithd);
1550 if (thread & FILTER_HANDLED) {
1551 if (ie->ie_post_filter != NULL)
1552 ie->ie_post_filter(ie->ie_source);
1553 } else {
1554 if (ie->ie_pre_ithread != NULL)
1555 ie->ie_pre_ithread(ie->ie_source);
1556 }
1557 critical_exit();
1558
1559 /* Interrupt storm logic */
1560 if (thread & FILTER_STRAY) {
1561 ie->ie_count++;
1562 if (ie->ie_count < intr_storm_threshold)
1563 printf("Interrupt stray detection not present\n");
1564 }
1565
1566 /* Schedule an ithread if needed. */
1567 if (thread & FILTER_SCHEDULE_THREAD) {
1568 if (intr_event_schedule_thread(ie, ithd) != 0)
1569 panic("%s: impossible stray interrupt", __func__);
1570 }
1571 td->td_intr_nesting_level--;
1572 return (0);
1573 }
1574 #endif
1575
1576 #ifdef DDB
1577 /*
1578 * Dump details about an interrupt handler
1579 */
1580 static void
1581 db_dump_intrhand(struct intr_handler *ih)
1582 {
1583 int comma;
1584
1585 db_printf("\t%-10s ", ih->ih_name);
1586 switch (ih->ih_pri) {
1587 case PI_REALTIME:
1588 db_printf("CLK ");
1589 break;
1590 case PI_AV:
1591 db_printf("AV ");
1592 break;
1593 case PI_TTY:
1594 db_printf("TTY ");
1595 break;
1596 case PI_NET:
1597 db_printf("NET ");
1598 break;
1599 case PI_DISK:
1600 db_printf("DISK");
1601 break;
1602 case PI_DULL:
1603 db_printf("DULL");
1604 break;
1605 default:
1606 if (ih->ih_pri >= PI_SOFT)
1607 db_printf("SWI ");
1608 else
1609 db_printf("%4u", ih->ih_pri);
1610 break;
1611 }
1612 db_printf(" ");
1613 db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC);
1614 db_printf("(%p)", ih->ih_argument);
1615 if (ih->ih_need ||
1616 (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD |
1617 IH_MPSAFE)) != 0) {
1618 db_printf(" {");
1619 comma = 0;
1620 if (ih->ih_flags & IH_EXCLUSIVE) {
1621 if (comma)
1622 db_printf(", ");
1623 db_printf("EXCL");
1624 comma = 1;
1625 }
1626 if (ih->ih_flags & IH_ENTROPY) {
1627 if (comma)
1628 db_printf(", ");
1629 db_printf("ENTROPY");
1630 comma = 1;
1631 }
1632 if (ih->ih_flags & IH_DEAD) {
1633 if (comma)
1634 db_printf(", ");
1635 db_printf("DEAD");
1636 comma = 1;
1637 }
1638 if (ih->ih_flags & IH_MPSAFE) {
1639 if (comma)
1640 db_printf(", ");
1641 db_printf("MPSAFE");
1642 comma = 1;
1643 }
1644 if (ih->ih_need) {
1645 if (comma)
1646 db_printf(", ");
1647 db_printf("NEED");
1648 }
1649 db_printf("}");
1650 }
1651 db_printf("\n");
1652 }
1653
1654 /*
1655 * Dump details about a event.
1656 */
1657 void
1658 db_dump_intr_event(struct intr_event *ie, int handlers)
1659 {
1660 struct intr_handler *ih;
1661 struct intr_thread *it;
1662 int comma;
1663
1664 db_printf("%s ", ie->ie_fullname);
1665 it = ie->ie_thread;
1666 if (it != NULL)
1667 db_printf("(pid %d)", it->it_thread->td_proc->p_pid);
1668 else
1669 db_printf("(no thread)");
1670 if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 ||
1671 (it != NULL && it->it_need)) {
1672 db_printf(" {");
1673 comma = 0;
1674 if (ie->ie_flags & IE_SOFT) {
1675 db_printf("SOFT");
1676 comma = 1;
1677 }
1678 if (ie->ie_flags & IE_ENTROPY) {
1679 if (comma)
1680 db_printf(", ");
1681 db_printf("ENTROPY");
1682 comma = 1;
1683 }
1684 if (ie->ie_flags & IE_ADDING_THREAD) {
1685 if (comma)
1686 db_printf(", ");
1687 db_printf("ADDING_THREAD");
1688 comma = 1;
1689 }
1690 if (it != NULL && it->it_need) {
1691 if (comma)
1692 db_printf(", ");
1693 db_printf("NEED");
1694 }
1695 db_printf("}");
1696 }
1697 db_printf("\n");
1698
1699 if (handlers)
1700 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
1701 db_dump_intrhand(ih);
1702 }
1703
1704 /*
1705 * Dump data about interrupt handlers
1706 */
1707 DB_SHOW_COMMAND(intr, db_show_intr)
1708 {
1709 struct intr_event *ie;
1710 int all, verbose;
1711
1712 verbose = index(modif, 'v') != NULL;
1713 all = index(modif, 'a') != NULL;
1714 TAILQ_FOREACH(ie, &event_list, ie_list) {
1715 if (!all && TAILQ_EMPTY(&ie->ie_handlers))
1716 continue;
1717 db_dump_intr_event(ie, verbose);
1718 if (db_pager_quit)
1719 break;
1720 }
1721 }
1722 #endif /* DDB */
1723
1724 /*
1725 * Start standard software interrupt threads
1726 */
1727 static void
1728 start_softintr(void *dummy)
1729 {
1730 struct proc *p;
1731
1732 if (swi_add(&clk_intr_event, "clock", softclock, NULL, SWI_CLOCK,
1733 INTR_MPSAFE, &softclock_ih) ||
1734 swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
1735 panic("died while creating standard software ithreads");
1736
1737 p = clk_intr_event->ie_thread->it_thread->td_proc;
1738 PROC_LOCK(p);
1739 p->p_flag |= P_NOLOAD;
1740 PROC_UNLOCK(p);
1741 }
1742 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr,
1743 NULL);
1744
1745 /*
1746 * Sysctls used by systat and others: hw.intrnames and hw.intrcnt.
1747 * The data for this machine dependent, and the declarations are in machine
1748 * dependent code. The layout of intrnames and intrcnt however is machine
1749 * independent.
1750 *
1751 * We do not know the length of intrcnt and intrnames at compile time, so
1752 * calculate things at run time.
1753 */
1754 static int
1755 sysctl_intrnames(SYSCTL_HANDLER_ARGS)
1756 {
1757 return (sysctl_handle_opaque(oidp, intrnames, eintrnames - intrnames,
1758 req));
1759 }
1760
1761 SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD,
1762 NULL, 0, sysctl_intrnames, "", "Interrupt Names");
1763
1764 static int
1765 sysctl_intrcnt(SYSCTL_HANDLER_ARGS)
1766 {
1767 return (sysctl_handle_opaque(oidp, intrcnt,
1768 (char *)eintrcnt - (char *)intrcnt, req));
1769 }
1770
1771 SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD,
1772 NULL, 0, sysctl_intrcnt, "", "Interrupt Counts");
1773
1774 #ifdef DDB
1775 /*
1776 * DDB command to dump the interrupt statistics.
1777 */
1778 DB_SHOW_COMMAND(intrcnt, db_show_intrcnt)
1779 {
1780 u_long *i;
1781 char *cp;
1782
1783 cp = intrnames;
1784 for (i = intrcnt; i != eintrcnt && !db_pager_quit; i++) {
1785 if (*cp == '\0')
1786 break;
1787 if (*i != 0)
1788 db_printf("%s\t%lu\n", cp, *i);
1789 cp += strlen(cp) + 1;
1790 }
1791 }
1792 #endif
Cache object: f790167dfe8599475912dab77dc5c822
|