The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_intr.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice unmodified, this list of conditions, and the following
   10  *    disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD$");
   29 
   30 #include "opt_ddb.h"
   31 
   32 #include <sys/param.h>
   33 #include <sys/bus.h>
   34 #include <sys/conf.h>
   35 #include <sys/cpuset.h>
   36 #include <sys/rtprio.h>
   37 #include <sys/systm.h>
   38 #include <sys/interrupt.h>
   39 #include <sys/kernel.h>
   40 #include <sys/kthread.h>
   41 #include <sys/ktr.h>
   42 #include <sys/limits.h>
   43 #include <sys/lock.h>
   44 #include <sys/malloc.h>
   45 #include <sys/mutex.h>
   46 #include <sys/proc.h>
   47 #include <sys/random.h>
   48 #include <sys/resourcevar.h>
   49 #include <sys/sched.h>
   50 #include <sys/smp.h>
   51 #include <sys/sysctl.h>
   52 #include <sys/unistd.h>
   53 #include <sys/vmmeter.h>
   54 #include <machine/atomic.h>
   55 #include <machine/cpu.h>
   56 #include <machine/md_var.h>
   57 #include <machine/stdarg.h>
   58 #ifdef DDB
   59 #include <ddb/ddb.h>
   60 #include <ddb/db_sym.h>
   61 #endif
   62 
   63 /*
   64  * Describe an interrupt thread.  There is one of these per interrupt event.
   65  */
   66 struct intr_thread {
   67         struct intr_event *it_event;
   68         struct thread *it_thread;       /* Kernel thread. */
   69         int     it_flags;               /* (j) IT_* flags. */
   70         int     it_need;                /* Needs service. */
   71 };
   72 
   73 /* Interrupt thread flags kept in it_flags */
   74 #define IT_DEAD         0x000001        /* Thread is waiting to exit. */
   75 
   76 struct  intr_entropy {
   77         struct  thread *td;
   78         uintptr_t event;
   79 };
   80 
   81 struct  intr_event *clk_intr_event;
   82 struct  intr_event *tty_intr_event;
   83 void    *softclock_ih;
   84 void    *vm_ih;
   85 
   86 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads");
   87 
   88 static int intr_storm_threshold = 1000;
   89 TUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold);
   90 SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW,
   91     &intr_storm_threshold, 0,
   92     "Number of consecutive interrupts before storm protection is enabled");
   93 static TAILQ_HEAD(, intr_event) event_list =
   94     TAILQ_HEAD_INITIALIZER(event_list);
   95 static struct mtx event_lock;
   96 MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF);
   97 
   98 static void     intr_event_update(struct intr_event *ie);
   99 #ifdef INTR_FILTER
  100 static int      intr_event_schedule_thread(struct intr_event *ie,
  101                     struct intr_thread *ithd);
  102 static int      intr_filter_loop(struct intr_event *ie,
  103                     struct trapframe *frame, struct intr_thread **ithd);
  104 static struct intr_thread *ithread_create(const char *name,
  105                               struct intr_handler *ih);
  106 #else
  107 static int      intr_event_schedule_thread(struct intr_event *ie);
  108 static struct intr_thread *ithread_create(const char *name);
  109 #endif
  110 static void     ithread_destroy(struct intr_thread *ithread);
  111 static void     ithread_execute_handlers(struct proc *p, 
  112                     struct intr_event *ie);
  113 #ifdef INTR_FILTER
  114 static void     priv_ithread_execute_handler(struct proc *p, 
  115                     struct intr_handler *ih);
  116 #endif
  117 static void     ithread_loop(void *);
  118 static void     ithread_update(struct intr_thread *ithd);
  119 static void     start_softintr(void *);
  120 
  121 /* Map an interrupt type to an ithread priority. */
  122 u_char
  123 intr_priority(enum intr_type flags)
  124 {
  125         u_char pri;
  126 
  127         flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET |
  128             INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV);
  129         switch (flags) {
  130         case INTR_TYPE_TTY:
  131                 pri = PI_TTY;
  132                 break;
  133         case INTR_TYPE_BIO:
  134                 pri = PI_DISK;
  135                 break;
  136         case INTR_TYPE_NET:
  137                 pri = PI_NET;
  138                 break;
  139         case INTR_TYPE_CAM:
  140                 pri = PI_DISK;
  141                 break;
  142         case INTR_TYPE_AV:
  143                 pri = PI_AV;
  144                 break;
  145         case INTR_TYPE_CLK:
  146                 pri = PI_REALTIME;
  147                 break;
  148         case INTR_TYPE_MISC:
  149                 pri = PI_DULL;          /* don't care */
  150                 break;
  151         default:
  152                 /* We didn't specify an interrupt level. */
  153                 panic("intr_priority: no interrupt type in flags");
  154         }
  155 
  156         return pri;
  157 }
  158 
  159 /*
  160  * Update an ithread based on the associated intr_event.
  161  */
  162 static void
  163 ithread_update(struct intr_thread *ithd)
  164 {
  165         struct intr_event *ie;
  166         struct thread *td;
  167         u_char pri;
  168 
  169         ie = ithd->it_event;
  170         td = ithd->it_thread;
  171 
  172         /* Determine the overall priority of this event. */
  173         if (TAILQ_EMPTY(&ie->ie_handlers))
  174                 pri = PRI_MAX_ITHD;
  175         else
  176                 pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri;
  177 
  178         /* Update name and priority. */
  179         strlcpy(td->td_proc->p_comm, ie->ie_fullname,
  180             sizeof(td->td_proc->p_comm));
  181         thread_lock(td);
  182         sched_prio(td, pri);
  183         thread_unlock(td);
  184 }
  185 
  186 /*
  187  * Regenerate the full name of an interrupt event and update its priority.
  188  */
  189 static void
  190 intr_event_update(struct intr_event *ie)
  191 {
  192         struct intr_handler *ih;
  193         char *last;
  194         int missed, space;
  195 
  196         /* Start off with no entropy and just the name of the event. */
  197         mtx_assert(&ie->ie_lock, MA_OWNED);
  198         strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
  199         ie->ie_flags &= ~IE_ENTROPY;
  200         missed = 0;
  201         space = 1;
  202 
  203         /* Run through all the handlers updating values. */
  204         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  205                 if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <
  206                     sizeof(ie->ie_fullname)) {
  207                         strcat(ie->ie_fullname, " ");
  208                         strcat(ie->ie_fullname, ih->ih_name);
  209                         space = 0;
  210                 } else
  211                         missed++;
  212                 if (ih->ih_flags & IH_ENTROPY)
  213                         ie->ie_flags |= IE_ENTROPY;
  214         }
  215 
  216         /*
  217          * If the handler names were too long, add +'s to indicate missing
  218          * names. If we run out of room and still have +'s to add, change
  219          * the last character from a + to a *.
  220          */
  221         last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];
  222         while (missed-- > 0) {
  223                 if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {
  224                         if (*last == '+') {
  225                                 *last = '*';
  226                                 break;
  227                         } else
  228                                 *last = '+';
  229                 } else if (space) {
  230                         strcat(ie->ie_fullname, " +");
  231                         space = 0;
  232                 } else
  233                         strcat(ie->ie_fullname, "+");
  234         }
  235 
  236         /*
  237          * If this event has an ithread, update it's priority and
  238          * name.
  239          */
  240         if (ie->ie_thread != NULL)
  241                 ithread_update(ie->ie_thread);
  242         CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);
  243 }
  244 
  245 int
  246 intr_event_create(struct intr_event **event, void *source, int flags, int irq,
  247     void (*pre_ithread)(void *), void (*post_ithread)(void *),
  248     void (*post_filter)(void *), int (*assign_cpu)(void *, u_char),
  249     const char *fmt, ...)
  250 {
  251         struct intr_event *ie;
  252         va_list ap;
  253 
  254         /* The only valid flag during creation is IE_SOFT. */
  255         if ((flags & ~IE_SOFT) != 0)
  256                 return (EINVAL);
  257         ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);
  258         ie->ie_source = source;
  259         ie->ie_pre_ithread = pre_ithread;
  260         ie->ie_post_ithread = post_ithread;
  261         ie->ie_post_filter = post_filter;
  262         ie->ie_assign_cpu = assign_cpu;
  263         ie->ie_flags = flags;
  264         ie->ie_irq = irq;
  265         ie->ie_cpu = NOCPU;
  266         TAILQ_INIT(&ie->ie_handlers);
  267         mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
  268 
  269         va_start(ap, fmt);
  270         vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);
  271         va_end(ap);
  272         strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
  273         mtx_lock(&event_lock);
  274         TAILQ_INSERT_TAIL(&event_list, ie, ie_list);
  275         mtx_unlock(&event_lock);
  276         if (event != NULL)
  277                 *event = ie;
  278         CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);
  279         return (0);
  280 }
  281 
  282 /*
  283  * Bind an interrupt event to the specified CPU.  Note that not all
  284  * platforms support binding an interrupt to a CPU.  For those
  285  * platforms this request will fail.  For supported platforms, any
  286  * associated ithreads as well as the primary interrupt context will
  287  * be bound to the specificed CPU.  Using a cpu id of NOCPU unbinds
  288  * the interrupt event.
  289  */
  290 int
  291 intr_event_bind(struct intr_event *ie, u_char cpu)
  292 {
  293         cpuset_t mask;
  294         lwpid_t id;
  295         int error;
  296 
  297         /* Need a CPU to bind to. */
  298         if (cpu != NOCPU && CPU_ABSENT(cpu))
  299                 return (EINVAL);
  300 
  301         if (ie->ie_assign_cpu == NULL)
  302                 return (EOPNOTSUPP);
  303         /*
  304          * If we have any ithreads try to set their mask first since this
  305          * can fail.
  306          */
  307         mtx_lock(&ie->ie_lock);
  308         if (ie->ie_thread != NULL) {
  309                 CPU_ZERO(&mask);
  310                 if (cpu == NOCPU)
  311                         CPU_COPY(cpuset_root, &mask);
  312                 else
  313                         CPU_SET(cpu, &mask);
  314                 id = ie->ie_thread->it_thread->td_tid;
  315                 mtx_unlock(&ie->ie_lock);
  316                 error = cpuset_setthread(id, &mask);
  317                 if (error)
  318                         return (error);
  319         } else
  320                 mtx_unlock(&ie->ie_lock);
  321         error = ie->ie_assign_cpu(ie->ie_source, cpu);
  322         if (error)
  323                 return (error);
  324         mtx_lock(&ie->ie_lock);
  325         ie->ie_cpu = cpu;
  326         mtx_unlock(&ie->ie_lock);
  327 
  328         return (error);
  329 }
  330 
  331 static struct intr_event *
  332 intr_lookup(int irq)
  333 {
  334         struct intr_event *ie;
  335 
  336         mtx_lock(&event_lock);
  337         TAILQ_FOREACH(ie, &event_list, ie_list)
  338                 if (ie->ie_irq == irq &&
  339                     (ie->ie_flags & IE_SOFT) == 0 &&
  340                     TAILQ_FIRST(&ie->ie_handlers) != NULL)
  341                         break;
  342         mtx_unlock(&event_lock);
  343         return (ie);
  344 }
  345 
  346 int
  347 intr_setaffinity(int irq, void *m)
  348 {
  349         struct intr_event *ie;
  350         cpuset_t *mask;
  351         u_char cpu;
  352         int n;
  353 
  354         mask = m;
  355         cpu = NOCPU;
  356         /*
  357          * If we're setting all cpus we can unbind.  Otherwise make sure
  358          * only one cpu is in the set.
  359          */
  360         if (CPU_CMP(cpuset_root, mask)) {
  361                 for (n = 0; n < CPU_SETSIZE; n++) {
  362                         if (!CPU_ISSET(n, mask))
  363                                 continue;
  364                         if (cpu != NOCPU)
  365                                 return (EINVAL);
  366                         cpu = (u_char)n;
  367                 }
  368         }
  369         ie = intr_lookup(irq);
  370         if (ie == NULL)
  371                 return (ESRCH);
  372         return (intr_event_bind(ie, cpu));
  373 }
  374 
  375 int
  376 intr_getaffinity(int irq, void *m)
  377 {
  378         struct intr_event *ie;
  379         cpuset_t *mask;
  380 
  381         mask = m;
  382         ie = intr_lookup(irq);
  383         if (ie == NULL)
  384                 return (ESRCH);
  385         CPU_ZERO(mask);
  386         mtx_lock(&ie->ie_lock);
  387         if (ie->ie_cpu == NOCPU)
  388                 CPU_COPY(cpuset_root, mask);
  389         else
  390                 CPU_SET(ie->ie_cpu, mask);
  391         mtx_unlock(&ie->ie_lock);
  392         return (0);
  393 }
  394 
  395 int
  396 intr_event_destroy(struct intr_event *ie)
  397 {
  398 
  399         mtx_lock(&event_lock);
  400         mtx_lock(&ie->ie_lock);
  401         if (!TAILQ_EMPTY(&ie->ie_handlers)) {
  402                 mtx_unlock(&ie->ie_lock);
  403                 mtx_unlock(&event_lock);
  404                 return (EBUSY);
  405         }
  406         TAILQ_REMOVE(&event_list, ie, ie_list);
  407 #ifndef notyet
  408         if (ie->ie_thread != NULL) {
  409                 ithread_destroy(ie->ie_thread);
  410                 ie->ie_thread = NULL;
  411         }
  412 #endif
  413         mtx_unlock(&ie->ie_lock);
  414         mtx_unlock(&event_lock);
  415         mtx_destroy(&ie->ie_lock);
  416         free(ie, M_ITHREAD);
  417         return (0);
  418 }
  419 
  420 #ifndef INTR_FILTER
  421 static struct intr_thread *
  422 ithread_create(const char *name)
  423 {
  424         struct intr_thread *ithd;
  425         struct thread *td;
  426         struct proc *p;
  427         int error;
  428 
  429         ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
  430 
  431         error = kthread_create(ithread_loop, ithd, &p, RFSTOPPED | RFHIGHPID,
  432             0, "%s", name);
  433         if (error)
  434                 panic("kthread_create() failed with %d", error);
  435         td = FIRST_THREAD_IN_PROC(p);   /* XXXKSE */
  436         thread_lock(td);
  437         sched_class(td, PRI_ITHD);
  438         TD_SET_IWAIT(td);
  439         thread_unlock(td);
  440         td->td_pflags |= TDP_ITHREAD;
  441         ithd->it_thread = td;
  442         CTR2(KTR_INTR, "%s: created %s", __func__, name);
  443         return (ithd);
  444 }
  445 #else
  446 static struct intr_thread *
  447 ithread_create(const char *name, struct intr_handler *ih)
  448 {
  449         struct intr_thread *ithd;
  450         struct thread *td;
  451         struct proc *p;
  452         int error;
  453 
  454         ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
  455 
  456         error = kthread_create(ithread_loop, ih, &p, RFSTOPPED | RFHIGHPID,
  457             0, "%s", name);
  458         if (error)
  459                 panic("kthread_create() failed with %d", error);
  460         td = FIRST_THREAD_IN_PROC(p);   /* XXXKSE */
  461         thread_lock(td);
  462         sched_class(td, PRI_ITHD);
  463         TD_SET_IWAIT(td);
  464         thread_unlock(td);
  465         td->td_pflags |= TDP_ITHREAD;
  466         ithd->it_thread = td;
  467         CTR2(KTR_INTR, "%s: created %s", __func__, name);
  468         return (ithd);
  469 }
  470 #endif
  471 
  472 static void
  473 ithread_destroy(struct intr_thread *ithread)
  474 {
  475         struct thread *td;
  476 
  477         CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name);
  478         td = ithread->it_thread;
  479         thread_lock(td);
  480         ithread->it_flags |= IT_DEAD;
  481         if (TD_AWAITING_INTR(td)) {
  482                 TD_CLR_IWAIT(td);
  483                 sched_add(td, SRQ_INTR);
  484         }
  485         thread_unlock(td);
  486 }
  487 
  488 #ifndef INTR_FILTER
  489 int
  490 intr_event_add_handler(struct intr_event *ie, const char *name,
  491     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
  492     enum intr_type flags, void **cookiep)
  493 {
  494         struct intr_handler *ih, *temp_ih;
  495         struct intr_thread *it;
  496 
  497         if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
  498                 return (EINVAL);
  499 
  500         /* Allocate and populate an interrupt handler structure. */
  501         ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
  502         ih->ih_filter = filter;
  503         ih->ih_handler = handler;
  504         ih->ih_argument = arg;
  505         strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
  506         ih->ih_event = ie;
  507         ih->ih_pri = pri;
  508         if (flags & INTR_EXCL)
  509                 ih->ih_flags = IH_EXCLUSIVE;
  510         if (flags & INTR_MPSAFE)
  511                 ih->ih_flags |= IH_MPSAFE;
  512         if (flags & INTR_ENTROPY)
  513                 ih->ih_flags |= IH_ENTROPY;
  514 
  515         /* We can only have one exclusive handler in a event. */
  516         mtx_lock(&ie->ie_lock);
  517         if (!TAILQ_EMPTY(&ie->ie_handlers)) {
  518                 if ((flags & INTR_EXCL) ||
  519                     (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
  520                         mtx_unlock(&ie->ie_lock);
  521                         free(ih, M_ITHREAD);
  522                         return (EINVAL);
  523                 }
  524         }
  525 
  526         /* Add the new handler to the event in priority order. */
  527         TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
  528                 if (temp_ih->ih_pri > ih->ih_pri)
  529                         break;
  530         }
  531         if (temp_ih == NULL)
  532                 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
  533         else
  534                 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
  535         intr_event_update(ie);
  536 
  537         /* Create a thread if we need one. */
  538         while (ie->ie_thread == NULL && handler != NULL) {
  539                 if (ie->ie_flags & IE_ADDING_THREAD)
  540                         msleep(ie, &ie->ie_lock, 0, "ithread", 0);
  541                 else {
  542                         ie->ie_flags |= IE_ADDING_THREAD;
  543                         mtx_unlock(&ie->ie_lock);
  544                         it = ithread_create("intr: newborn");
  545                         mtx_lock(&ie->ie_lock);
  546                         ie->ie_flags &= ~IE_ADDING_THREAD;
  547                         ie->ie_thread = it;
  548                         it->it_event = ie;
  549                         ithread_update(it);
  550                         wakeup(ie);
  551                 }
  552         }
  553         CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
  554             ie->ie_name);
  555         mtx_unlock(&ie->ie_lock);
  556 
  557         if (cookiep != NULL)
  558                 *cookiep = ih;
  559         return (0);
  560 }
  561 #else
  562 int
  563 intr_event_add_handler(struct intr_event *ie, const char *name,
  564     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
  565     enum intr_type flags, void **cookiep)
  566 {
  567         struct intr_handler *ih, *temp_ih;
  568         struct intr_thread *it;
  569 
  570         if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
  571                 return (EINVAL);
  572 
  573         /* Allocate and populate an interrupt handler structure. */
  574         ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
  575         ih->ih_filter = filter;
  576         ih->ih_handler = handler;
  577         ih->ih_argument = arg;
  578         strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
  579         ih->ih_event = ie;
  580         ih->ih_pri = pri;
  581         if (flags & INTR_EXCL)
  582                 ih->ih_flags = IH_EXCLUSIVE;
  583         if (flags & INTR_MPSAFE)
  584                 ih->ih_flags |= IH_MPSAFE;
  585         if (flags & INTR_ENTROPY)
  586                 ih->ih_flags |= IH_ENTROPY;
  587 
  588         /* We can only have one exclusive handler in a event. */
  589         mtx_lock(&ie->ie_lock);
  590         if (!TAILQ_EMPTY(&ie->ie_handlers)) {
  591                 if ((flags & INTR_EXCL) ||
  592                     (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
  593                         mtx_unlock(&ie->ie_lock);
  594                         free(ih, M_ITHREAD);
  595                         return (EINVAL);
  596                 }
  597         }
  598 
  599         /* Add the new handler to the event in priority order. */
  600         TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
  601                 if (temp_ih->ih_pri > ih->ih_pri)
  602                         break;
  603         }
  604         if (temp_ih == NULL)
  605                 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
  606         else
  607                 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
  608         intr_event_update(ie);
  609 
  610         /* For filtered handlers, create a private ithread to run on. */
  611         if (filter != NULL && handler != NULL) { 
  612                 mtx_unlock(&ie->ie_lock);
  613                 it = ithread_create("intr: newborn", ih);               
  614                 mtx_lock(&ie->ie_lock);
  615                 it->it_event = ie; 
  616                 ih->ih_thread = it;
  617                 ithread_update(it); // XXX - do we really need this?!?!?
  618         } else { /* Create the global per-event thread if we need one. */
  619                 while (ie->ie_thread == NULL && handler != NULL) {
  620                         if (ie->ie_flags & IE_ADDING_THREAD)
  621                                 msleep(ie, &ie->ie_lock, 0, "ithread", 0);
  622                         else {
  623                                 ie->ie_flags |= IE_ADDING_THREAD;
  624                                 mtx_unlock(&ie->ie_lock);
  625                                 it = ithread_create("intr: newborn", ih);
  626                                 mtx_lock(&ie->ie_lock);
  627                                 ie->ie_flags &= ~IE_ADDING_THREAD;
  628                                 ie->ie_thread = it;
  629                                 it->it_event = ie;
  630                                 ithread_update(it);
  631                                 wakeup(ie);
  632                         }
  633                 }
  634         }
  635         CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
  636             ie->ie_name);
  637         mtx_unlock(&ie->ie_lock);
  638 
  639         if (cookiep != NULL)
  640                 *cookiep = ih;
  641         return (0);
  642 }
  643 #endif
  644 
  645 /*
  646  * Append a description preceded by a ':' to the name of the specified
  647  * interrupt handler.
  648  */
  649 int
  650 intr_event_describe_handler(struct intr_event *ie, void *cookie,
  651     const char *descr)
  652 {
  653         struct intr_handler *ih;
  654         size_t space;
  655         char *start;
  656 
  657         mtx_lock(&ie->ie_lock);
  658 #ifdef INVARIANTS
  659         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  660                 if (ih == cookie)
  661                         break;
  662         }
  663         if (ih == NULL) {
  664                 mtx_unlock(&ie->ie_lock);
  665                 panic("handler %p not found in interrupt event %p", cookie, ie);
  666         }
  667 #endif
  668         ih = cookie;
  669 
  670         /*
  671          * Look for an existing description by checking for an
  672          * existing ":".  This assumes device names do not include
  673          * colons.  If one is found, prepare to insert the new
  674          * description at that point.  If one is not found, find the
  675          * end of the name to use as the insertion point.
  676          */
  677         start = index(ih->ih_name, ':');
  678         if (start == NULL)
  679                 start = index(ih->ih_name, 0);
  680 
  681         /*
  682          * See if there is enough remaining room in the string for the
  683          * description + ":".  The "- 1" leaves room for the trailing
  684          * '\0'.  The "+ 1" accounts for the colon.
  685          */
  686         space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1;
  687         if (strlen(descr) + 1 > space) {
  688                 mtx_unlock(&ie->ie_lock);
  689                 return (ENOSPC);
  690         }
  691 
  692         /* Append a colon followed by the description. */
  693         *start = ':';
  694         strcpy(start + 1, descr);
  695         intr_event_update(ie);
  696         mtx_unlock(&ie->ie_lock);
  697         return (0);
  698 }
  699 
  700 /*
  701  * Return the ie_source field from the intr_event an intr_handler is
  702  * associated with.
  703  */
  704 void *
  705 intr_handler_source(void *cookie)
  706 {
  707         struct intr_handler *ih;
  708         struct intr_event *ie;
  709 
  710         ih = (struct intr_handler *)cookie;
  711         if (ih == NULL)
  712                 return (NULL);
  713         ie = ih->ih_event;
  714         KASSERT(ie != NULL,
  715             ("interrupt handler \"%s\" has a NULL interrupt event",
  716             ih->ih_name));
  717         return (ie->ie_source);
  718 }
  719 
  720 #ifndef INTR_FILTER
  721 int
  722 intr_event_remove_handler(void *cookie)
  723 {
  724         struct intr_handler *handler = (struct intr_handler *)cookie;
  725         struct intr_event *ie;
  726 #ifdef INVARIANTS
  727         struct intr_handler *ih;
  728 #endif
  729 #ifdef notyet
  730         int dead;
  731 #endif
  732 
  733         if (handler == NULL)
  734                 return (EINVAL);
  735         ie = handler->ih_event;
  736         KASSERT(ie != NULL,
  737             ("interrupt handler \"%s\" has a NULL interrupt event",
  738             handler->ih_name));
  739         mtx_lock(&ie->ie_lock);
  740         CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
  741             ie->ie_name);
  742 #ifdef INVARIANTS
  743         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
  744                 if (ih == handler)
  745                         goto ok;
  746         mtx_unlock(&ie->ie_lock);
  747         panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
  748             ih->ih_name, ie->ie_name);
  749 ok:
  750 #endif
  751         /*
  752          * If there is no ithread, then just remove the handler and return.
  753          * XXX: Note that an INTR_FAST handler might be running on another
  754          * CPU!
  755          */
  756         if (ie->ie_thread == NULL) {
  757                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  758                 mtx_unlock(&ie->ie_lock);
  759                 free(handler, M_ITHREAD);
  760                 return (0);
  761         }
  762 
  763         /*
  764          * If the interrupt thread is already running, then just mark this
  765          * handler as being dead and let the ithread do the actual removal.
  766          *
  767          * During a cold boot while cold is set, msleep() does not sleep,
  768          * so we have to remove the handler here rather than letting the
  769          * thread do it.
  770          */
  771         thread_lock(ie->ie_thread->it_thread);
  772         if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) {
  773                 handler->ih_flags |= IH_DEAD;
  774 
  775                 /*
  776                  * Ensure that the thread will process the handler list
  777                  * again and remove this handler if it has already passed
  778                  * it on the list.
  779                  */
  780                 ie->ie_thread->it_need = 1;
  781         } else
  782                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  783         thread_unlock(ie->ie_thread->it_thread);
  784         while (handler->ih_flags & IH_DEAD)
  785                 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
  786         intr_event_update(ie);
  787 #ifdef notyet
  788         /*
  789          * XXX: This could be bad in the case of ppbus(8).  Also, I think
  790          * this could lead to races of stale data when servicing an
  791          * interrupt.
  792          */
  793         dead = 1;
  794         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  795                 if (!(ih->ih_flags & IH_FAST)) {
  796                         dead = 0;
  797                         break;
  798                 }
  799         }
  800         if (dead) {
  801                 ithread_destroy(ie->ie_thread);
  802                 ie->ie_thread = NULL;
  803         }
  804 #endif
  805         mtx_unlock(&ie->ie_lock);
  806         free(handler, M_ITHREAD);
  807         return (0);
  808 }
  809 
  810 static int
  811 intr_event_schedule_thread(struct intr_event *ie)
  812 {
  813         struct intr_entropy entropy;
  814         struct intr_thread *it;
  815         struct thread *td;
  816         struct thread *ctd;
  817         struct proc *p;
  818 
  819         /*
  820          * If no ithread or no handlers, then we have a stray interrupt.
  821          */
  822         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) ||
  823             ie->ie_thread == NULL)
  824                 return (EINVAL);
  825 
  826         ctd = curthread;
  827         it = ie->ie_thread;
  828         td = it->it_thread;
  829         p = td->td_proc;
  830 
  831         /*
  832          * If any of the handlers for this ithread claim to be good
  833          * sources of entropy, then gather some.
  834          */
  835         if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
  836                 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
  837                     p->p_pid, p->p_comm);
  838                 entropy.event = (uintptr_t)ie;
  839                 entropy.td = ctd;
  840                 random_harvest(&entropy, sizeof(entropy), 2, 0,
  841                     RANDOM_INTERRUPT);
  842         }
  843 
  844         KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
  845 
  846         /*
  847          * Set it_need to tell the thread to keep running if it is already
  848          * running.  Then, lock the thread and see if we actually need to
  849          * put it on the runqueue.
  850          */
  851         it->it_need = 1;
  852         thread_lock(td);
  853         if (TD_AWAITING_INTR(td)) {
  854                 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
  855                     p->p_comm);
  856                 TD_CLR_IWAIT(td);
  857                 sched_add(td, SRQ_INTR);
  858         } else {
  859                 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
  860                     __func__, p->p_pid, p->p_comm, it->it_need, td->td_state);
  861         }
  862         thread_unlock(td);
  863 
  864         return (0);
  865 }
  866 #else
  867 int
  868 intr_event_remove_handler(void *cookie)
  869 {
  870         struct intr_handler *handler = (struct intr_handler *)cookie;
  871         struct intr_event *ie;
  872         struct intr_thread *it;
  873 #ifdef INVARIANTS
  874         struct intr_handler *ih;
  875 #endif
  876 #ifdef notyet
  877         int dead;
  878 #endif
  879 
  880         if (handler == NULL)
  881                 return (EINVAL);
  882         ie = handler->ih_event;
  883         KASSERT(ie != NULL,
  884             ("interrupt handler \"%s\" has a NULL interrupt event",
  885             handler->ih_name));
  886         mtx_lock(&ie->ie_lock);
  887         CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
  888             ie->ie_name);
  889 #ifdef INVARIANTS
  890         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
  891                 if (ih == handler)
  892                         goto ok;
  893         mtx_unlock(&ie->ie_lock);
  894         panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
  895             ih->ih_name, ie->ie_name);
  896 ok:
  897 #endif
  898         /*
  899          * If there are no ithreads (per event and per handler), then
  900          * just remove the handler and return.  
  901          * XXX: Note that an INTR_FAST handler might be running on another CPU!
  902          */
  903         if (ie->ie_thread == NULL && handler->ih_thread == NULL) {
  904                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  905                 mtx_unlock(&ie->ie_lock);
  906                 free(handler, M_ITHREAD);
  907                 return (0);
  908         }
  909 
  910         /* Private or global ithread? */
  911         it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread;
  912         /*
  913          * If the interrupt thread is already running, then just mark this
  914          * handler as being dead and let the ithread do the actual removal.
  915          *
  916          * During a cold boot while cold is set, msleep() does not sleep,
  917          * so we have to remove the handler here rather than letting the
  918          * thread do it.
  919          */
  920         thread_lock(it->it_thread);
  921         if (!TD_AWAITING_INTR(it->it_thread) && !cold) {
  922                 handler->ih_flags |= IH_DEAD;
  923 
  924                 /*
  925                  * Ensure that the thread will process the handler list
  926                  * again and remove this handler if it has already passed
  927                  * it on the list.
  928                  */
  929                 it->it_need = 1;
  930         } else
  931                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  932         thread_unlock(it->it_thread);
  933         while (handler->ih_flags & IH_DEAD)
  934                 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
  935         /* 
  936          * At this point, the handler has been disconnected from the event,
  937          * so we can kill the private ithread if any.
  938          */
  939         if (handler->ih_thread) {
  940                 ithread_destroy(handler->ih_thread);
  941                 handler->ih_thread = NULL;
  942         }
  943         intr_event_update(ie);
  944 #ifdef notyet
  945         /*
  946          * XXX: This could be bad in the case of ppbus(8).  Also, I think
  947          * this could lead to races of stale data when servicing an
  948          * interrupt.
  949          */
  950         dead = 1;
  951         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  952                 if (handler != NULL) {
  953                         dead = 0;
  954                         break;
  955                 }
  956         }
  957         if (dead) {
  958                 ithread_destroy(ie->ie_thread);
  959                 ie->ie_thread = NULL;
  960         }
  961 #endif
  962         mtx_unlock(&ie->ie_lock);
  963         free(handler, M_ITHREAD);
  964         return (0);
  965 }
  966 
  967 static int
  968 intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it)
  969 {
  970         struct intr_entropy entropy;
  971         struct thread *td;
  972         struct thread *ctd;
  973         struct proc *p;
  974 
  975         /*
  976          * If no ithread or no handlers, then we have a stray interrupt.
  977          */
  978         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL)
  979                 return (EINVAL);
  980 
  981         ctd = curthread;
  982         td = it->it_thread;
  983         p = td->td_proc;
  984 
  985         /*
  986          * If any of the handlers for this ithread claim to be good
  987          * sources of entropy, then gather some.
  988          */
  989         if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
  990                 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
  991                     p->p_pid, p->p_comm);
  992                 entropy.event = (uintptr_t)ie;
  993                 entropy.td = ctd;
  994                 random_harvest(&entropy, sizeof(entropy), 2, 0,
  995                     RANDOM_INTERRUPT);
  996         }
  997 
  998         KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
  999 
 1000         /*
 1001          * Set it_need to tell the thread to keep running if it is already
 1002          * running.  Then, lock the thread and see if we actually need to
 1003          * put it on the runqueue.
 1004          */
 1005         it->it_need = 1;
 1006         thread_lock(td);
 1007         if (TD_AWAITING_INTR(td)) {
 1008                 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
 1009                     p->p_comm);
 1010                 TD_CLR_IWAIT(td);
 1011                 sched_add(td, SRQ_INTR);
 1012         } else {
 1013                 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
 1014                     __func__, p->p_pid, p->p_comm, it->it_need, td->td_state);
 1015         }
 1016         thread_unlock(td);
 1017 
 1018         return (0);
 1019 }
 1020 #endif
 1021 
 1022 /*
 1023  * Add a software interrupt handler to a specified event.  If a given event
 1024  * is not specified, then a new event is created.
 1025  */
 1026 int
 1027 swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
 1028             void *arg, int pri, enum intr_type flags, void **cookiep)
 1029 {
 1030         struct intr_event *ie;
 1031         int error;
 1032 
 1033         if (flags & INTR_ENTROPY)
 1034                 return (EINVAL);
 1035 
 1036         ie = (eventp != NULL) ? *eventp : NULL;
 1037 
 1038         if (ie != NULL) {
 1039                 if (!(ie->ie_flags & IE_SOFT))
 1040                         return (EINVAL);
 1041         } else {
 1042                 error = intr_event_create(&ie, NULL, IE_SOFT, 0,
 1043                     NULL, NULL, NULL, NULL, "swi%d:", pri);
 1044                 if (error)
 1045                         return (error);
 1046                 if (eventp != NULL)
 1047                         *eventp = ie;
 1048         }
 1049         return (intr_event_add_handler(ie, name, NULL, handler, arg,
 1050                     PI_SWI(pri), flags, cookiep));
 1051                     /* XXKSE.. think of a better way to get separate queues */
 1052 }
 1053 
 1054 /*
 1055  * Schedule a software interrupt thread.
 1056  */
 1057 void
 1058 swi_sched(void *cookie, int flags)
 1059 {
 1060         struct intr_handler *ih = (struct intr_handler *)cookie;
 1061         struct intr_event *ie = ih->ih_event;
 1062         int error;
 1063 
 1064         CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
 1065             ih->ih_need);
 1066 
 1067         /*
 1068          * Set ih_need for this handler so that if the ithread is already
 1069          * running it will execute this handler on the next pass.  Otherwise,
 1070          * it will execute it the next time it runs.
 1071          */
 1072         atomic_store_rel_int(&ih->ih_need, 1);
 1073 
 1074         if (!(flags & SWI_DELAY)) {
 1075                 PCPU_INC(cnt.v_soft);
 1076 #ifdef INTR_FILTER
 1077                 error = intr_event_schedule_thread(ie, ie->ie_thread);
 1078 #else
 1079                 error = intr_event_schedule_thread(ie);
 1080 #endif
 1081                 KASSERT(error == 0, ("stray software interrupt"));
 1082         }
 1083 }
 1084 
 1085 /*
 1086  * Remove a software interrupt handler.  Currently this code does not
 1087  * remove the associated interrupt event if it becomes empty.  Calling code
 1088  * may do so manually via intr_event_destroy(), but that's not really
 1089  * an optimal interface.
 1090  */
 1091 int
 1092 swi_remove(void *cookie)
 1093 {
 1094 
 1095         return (intr_event_remove_handler(cookie));
 1096 }
 1097 
 1098 #ifdef INTR_FILTER
 1099 static void
 1100 priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih)
 1101 {
 1102         struct intr_event *ie;
 1103 
 1104         ie = ih->ih_event;
 1105         /*
 1106          * If this handler is marked for death, remove it from
 1107          * the list of handlers and wake up the sleeper.
 1108          */
 1109         if (ih->ih_flags & IH_DEAD) {
 1110                 mtx_lock(&ie->ie_lock);
 1111                 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
 1112                 ih->ih_flags &= ~IH_DEAD;
 1113                 wakeup(ih);
 1114                 mtx_unlock(&ie->ie_lock);
 1115                 return;
 1116         }
 1117         
 1118         /* Execute this handler. */
 1119         CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
 1120              __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument,
 1121              ih->ih_name, ih->ih_flags);
 1122         
 1123         if (!(ih->ih_flags & IH_MPSAFE))
 1124                 mtx_lock(&Giant);
 1125         ih->ih_handler(ih->ih_argument);
 1126         if (!(ih->ih_flags & IH_MPSAFE))
 1127                 mtx_unlock(&Giant);
 1128 }
 1129 #endif
 1130 
 1131 static void
 1132 ithread_execute_handlers(struct proc *p, struct intr_event *ie)
 1133 {
 1134         struct intr_handler *ih, *ihn;
 1135 
 1136         /* Interrupt handlers should not sleep. */
 1137         if (!(ie->ie_flags & IE_SOFT))
 1138                 THREAD_NO_SLEEPING();
 1139         TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {
 1140 
 1141                 /*
 1142                  * If this handler is marked for death, remove it from
 1143                  * the list of handlers and wake up the sleeper.
 1144                  */
 1145                 if (ih->ih_flags & IH_DEAD) {
 1146                         mtx_lock(&ie->ie_lock);
 1147                         TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
 1148                         ih->ih_flags &= ~IH_DEAD;
 1149                         wakeup(ih);
 1150                         mtx_unlock(&ie->ie_lock);
 1151                         continue;
 1152                 }
 1153 
 1154                 /* Skip filter only handlers */
 1155                 if (ih->ih_handler == NULL)
 1156                         continue;
 1157 
 1158                 /*
 1159                  * For software interrupt threads, we only execute
 1160                  * handlers that have their need flag set.  Hardware
 1161                  * interrupt threads always invoke all of their handlers.
 1162                  */
 1163                 if (ie->ie_flags & IE_SOFT) {
 1164                         if (!ih->ih_need)
 1165                                 continue;
 1166                         else
 1167                                 atomic_store_rel_int(&ih->ih_need, 0);
 1168                 }
 1169 
 1170                 /* Execute this handler. */
 1171                 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
 1172                     __func__, p->p_pid, (void *)ih->ih_handler, 
 1173                     ih->ih_argument, ih->ih_name, ih->ih_flags);
 1174 
 1175                 if (!(ih->ih_flags & IH_MPSAFE))
 1176                         mtx_lock(&Giant);
 1177                 ih->ih_handler(ih->ih_argument);
 1178                 if (!(ih->ih_flags & IH_MPSAFE))
 1179                         mtx_unlock(&Giant);
 1180         }
 1181         if (!(ie->ie_flags & IE_SOFT))
 1182                 THREAD_SLEEPING_OK();
 1183 
 1184         /*
 1185          * Interrupt storm handling:
 1186          *
 1187          * If this interrupt source is currently storming, then throttle
 1188          * it to only fire the handler once  per clock tick.
 1189          *
 1190          * If this interrupt source is not currently storming, but the
 1191          * number of back to back interrupts exceeds the storm threshold,
 1192          * then enter storming mode.
 1193          */
 1194         if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold &&
 1195             !(ie->ie_flags & IE_SOFT)) {
 1196                 /* Report the message only once every second. */
 1197                 if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {
 1198                         printf(
 1199         "interrupt storm detected on \"%s\"; throttling interrupt source\n",
 1200                             ie->ie_name);
 1201                 }
 1202                 pause("istorm", 1);
 1203         } else
 1204                 ie->ie_count++;
 1205 
 1206         /*
 1207          * Now that all the handlers have had a chance to run, reenable
 1208          * the interrupt source.
 1209          */
 1210         if (ie->ie_post_ithread != NULL)
 1211                 ie->ie_post_ithread(ie->ie_source);
 1212 }
 1213 
 1214 #ifndef INTR_FILTER
 1215 /*
 1216  * This is the main code for interrupt threads.
 1217  */
 1218 static void
 1219 ithread_loop(void *arg)
 1220 {
 1221         struct intr_thread *ithd;
 1222         struct intr_event *ie;
 1223         struct thread *td;
 1224         struct proc *p;
 1225 
 1226         td = curthread;
 1227         p = td->td_proc;
 1228         ithd = (struct intr_thread *)arg;
 1229         KASSERT(ithd->it_thread == td,
 1230             ("%s: ithread and proc linkage out of sync", __func__));
 1231         ie = ithd->it_event;
 1232         ie->ie_count = 0;
 1233 
 1234         /*
 1235          * As long as we have interrupts outstanding, go through the
 1236          * list of handlers, giving each one a go at it.
 1237          */
 1238         for (;;) {
 1239                 /*
 1240                  * If we are an orphaned thread, then just die.
 1241                  */
 1242                 if (ithd->it_flags & IT_DEAD) {
 1243                         CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
 1244                             p->p_pid, p->p_comm);
 1245                         free(ithd, M_ITHREAD);
 1246                         kthread_exit(0);
 1247                 }
 1248 
 1249                 /*
 1250                  * Service interrupts.  If another interrupt arrives while
 1251                  * we are running, it will set it_need to note that we
 1252                  * should make another pass.
 1253                  */
 1254                 while (ithd->it_need) {
 1255                         /*
 1256                          * This might need a full read and write barrier
 1257                          * to make sure that this write posts before any
 1258                          * of the memory or device accesses in the
 1259                          * handlers.
 1260                          */
 1261                         atomic_store_rel_int(&ithd->it_need, 0);
 1262                         ithread_execute_handlers(p, ie);
 1263                 }
 1264                 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
 1265                 mtx_assert(&Giant, MA_NOTOWNED);
 1266 
 1267                 /*
 1268                  * Processed all our interrupts.  Now get the sched
 1269                  * lock.  This may take a while and it_need may get
 1270                  * set again, so we have to check it again.
 1271                  */
 1272                 thread_lock(td);
 1273                 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) {
 1274                         TD_SET_IWAIT(td);
 1275                         ie->ie_count = 0;
 1276                         mi_switch(SW_VOL, NULL);
 1277                 }
 1278                 thread_unlock(td);
 1279         }
 1280 }
 1281 
 1282 /*
 1283  * Main interrupt handling body.
 1284  *
 1285  * Input:
 1286  * o ie:                        the event connected to this interrupt.
 1287  * o frame:                     some archs (i.e. i386) pass a frame to some.
 1288  *                              handlers as their main argument.
 1289  * Return value:
 1290  * o 0:                         everything ok.
 1291  * o EINVAL:                    stray interrupt.
 1292  */
 1293 int
 1294 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
 1295 {
 1296         struct intr_handler *ih;
 1297         struct thread *td;
 1298         int error, ret, thread;
 1299 
 1300         td = curthread;
 1301 
 1302         /* An interrupt with no event or handlers is a stray interrupt. */
 1303         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
 1304                 return (EINVAL);
 1305 
 1306         /*
 1307          * Execute fast interrupt handlers directly.
 1308          * To support clock handlers, if a handler registers
 1309          * with a NULL argument, then we pass it a pointer to
 1310          * a trapframe as its argument.
 1311          */
 1312         td->td_intr_nesting_level++;
 1313         thread = 0;
 1314         ret = 0;
 1315         critical_enter();
 1316         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
 1317                 if (ih->ih_filter == NULL) {
 1318                         thread = 1;
 1319                         continue;
 1320                 }
 1321                 CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__,
 1322                     ih->ih_filter, ih->ih_argument == NULL ? frame :
 1323                     ih->ih_argument, ih->ih_name);
 1324                 if (ih->ih_argument == NULL)
 1325                         ret = ih->ih_filter(frame);
 1326                 else
 1327                         ret = ih->ih_filter(ih->ih_argument);
 1328                 KASSERT(ret == FILTER_STRAY ||
 1329                     ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
 1330                     (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
 1331                     ("%s: incorrect return value %#x from %s", __func__, ret,
 1332                     ih->ih_name));
 1333 
 1334                 /* 
 1335                  * Wrapper handler special handling:
 1336                  *
 1337                  * in some particular cases (like pccard and pccbb), 
 1338                  * the _real_ device handler is wrapped in a couple of
 1339                  * functions - a filter wrapper and an ithread wrapper.
 1340                  * In this case (and just in this case), the filter wrapper 
 1341                  * could ask the system to schedule the ithread and mask
 1342                  * the interrupt source if the wrapped handler is composed
 1343                  * of just an ithread handler.
 1344                  *
 1345                  * TODO: write a generic wrapper to avoid people rolling 
 1346                  * their own
 1347                  */
 1348                 if (!thread) {
 1349                         if (ret == FILTER_SCHEDULE_THREAD)
 1350                                 thread = 1;
 1351                 }
 1352         }
 1353 
 1354         if (thread) {
 1355                 if (ie->ie_pre_ithread != NULL)
 1356                         ie->ie_pre_ithread(ie->ie_source);
 1357         } else {
 1358                 if (ie->ie_post_filter != NULL)
 1359                         ie->ie_post_filter(ie->ie_source);
 1360         }
 1361         
 1362         /* Schedule the ithread if needed. */
 1363         if (thread) {
 1364                 error = intr_event_schedule_thread(ie);
 1365                 KASSERT(error == 0, ("bad stray interrupt"));
 1366         }
 1367         critical_exit();
 1368         td->td_intr_nesting_level--;
 1369         return (0);
 1370 }
 1371 #else
 1372 /*
 1373  * This is the main code for interrupt threads.
 1374  */
 1375 static void
 1376 ithread_loop(void *arg)
 1377 {
 1378         struct intr_thread *ithd;
 1379         struct intr_handler *ih;
 1380         struct intr_event *ie;
 1381         struct thread *td;
 1382         struct proc *p;
 1383         int priv;
 1384 
 1385         td = curthread;
 1386         p = td->td_proc;
 1387         ih = (struct intr_handler *)arg;
 1388         priv = (ih->ih_thread != NULL) ? 1 : 0;
 1389         ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread;
 1390         KASSERT(ithd->it_thread == td,
 1391             ("%s: ithread and proc linkage out of sync", __func__));
 1392         ie = ithd->it_event;
 1393         ie->ie_count = 0;
 1394 
 1395         /*
 1396          * As long as we have interrupts outstanding, go through the
 1397          * list of handlers, giving each one a go at it.
 1398          */
 1399         for (;;) {
 1400                 /*
 1401                  * If we are an orphaned thread, then just die.
 1402                  */
 1403                 if (ithd->it_flags & IT_DEAD) {
 1404                         CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
 1405                             p->p_pid, p->p_comm);
 1406                         free(ithd, M_ITHREAD);
 1407                         kthread_exit(0);
 1408                 }
 1409 
 1410                 /*
 1411                  * Service interrupts.  If another interrupt arrives while
 1412                  * we are running, it will set it_need to note that we
 1413                  * should make another pass.
 1414                  */
 1415                 while (ithd->it_need) {
 1416                         /*
 1417                          * This might need a full read and write barrier
 1418                          * to make sure that this write posts before any
 1419                          * of the memory or device accesses in the
 1420                          * handlers.
 1421                          */
 1422                         atomic_store_rel_int(&ithd->it_need, 0);
 1423                         if (priv)
 1424                                 priv_ithread_execute_handler(p, ih);
 1425                         else 
 1426                                 ithread_execute_handlers(p, ie);
 1427                 }
 1428                 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
 1429                 mtx_assert(&Giant, MA_NOTOWNED);
 1430 
 1431                 /*
 1432                  * Processed all our interrupts.  Now get the sched
 1433                  * lock.  This may take a while and it_need may get
 1434                  * set again, so we have to check it again.
 1435                  */
 1436                 thread_lock(td);
 1437                 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) {
 1438                         TD_SET_IWAIT(td);
 1439                         ie->ie_count = 0;
 1440                         mi_switch(SW_VOL, NULL);
 1441                 }
 1442                 thread_unlock(td);
 1443         }
 1444 }
 1445 
 1446 /* 
 1447  * Main loop for interrupt filter.
 1448  *
 1449  * Some architectures (i386, amd64 and arm) require the optional frame 
 1450  * parameter, and use it as the main argument for fast handler execution
 1451  * when ih_argument == NULL.
 1452  *
 1453  * Return value:
 1454  * o FILTER_STRAY:              No filter recognized the event, and no
 1455  *                              filter-less handler is registered on this 
 1456  *                              line.
 1457  * o FILTER_HANDLED:            A filter claimed the event and served it.
 1458  * o FILTER_SCHEDULE_THREAD:    No filter claimed the event, but there's at
 1459  *                              least one filter-less handler on this line.
 1460  * o FILTER_HANDLED | 
 1461  *   FILTER_SCHEDULE_THREAD:    A filter claimed the event, and asked for
 1462  *                              scheduling the per-handler ithread.
 1463  *
 1464  * In case an ithread has to be scheduled, in *ithd there will be a 
 1465  * pointer to a struct intr_thread containing the thread to be
 1466  * scheduled.
 1467  */
 1468 
 1469 static int
 1470 intr_filter_loop(struct intr_event *ie, struct trapframe *frame, 
 1471                  struct intr_thread **ithd) 
 1472 {
 1473         struct intr_handler *ih;
 1474         void *arg;
 1475         int ret, thread_only;
 1476 
 1477         ret = 0;
 1478         thread_only = 0;
 1479         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
 1480                 /*
 1481                  * Execute fast interrupt handlers directly.
 1482                  * To support clock handlers, if a handler registers
 1483                  * with a NULL argument, then we pass it a pointer to
 1484                  * a trapframe as its argument.
 1485                  */
 1486                 arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument);
 1487                 
 1488                 CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__,
 1489                      ih->ih_filter, ih->ih_handler, arg, ih->ih_name);
 1490 
 1491                 if (ih->ih_filter != NULL)
 1492                         ret = ih->ih_filter(arg);
 1493                 else {
 1494                         thread_only = 1;
 1495                         continue;
 1496                 }
 1497                 KASSERT(ret == FILTER_STRAY ||
 1498                     ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
 1499                     (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
 1500                     ("%s: incorrect return value %#x from %s", __func__, ret,
 1501                     ih->ih_name));
 1502                 if (ret & FILTER_STRAY)
 1503                         continue;
 1504                 else { 
 1505                         *ithd = ih->ih_thread;
 1506                         return (ret);
 1507                 }
 1508         }
 1509 
 1510         /*
 1511          * No filters handled the interrupt and we have at least
 1512          * one handler without a filter.  In this case, we schedule
 1513          * all of the filter-less handlers to run in the ithread.
 1514          */     
 1515         if (thread_only) {
 1516                 *ithd = ie->ie_thread;
 1517                 return (FILTER_SCHEDULE_THREAD);
 1518         }
 1519         return (FILTER_STRAY);
 1520 }
 1521 
 1522 /*
 1523  * Main interrupt handling body.
 1524  *
 1525  * Input:
 1526  * o ie:                        the event connected to this interrupt.
 1527  * o frame:                     some archs (i.e. i386) pass a frame to some.
 1528  *                              handlers as their main argument.
 1529  * Return value:
 1530  * o 0:                         everything ok.
 1531  * o EINVAL:                    stray interrupt.
 1532  */
 1533 int
 1534 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
 1535 {
 1536         struct intr_thread *ithd;
 1537         struct thread *td;
 1538         int thread;
 1539 
 1540         ithd = NULL;
 1541         td = curthread;
 1542 
 1543         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
 1544                 return (EINVAL);
 1545 
 1546         td->td_intr_nesting_level++;
 1547         thread = 0;
 1548         critical_enter();
 1549         thread = intr_filter_loop(ie, frame, &ithd);    
 1550         if (thread & FILTER_HANDLED) {
 1551                 if (ie->ie_post_filter != NULL)
 1552                         ie->ie_post_filter(ie->ie_source);
 1553         } else {
 1554                 if (ie->ie_pre_ithread != NULL)
 1555                         ie->ie_pre_ithread(ie->ie_source);
 1556         }
 1557         critical_exit();
 1558         
 1559         /* Interrupt storm logic */
 1560         if (thread & FILTER_STRAY) {
 1561                 ie->ie_count++;
 1562                 if (ie->ie_count < intr_storm_threshold)
 1563                         printf("Interrupt stray detection not present\n");
 1564         }
 1565 
 1566         /* Schedule an ithread if needed. */
 1567         if (thread & FILTER_SCHEDULE_THREAD) {
 1568                 if (intr_event_schedule_thread(ie, ithd) != 0)
 1569                         panic("%s: impossible stray interrupt", __func__);
 1570         }
 1571         td->td_intr_nesting_level--;
 1572         return (0);
 1573 }
 1574 #endif
 1575 
 1576 #ifdef DDB
 1577 /*
 1578  * Dump details about an interrupt handler
 1579  */
 1580 static void
 1581 db_dump_intrhand(struct intr_handler *ih)
 1582 {
 1583         int comma;
 1584 
 1585         db_printf("\t%-10s ", ih->ih_name);
 1586         switch (ih->ih_pri) {
 1587         case PI_REALTIME:
 1588                 db_printf("CLK ");
 1589                 break;
 1590         case PI_AV:
 1591                 db_printf("AV  ");
 1592                 break;
 1593         case PI_TTY:
 1594                 db_printf("TTY ");
 1595                 break;
 1596         case PI_NET:
 1597                 db_printf("NET ");
 1598                 break;
 1599         case PI_DISK:
 1600                 db_printf("DISK");
 1601                 break;
 1602         case PI_DULL:
 1603                 db_printf("DULL");
 1604                 break;
 1605         default:
 1606                 if (ih->ih_pri >= PI_SOFT)
 1607                         db_printf("SWI ");
 1608                 else
 1609                         db_printf("%4u", ih->ih_pri);
 1610                 break;
 1611         }
 1612         db_printf(" ");
 1613         db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC);
 1614         db_printf("(%p)", ih->ih_argument);
 1615         if (ih->ih_need ||
 1616             (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD |
 1617             IH_MPSAFE)) != 0) {
 1618                 db_printf(" {");
 1619                 comma = 0;
 1620                 if (ih->ih_flags & IH_EXCLUSIVE) {
 1621                         if (comma)
 1622                                 db_printf(", ");
 1623                         db_printf("EXCL");
 1624                         comma = 1;
 1625                 }
 1626                 if (ih->ih_flags & IH_ENTROPY) {
 1627                         if (comma)
 1628                                 db_printf(", ");
 1629                         db_printf("ENTROPY");
 1630                         comma = 1;
 1631                 }
 1632                 if (ih->ih_flags & IH_DEAD) {
 1633                         if (comma)
 1634                                 db_printf(", ");
 1635                         db_printf("DEAD");
 1636                         comma = 1;
 1637                 }
 1638                 if (ih->ih_flags & IH_MPSAFE) {
 1639                         if (comma)
 1640                                 db_printf(", ");
 1641                         db_printf("MPSAFE");
 1642                         comma = 1;
 1643                 }
 1644                 if (ih->ih_need) {
 1645                         if (comma)
 1646                                 db_printf(", ");
 1647                         db_printf("NEED");
 1648                 }
 1649                 db_printf("}");
 1650         }
 1651         db_printf("\n");
 1652 }
 1653 
 1654 /*
 1655  * Dump details about a event.
 1656  */
 1657 void
 1658 db_dump_intr_event(struct intr_event *ie, int handlers)
 1659 {
 1660         struct intr_handler *ih;
 1661         struct intr_thread *it;
 1662         int comma;
 1663 
 1664         db_printf("%s ", ie->ie_fullname);
 1665         it = ie->ie_thread;
 1666         if (it != NULL)
 1667                 db_printf("(pid %d)", it->it_thread->td_proc->p_pid);
 1668         else
 1669                 db_printf("(no thread)");
 1670         if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 ||
 1671             (it != NULL && it->it_need)) {
 1672                 db_printf(" {");
 1673                 comma = 0;
 1674                 if (ie->ie_flags & IE_SOFT) {
 1675                         db_printf("SOFT");
 1676                         comma = 1;
 1677                 }
 1678                 if (ie->ie_flags & IE_ENTROPY) {
 1679                         if (comma)
 1680                                 db_printf(", ");
 1681                         db_printf("ENTROPY");
 1682                         comma = 1;
 1683                 }
 1684                 if (ie->ie_flags & IE_ADDING_THREAD) {
 1685                         if (comma)
 1686                                 db_printf(", ");
 1687                         db_printf("ADDING_THREAD");
 1688                         comma = 1;
 1689                 }
 1690                 if (it != NULL && it->it_need) {
 1691                         if (comma)
 1692                                 db_printf(", ");
 1693                         db_printf("NEED");
 1694                 }
 1695                 db_printf("}");
 1696         }
 1697         db_printf("\n");
 1698 
 1699         if (handlers)
 1700                 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
 1701                     db_dump_intrhand(ih);
 1702 }
 1703 
 1704 /*
 1705  * Dump data about interrupt handlers
 1706  */
 1707 DB_SHOW_COMMAND(intr, db_show_intr)
 1708 {
 1709         struct intr_event *ie;
 1710         int all, verbose;
 1711 
 1712         verbose = index(modif, 'v') != NULL;
 1713         all = index(modif, 'a') != NULL;
 1714         TAILQ_FOREACH(ie, &event_list, ie_list) {
 1715                 if (!all && TAILQ_EMPTY(&ie->ie_handlers))
 1716                         continue;
 1717                 db_dump_intr_event(ie, verbose);
 1718                 if (db_pager_quit)
 1719                         break;
 1720         }
 1721 }
 1722 #endif /* DDB */
 1723 
 1724 /*
 1725  * Start standard software interrupt threads
 1726  */
 1727 static void
 1728 start_softintr(void *dummy)
 1729 {
 1730         struct proc *p;
 1731 
 1732         if (swi_add(&clk_intr_event, "clock", softclock, NULL, SWI_CLOCK,
 1733                 INTR_MPSAFE, &softclock_ih) ||
 1734             swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
 1735                 panic("died while creating standard software ithreads");
 1736 
 1737         p = clk_intr_event->ie_thread->it_thread->td_proc;
 1738         PROC_LOCK(p);
 1739         p->p_flag |= P_NOLOAD;
 1740         PROC_UNLOCK(p);
 1741 }
 1742 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr,
 1743     NULL);
 1744 
 1745 /*
 1746  * Sysctls used by systat and others: hw.intrnames and hw.intrcnt.
 1747  * The data for this machine dependent, and the declarations are in machine
 1748  * dependent code.  The layout of intrnames and intrcnt however is machine
 1749  * independent.
 1750  *
 1751  * We do not know the length of intrcnt and intrnames at compile time, so
 1752  * calculate things at run time.
 1753  */
 1754 static int
 1755 sysctl_intrnames(SYSCTL_HANDLER_ARGS)
 1756 {
 1757         return (sysctl_handle_opaque(oidp, intrnames, eintrnames - intrnames,
 1758            req));
 1759 }
 1760 
 1761 SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD,
 1762     NULL, 0, sysctl_intrnames, "", "Interrupt Names");
 1763 
 1764 static int
 1765 sysctl_intrcnt(SYSCTL_HANDLER_ARGS)
 1766 {
 1767         return (sysctl_handle_opaque(oidp, intrcnt,
 1768             (char *)eintrcnt - (char *)intrcnt, req));
 1769 }
 1770 
 1771 SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD,
 1772     NULL, 0, sysctl_intrcnt, "", "Interrupt Counts");
 1773 
 1774 #ifdef DDB
 1775 /*
 1776  * DDB command to dump the interrupt statistics.
 1777  */
 1778 DB_SHOW_COMMAND(intrcnt, db_show_intrcnt)
 1779 {
 1780         u_long *i;
 1781         char *cp;
 1782 
 1783         cp = intrnames;
 1784         for (i = intrcnt; i != eintrcnt && !db_pager_quit; i++) {
 1785                 if (*cp == '\0')
 1786                         break;
 1787                 if (*i != 0)
 1788                         db_printf("%s\t%lu\n", cp, *i);
 1789                 cp += strlen(cp) + 1;
 1790         }
 1791 }
 1792 #endif

Cache object: f790167dfe8599475912dab77dc5c822


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.