The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_intr.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice unmodified, this list of conditions, and the following
   10  *    disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD: releng/8.2/sys/kern/kern_intr.c 204482 2010-02-28 21:28:24Z avg $");
   29 
   30 #include "opt_ddb.h"
   31 
   32 #include <sys/param.h>
   33 #include <sys/bus.h>
   34 #include <sys/conf.h>
   35 #include <sys/cpuset.h>
   36 #include <sys/rtprio.h>
   37 #include <sys/systm.h>
   38 #include <sys/interrupt.h>
   39 #include <sys/kernel.h>
   40 #include <sys/kthread.h>
   41 #include <sys/ktr.h>
   42 #include <sys/limits.h>
   43 #include <sys/lock.h>
   44 #include <sys/malloc.h>
   45 #include <sys/mutex.h>
   46 #include <sys/priv.h>
   47 #include <sys/proc.h>
   48 #include <sys/random.h>
   49 #include <sys/resourcevar.h>
   50 #include <sys/sched.h>
   51 #include <sys/smp.h>
   52 #include <sys/sysctl.h>
   53 #include <sys/syslog.h>
   54 #include <sys/unistd.h>
   55 #include <sys/vmmeter.h>
   56 #include <machine/atomic.h>
   57 #include <machine/cpu.h>
   58 #include <machine/md_var.h>
   59 #include <machine/stdarg.h>
   60 #ifdef DDB
   61 #include <ddb/ddb.h>
   62 #include <ddb/db_sym.h>
   63 #endif
   64 
   65 /*
   66  * Describe an interrupt thread.  There is one of these per interrupt event.
   67  */
   68 struct intr_thread {
   69         struct intr_event *it_event;
   70         struct thread *it_thread;       /* Kernel thread. */
   71         int     it_flags;               /* (j) IT_* flags. */
   72         int     it_need;                /* Needs service. */
   73 };
   74 
   75 /* Interrupt thread flags kept in it_flags */
   76 #define IT_DEAD         0x000001        /* Thread is waiting to exit. */
   77 
   78 struct  intr_entropy {
   79         struct  thread *td;
   80         uintptr_t event;
   81 };
   82 
   83 struct  intr_event *clk_intr_event;
   84 struct  intr_event *tty_intr_event;
   85 void    *vm_ih;
   86 struct proc *intrproc;
   87 
   88 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads");
   89 
   90 static int intr_storm_threshold = 1000;
   91 TUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold);
   92 SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW,
   93     &intr_storm_threshold, 0,
   94     "Number of consecutive interrupts before storm protection is enabled");
   95 static TAILQ_HEAD(, intr_event) event_list =
   96     TAILQ_HEAD_INITIALIZER(event_list);
   97 static struct mtx event_lock;
   98 MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF);
   99 
  100 static void     intr_event_update(struct intr_event *ie);
  101 #ifdef INTR_FILTER
  102 static int      intr_event_schedule_thread(struct intr_event *ie,
  103                     struct intr_thread *ithd);
  104 static int      intr_filter_loop(struct intr_event *ie,
  105                     struct trapframe *frame, struct intr_thread **ithd);
  106 static struct intr_thread *ithread_create(const char *name,
  107                               struct intr_handler *ih);
  108 #else
  109 static int      intr_event_schedule_thread(struct intr_event *ie);
  110 static struct intr_thread *ithread_create(const char *name);
  111 #endif
  112 static void     ithread_destroy(struct intr_thread *ithread);
  113 static void     ithread_execute_handlers(struct proc *p, 
  114                     struct intr_event *ie);
  115 #ifdef INTR_FILTER
  116 static void     priv_ithread_execute_handler(struct proc *p, 
  117                     struct intr_handler *ih);
  118 #endif
  119 static void     ithread_loop(void *);
  120 static void     ithread_update(struct intr_thread *ithd);
  121 static void     start_softintr(void *);
  122 
  123 /* Map an interrupt type to an ithread priority. */
  124 u_char
  125 intr_priority(enum intr_type flags)
  126 {
  127         u_char pri;
  128 
  129         flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET |
  130             INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV);
  131         switch (flags) {
  132         case INTR_TYPE_TTY:
  133                 pri = PI_TTYLOW;
  134                 break;
  135         case INTR_TYPE_BIO:
  136                 /*
  137                  * XXX We need to refine this.  BSD/OS distinguishes
  138                  * between tape and disk priorities.
  139                  */
  140                 pri = PI_DISK;
  141                 break;
  142         case INTR_TYPE_NET:
  143                 pri = PI_NET;
  144                 break;
  145         case INTR_TYPE_CAM:
  146                 pri = PI_DISK;          /* XXX or PI_CAM? */
  147                 break;
  148         case INTR_TYPE_AV:              /* Audio/video */
  149                 pri = PI_AV;
  150                 break;
  151         case INTR_TYPE_CLK:
  152                 pri = PI_REALTIME;
  153                 break;
  154         case INTR_TYPE_MISC:
  155                 pri = PI_DULL;          /* don't care */
  156                 break;
  157         default:
  158                 /* We didn't specify an interrupt level. */
  159                 panic("intr_priority: no interrupt type in flags");
  160         }
  161 
  162         return pri;
  163 }
  164 
  165 /*
  166  * Update an ithread based on the associated intr_event.
  167  */
  168 static void
  169 ithread_update(struct intr_thread *ithd)
  170 {
  171         struct intr_event *ie;
  172         struct thread *td;
  173         u_char pri;
  174 
  175         ie = ithd->it_event;
  176         td = ithd->it_thread;
  177 
  178         /* Determine the overall priority of this event. */
  179         if (TAILQ_EMPTY(&ie->ie_handlers))
  180                 pri = PRI_MAX_ITHD;
  181         else
  182                 pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri;
  183 
  184         /* Update name and priority. */
  185         strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name));
  186         thread_lock(td);
  187         sched_prio(td, pri);
  188         thread_unlock(td);
  189 }
  190 
  191 /*
  192  * Regenerate the full name of an interrupt event and update its priority.
  193  */
  194 static void
  195 intr_event_update(struct intr_event *ie)
  196 {
  197         struct intr_handler *ih;
  198         char *last;
  199         int missed, space;
  200 
  201         /* Start off with no entropy and just the name of the event. */
  202         mtx_assert(&ie->ie_lock, MA_OWNED);
  203         strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
  204         ie->ie_flags &= ~IE_ENTROPY;
  205         missed = 0;
  206         space = 1;
  207 
  208         /* Run through all the handlers updating values. */
  209         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  210                 if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <
  211                     sizeof(ie->ie_fullname)) {
  212                         strcat(ie->ie_fullname, " ");
  213                         strcat(ie->ie_fullname, ih->ih_name);
  214                         space = 0;
  215                 } else
  216                         missed++;
  217                 if (ih->ih_flags & IH_ENTROPY)
  218                         ie->ie_flags |= IE_ENTROPY;
  219         }
  220 
  221         /*
  222          * If the handler names were too long, add +'s to indicate missing
  223          * names. If we run out of room and still have +'s to add, change
  224          * the last character from a + to a *.
  225          */
  226         last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];
  227         while (missed-- > 0) {
  228                 if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {
  229                         if (*last == '+') {
  230                                 *last = '*';
  231                                 break;
  232                         } else
  233                                 *last = '+';
  234                 } else if (space) {
  235                         strcat(ie->ie_fullname, " +");
  236                         space = 0;
  237                 } else
  238                         strcat(ie->ie_fullname, "+");
  239         }
  240 
  241         /*
  242          * If this event has an ithread, update it's priority and
  243          * name.
  244          */
  245         if (ie->ie_thread != NULL)
  246                 ithread_update(ie->ie_thread);
  247         CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);
  248 }
  249 
  250 int
  251 intr_event_create(struct intr_event **event, void *source, int flags, int irq,
  252     void (*pre_ithread)(void *), void (*post_ithread)(void *),
  253     void (*post_filter)(void *), int (*assign_cpu)(void *, u_char),
  254     const char *fmt, ...)
  255 {
  256         struct intr_event *ie;
  257         va_list ap;
  258 
  259         /* The only valid flag during creation is IE_SOFT. */
  260         if ((flags & ~IE_SOFT) != 0)
  261                 return (EINVAL);
  262         ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);
  263         ie->ie_source = source;
  264         ie->ie_pre_ithread = pre_ithread;
  265         ie->ie_post_ithread = post_ithread;
  266         ie->ie_post_filter = post_filter;
  267         ie->ie_assign_cpu = assign_cpu;
  268         ie->ie_flags = flags;
  269         ie->ie_irq = irq;
  270         ie->ie_cpu = NOCPU;
  271         TAILQ_INIT(&ie->ie_handlers);
  272         mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
  273 
  274         va_start(ap, fmt);
  275         vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);
  276         va_end(ap);
  277         strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
  278         mtx_lock(&event_lock);
  279         TAILQ_INSERT_TAIL(&event_list, ie, ie_list);
  280         mtx_unlock(&event_lock);
  281         if (event != NULL)
  282                 *event = ie;
  283         CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);
  284         return (0);
  285 }
  286 
  287 /*
  288  * Bind an interrupt event to the specified CPU.  Note that not all
  289  * platforms support binding an interrupt to a CPU.  For those
  290  * platforms this request will fail.  For supported platforms, any
  291  * associated ithreads as well as the primary interrupt context will
  292  * be bound to the specificed CPU.  Using a cpu id of NOCPU unbinds
  293  * the interrupt event.
  294  */
  295 int
  296 intr_event_bind(struct intr_event *ie, u_char cpu)
  297 {
  298         cpuset_t mask;
  299         lwpid_t id;
  300         int error;
  301 
  302         /* Need a CPU to bind to. */
  303         if (cpu != NOCPU && CPU_ABSENT(cpu))
  304                 return (EINVAL);
  305 
  306         if (ie->ie_assign_cpu == NULL)
  307                 return (EOPNOTSUPP);
  308 
  309         error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR);
  310         if (error)
  311                 return (error);
  312 
  313         /*
  314          * If we have any ithreads try to set their mask first to verify
  315          * permissions, etc.
  316          */
  317         mtx_lock(&ie->ie_lock);
  318         if (ie->ie_thread != NULL) {
  319                 CPU_ZERO(&mask);
  320                 if (cpu == NOCPU)
  321                         CPU_COPY(cpuset_root, &mask);
  322                 else
  323                         CPU_SET(cpu, &mask);
  324                 id = ie->ie_thread->it_thread->td_tid;
  325                 mtx_unlock(&ie->ie_lock);
  326                 error = cpuset_setthread(id, &mask);
  327                 if (error)
  328                         return (error);
  329         } else
  330                 mtx_unlock(&ie->ie_lock);
  331         error = ie->ie_assign_cpu(ie->ie_source, cpu);
  332         if (error) {
  333                 mtx_lock(&ie->ie_lock);
  334                 if (ie->ie_thread != NULL) {
  335                         CPU_ZERO(&mask);
  336                         if (ie->ie_cpu == NOCPU)
  337                                 CPU_COPY(cpuset_root, &mask);
  338                         else
  339                                 CPU_SET(cpu, &mask);
  340                         id = ie->ie_thread->it_thread->td_tid;
  341                         mtx_unlock(&ie->ie_lock);
  342                         (void)cpuset_setthread(id, &mask);
  343                 } else
  344                         mtx_unlock(&ie->ie_lock);
  345                 return (error);
  346         }
  347 
  348         mtx_lock(&ie->ie_lock);
  349         ie->ie_cpu = cpu;
  350         mtx_unlock(&ie->ie_lock);
  351 
  352         return (error);
  353 }
  354 
  355 static struct intr_event *
  356 intr_lookup(int irq)
  357 {
  358         struct intr_event *ie;
  359 
  360         mtx_lock(&event_lock);
  361         TAILQ_FOREACH(ie, &event_list, ie_list)
  362                 if (ie->ie_irq == irq &&
  363                     (ie->ie_flags & IE_SOFT) == 0 &&
  364                     TAILQ_FIRST(&ie->ie_handlers) != NULL)
  365                         break;
  366         mtx_unlock(&event_lock);
  367         return (ie);
  368 }
  369 
  370 int
  371 intr_setaffinity(int irq, void *m)
  372 {
  373         struct intr_event *ie;
  374         cpuset_t *mask;
  375         u_char cpu;
  376         int n;
  377 
  378         mask = m;
  379         cpu = NOCPU;
  380         /*
  381          * If we're setting all cpus we can unbind.  Otherwise make sure
  382          * only one cpu is in the set.
  383          */
  384         if (CPU_CMP(cpuset_root, mask)) {
  385                 for (n = 0; n < CPU_SETSIZE; n++) {
  386                         if (!CPU_ISSET(n, mask))
  387                                 continue;
  388                         if (cpu != NOCPU)
  389                                 return (EINVAL);
  390                         cpu = (u_char)n;
  391                 }
  392         }
  393         ie = intr_lookup(irq);
  394         if (ie == NULL)
  395                 return (ESRCH);
  396         return (intr_event_bind(ie, cpu));
  397 }
  398 
  399 int
  400 intr_getaffinity(int irq, void *m)
  401 {
  402         struct intr_event *ie;
  403         cpuset_t *mask;
  404 
  405         mask = m;
  406         ie = intr_lookup(irq);
  407         if (ie == NULL)
  408                 return (ESRCH);
  409         CPU_ZERO(mask);
  410         mtx_lock(&ie->ie_lock);
  411         if (ie->ie_cpu == NOCPU)
  412                 CPU_COPY(cpuset_root, mask);
  413         else
  414                 CPU_SET(ie->ie_cpu, mask);
  415         mtx_unlock(&ie->ie_lock);
  416         return (0);
  417 }
  418 
  419 int
  420 intr_event_destroy(struct intr_event *ie)
  421 {
  422 
  423         mtx_lock(&event_lock);
  424         mtx_lock(&ie->ie_lock);
  425         if (!TAILQ_EMPTY(&ie->ie_handlers)) {
  426                 mtx_unlock(&ie->ie_lock);
  427                 mtx_unlock(&event_lock);
  428                 return (EBUSY);
  429         }
  430         TAILQ_REMOVE(&event_list, ie, ie_list);
  431 #ifndef notyet
  432         if (ie->ie_thread != NULL) {
  433                 ithread_destroy(ie->ie_thread);
  434                 ie->ie_thread = NULL;
  435         }
  436 #endif
  437         mtx_unlock(&ie->ie_lock);
  438         mtx_unlock(&event_lock);
  439         mtx_destroy(&ie->ie_lock);
  440         free(ie, M_ITHREAD);
  441         return (0);
  442 }
  443 
  444 #ifndef INTR_FILTER
  445 static struct intr_thread *
  446 ithread_create(const char *name)
  447 {
  448         struct intr_thread *ithd;
  449         struct thread *td;
  450         int error;
  451 
  452         ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
  453 
  454         error = kproc_kthread_add(ithread_loop, ithd, &intrproc,
  455                     &td, RFSTOPPED | RFHIGHPID,
  456                     0, "intr", "%s", name);
  457         if (error)
  458                 panic("kproc_create() failed with %d", error);
  459         thread_lock(td);
  460         sched_class(td, PRI_ITHD);
  461         TD_SET_IWAIT(td);
  462         thread_unlock(td);
  463         td->td_pflags |= TDP_ITHREAD;
  464         ithd->it_thread = td;
  465         CTR2(KTR_INTR, "%s: created %s", __func__, name);
  466         return (ithd);
  467 }
  468 #else
  469 static struct intr_thread *
  470 ithread_create(const char *name, struct intr_handler *ih)
  471 {
  472         struct intr_thread *ithd;
  473         struct thread *td;
  474         int error;
  475 
  476         ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
  477 
  478         error = kproc_kthread_add(ithread_loop, ih, &intrproc,
  479                     &td, RFSTOPPED | RFHIGHPID,
  480                     0, "intr", "%s", name);
  481         if (error)
  482                 panic("kproc_create() failed with %d", error);
  483         thread_lock(td);
  484         sched_class(td, PRI_ITHD);
  485         TD_SET_IWAIT(td);
  486         thread_unlock(td);
  487         td->td_pflags |= TDP_ITHREAD;
  488         ithd->it_thread = td;
  489         CTR2(KTR_INTR, "%s: created %s", __func__, name);
  490         return (ithd);
  491 }
  492 #endif
  493 
  494 static void
  495 ithread_destroy(struct intr_thread *ithread)
  496 {
  497         struct thread *td;
  498 
  499         CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name);
  500         td = ithread->it_thread;
  501         thread_lock(td);
  502         ithread->it_flags |= IT_DEAD;
  503         if (TD_AWAITING_INTR(td)) {
  504                 TD_CLR_IWAIT(td);
  505                 sched_add(td, SRQ_INTR);
  506         }
  507         thread_unlock(td);
  508 }
  509 
  510 #ifndef INTR_FILTER
  511 int
  512 intr_event_add_handler(struct intr_event *ie, const char *name,
  513     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
  514     enum intr_type flags, void **cookiep)
  515 {
  516         struct intr_handler *ih, *temp_ih;
  517         struct intr_thread *it;
  518 
  519         if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
  520                 return (EINVAL);
  521 
  522         /* Allocate and populate an interrupt handler structure. */
  523         ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
  524         ih->ih_filter = filter;
  525         ih->ih_handler = handler;
  526         ih->ih_argument = arg;
  527         strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
  528         ih->ih_event = ie;
  529         ih->ih_pri = pri;
  530         if (flags & INTR_EXCL)
  531                 ih->ih_flags = IH_EXCLUSIVE;
  532         if (flags & INTR_MPSAFE)
  533                 ih->ih_flags |= IH_MPSAFE;
  534         if (flags & INTR_ENTROPY)
  535                 ih->ih_flags |= IH_ENTROPY;
  536 
  537         /* We can only have one exclusive handler in a event. */
  538         mtx_lock(&ie->ie_lock);
  539         if (!TAILQ_EMPTY(&ie->ie_handlers)) {
  540                 if ((flags & INTR_EXCL) ||
  541                     (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
  542                         mtx_unlock(&ie->ie_lock);
  543                         free(ih, M_ITHREAD);
  544                         return (EINVAL);
  545                 }
  546         }
  547 
  548         /* Add the new handler to the event in priority order. */
  549         TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
  550                 if (temp_ih->ih_pri > ih->ih_pri)
  551                         break;
  552         }
  553         if (temp_ih == NULL)
  554                 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
  555         else
  556                 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
  557         intr_event_update(ie);
  558 
  559         /* Create a thread if we need one. */
  560         while (ie->ie_thread == NULL && handler != NULL) {
  561                 if (ie->ie_flags & IE_ADDING_THREAD)
  562                         msleep(ie, &ie->ie_lock, 0, "ithread", 0);
  563                 else {
  564                         ie->ie_flags |= IE_ADDING_THREAD;
  565                         mtx_unlock(&ie->ie_lock);
  566                         it = ithread_create("intr: newborn");
  567                         mtx_lock(&ie->ie_lock);
  568                         ie->ie_flags &= ~IE_ADDING_THREAD;
  569                         ie->ie_thread = it;
  570                         it->it_event = ie;
  571                         ithread_update(it);
  572                         wakeup(ie);
  573                 }
  574         }
  575         CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
  576             ie->ie_name);
  577         mtx_unlock(&ie->ie_lock);
  578 
  579         if (cookiep != NULL)
  580                 *cookiep = ih;
  581         return (0);
  582 }
  583 #else
  584 int
  585 intr_event_add_handler(struct intr_event *ie, const char *name,
  586     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
  587     enum intr_type flags, void **cookiep)
  588 {
  589         struct intr_handler *ih, *temp_ih;
  590         struct intr_thread *it;
  591 
  592         if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
  593                 return (EINVAL);
  594 
  595         /* Allocate and populate an interrupt handler structure. */
  596         ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
  597         ih->ih_filter = filter;
  598         ih->ih_handler = handler;
  599         ih->ih_argument = arg;
  600         strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
  601         ih->ih_event = ie;
  602         ih->ih_pri = pri;
  603         if (flags & INTR_EXCL)
  604                 ih->ih_flags = IH_EXCLUSIVE;
  605         if (flags & INTR_MPSAFE)
  606                 ih->ih_flags |= IH_MPSAFE;
  607         if (flags & INTR_ENTROPY)
  608                 ih->ih_flags |= IH_ENTROPY;
  609 
  610         /* We can only have one exclusive handler in a event. */
  611         mtx_lock(&ie->ie_lock);
  612         if (!TAILQ_EMPTY(&ie->ie_handlers)) {
  613                 if ((flags & INTR_EXCL) ||
  614                     (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
  615                         mtx_unlock(&ie->ie_lock);
  616                         free(ih, M_ITHREAD);
  617                         return (EINVAL);
  618                 }
  619         }
  620 
  621         /* Add the new handler to the event in priority order. */
  622         TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
  623                 if (temp_ih->ih_pri > ih->ih_pri)
  624                         break;
  625         }
  626         if (temp_ih == NULL)
  627                 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
  628         else
  629                 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
  630         intr_event_update(ie);
  631 
  632         /* For filtered handlers, create a private ithread to run on. */
  633         if (filter != NULL && handler != NULL) { 
  634                 mtx_unlock(&ie->ie_lock);
  635                 it = ithread_create("intr: newborn", ih);               
  636                 mtx_lock(&ie->ie_lock);
  637                 it->it_event = ie; 
  638                 ih->ih_thread = it;
  639                 ithread_update(it); // XXX - do we really need this?!?!?
  640         } else { /* Create the global per-event thread if we need one. */
  641                 while (ie->ie_thread == NULL && handler != NULL) {
  642                         if (ie->ie_flags & IE_ADDING_THREAD)
  643                                 msleep(ie, &ie->ie_lock, 0, "ithread", 0);
  644                         else {
  645                                 ie->ie_flags |= IE_ADDING_THREAD;
  646                                 mtx_unlock(&ie->ie_lock);
  647                                 it = ithread_create("intr: newborn", ih);
  648                                 mtx_lock(&ie->ie_lock);
  649                                 ie->ie_flags &= ~IE_ADDING_THREAD;
  650                                 ie->ie_thread = it;
  651                                 it->it_event = ie;
  652                                 ithread_update(it);
  653                                 wakeup(ie);
  654                         }
  655                 }
  656         }
  657         CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
  658             ie->ie_name);
  659         mtx_unlock(&ie->ie_lock);
  660 
  661         if (cookiep != NULL)
  662                 *cookiep = ih;
  663         return (0);
  664 }
  665 #endif
  666 
  667 /*
  668  * Append a description preceded by a ':' to the name of the specified
  669  * interrupt handler.
  670  */
  671 int
  672 intr_event_describe_handler(struct intr_event *ie, void *cookie,
  673     const char *descr)
  674 {
  675         struct intr_handler *ih;
  676         size_t space;
  677         char *start;
  678 
  679         mtx_lock(&ie->ie_lock);
  680 #ifdef INVARIANTS
  681         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  682                 if (ih == cookie)
  683                         break;
  684         }
  685         if (ih == NULL) {
  686                 mtx_unlock(&ie->ie_lock);
  687                 panic("handler %p not found in interrupt event %p", cookie, ie);
  688         }
  689 #endif
  690         ih = cookie;
  691 
  692         /*
  693          * Look for an existing description by checking for an
  694          * existing ":".  This assumes device names do not include
  695          * colons.  If one is found, prepare to insert the new
  696          * description at that point.  If one is not found, find the
  697          * end of the name to use as the insertion point.
  698          */
  699         start = index(ih->ih_name, ':');
  700         if (start == NULL)
  701                 start = index(ih->ih_name, 0);
  702 
  703         /*
  704          * See if there is enough remaining room in the string for the
  705          * description + ":".  The "- 1" leaves room for the trailing
  706          * '\0'.  The "+ 1" accounts for the colon.
  707          */
  708         space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1;
  709         if (strlen(descr) + 1 > space) {
  710                 mtx_unlock(&ie->ie_lock);
  711                 return (ENOSPC);
  712         }
  713 
  714         /* Append a colon followed by the description. */
  715         *start = ':';
  716         strcpy(start + 1, descr);
  717         intr_event_update(ie);
  718         mtx_unlock(&ie->ie_lock);
  719         return (0);
  720 }
  721 
  722 /*
  723  * Return the ie_source field from the intr_event an intr_handler is
  724  * associated with.
  725  */
  726 void *
  727 intr_handler_source(void *cookie)
  728 {
  729         struct intr_handler *ih;
  730         struct intr_event *ie;
  731 
  732         ih = (struct intr_handler *)cookie;
  733         if (ih == NULL)
  734                 return (NULL);
  735         ie = ih->ih_event;
  736         KASSERT(ie != NULL,
  737             ("interrupt handler \"%s\" has a NULL interrupt event",
  738             ih->ih_name));
  739         return (ie->ie_source);
  740 }
  741 
  742 #ifndef INTR_FILTER
  743 int
  744 intr_event_remove_handler(void *cookie)
  745 {
  746         struct intr_handler *handler = (struct intr_handler *)cookie;
  747         struct intr_event *ie;
  748 #ifdef INVARIANTS
  749         struct intr_handler *ih;
  750 #endif
  751 #ifdef notyet
  752         int dead;
  753 #endif
  754 
  755         if (handler == NULL)
  756                 return (EINVAL);
  757         ie = handler->ih_event;
  758         KASSERT(ie != NULL,
  759             ("interrupt handler \"%s\" has a NULL interrupt event",
  760             handler->ih_name));
  761         mtx_lock(&ie->ie_lock);
  762         CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
  763             ie->ie_name);
  764 #ifdef INVARIANTS
  765         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
  766                 if (ih == handler)
  767                         goto ok;
  768         mtx_unlock(&ie->ie_lock);
  769         panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
  770             ih->ih_name, ie->ie_name);
  771 ok:
  772 #endif
  773         /*
  774          * If there is no ithread, then just remove the handler and return.
  775          * XXX: Note that an INTR_FAST handler might be running on another
  776          * CPU!
  777          */
  778         if (ie->ie_thread == NULL) {
  779                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  780                 mtx_unlock(&ie->ie_lock);
  781                 free(handler, M_ITHREAD);
  782                 return (0);
  783         }
  784 
  785         /*
  786          * If the interrupt thread is already running, then just mark this
  787          * handler as being dead and let the ithread do the actual removal.
  788          *
  789          * During a cold boot while cold is set, msleep() does not sleep,
  790          * so we have to remove the handler here rather than letting the
  791          * thread do it.
  792          */
  793         thread_lock(ie->ie_thread->it_thread);
  794         if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) {
  795                 handler->ih_flags |= IH_DEAD;
  796 
  797                 /*
  798                  * Ensure that the thread will process the handler list
  799                  * again and remove this handler if it has already passed
  800                  * it on the list.
  801                  */
  802                 ie->ie_thread->it_need = 1;
  803         } else
  804                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  805         thread_unlock(ie->ie_thread->it_thread);
  806         while (handler->ih_flags & IH_DEAD)
  807                 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
  808         intr_event_update(ie);
  809 #ifdef notyet
  810         /*
  811          * XXX: This could be bad in the case of ppbus(8).  Also, I think
  812          * this could lead to races of stale data when servicing an
  813          * interrupt.
  814          */
  815         dead = 1;
  816         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  817                 if (!(ih->ih_flags & IH_FAST)) {
  818                         dead = 0;
  819                         break;
  820                 }
  821         }
  822         if (dead) {
  823                 ithread_destroy(ie->ie_thread);
  824                 ie->ie_thread = NULL;
  825         }
  826 #endif
  827         mtx_unlock(&ie->ie_lock);
  828         free(handler, M_ITHREAD);
  829         return (0);
  830 }
  831 
  832 static int
  833 intr_event_schedule_thread(struct intr_event *ie)
  834 {
  835         struct intr_entropy entropy;
  836         struct intr_thread *it;
  837         struct thread *td;
  838         struct thread *ctd;
  839         struct proc *p;
  840 
  841         /*
  842          * If no ithread or no handlers, then we have a stray interrupt.
  843          */
  844         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) ||
  845             ie->ie_thread == NULL)
  846                 return (EINVAL);
  847 
  848         ctd = curthread;
  849         it = ie->ie_thread;
  850         td = it->it_thread;
  851         p = td->td_proc;
  852 
  853         /*
  854          * If any of the handlers for this ithread claim to be good
  855          * sources of entropy, then gather some.
  856          */
  857         if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
  858                 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
  859                     p->p_pid, td->td_name);
  860                 entropy.event = (uintptr_t)ie;
  861                 entropy.td = ctd;
  862                 random_harvest(&entropy, sizeof(entropy), 2, 0,
  863                     RANDOM_INTERRUPT);
  864         }
  865 
  866         KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
  867 
  868         /*
  869          * Set it_need to tell the thread to keep running if it is already
  870          * running.  Then, lock the thread and see if we actually need to
  871          * put it on the runqueue.
  872          */
  873         it->it_need = 1;
  874         thread_lock(td);
  875         if (TD_AWAITING_INTR(td)) {
  876                 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
  877                     td->td_name);
  878                 TD_CLR_IWAIT(td);
  879                 sched_add(td, SRQ_INTR);
  880         } else {
  881                 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
  882                     __func__, p->p_pid, td->td_name, it->it_need, td->td_state);
  883         }
  884         thread_unlock(td);
  885 
  886         return (0);
  887 }
  888 #else
  889 int
  890 intr_event_remove_handler(void *cookie)
  891 {
  892         struct intr_handler *handler = (struct intr_handler *)cookie;
  893         struct intr_event *ie;
  894         struct intr_thread *it;
  895 #ifdef INVARIANTS
  896         struct intr_handler *ih;
  897 #endif
  898 #ifdef notyet
  899         int dead;
  900 #endif
  901 
  902         if (handler == NULL)
  903                 return (EINVAL);
  904         ie = handler->ih_event;
  905         KASSERT(ie != NULL,
  906             ("interrupt handler \"%s\" has a NULL interrupt event",
  907             handler->ih_name));
  908         mtx_lock(&ie->ie_lock);
  909         CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
  910             ie->ie_name);
  911 #ifdef INVARIANTS
  912         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
  913                 if (ih == handler)
  914                         goto ok;
  915         mtx_unlock(&ie->ie_lock);
  916         panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
  917             ih->ih_name, ie->ie_name);
  918 ok:
  919 #endif
  920         /*
  921          * If there are no ithreads (per event and per handler), then
  922          * just remove the handler and return.  
  923          * XXX: Note that an INTR_FAST handler might be running on another CPU!
  924          */
  925         if (ie->ie_thread == NULL && handler->ih_thread == NULL) {
  926                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  927                 mtx_unlock(&ie->ie_lock);
  928                 free(handler, M_ITHREAD);
  929                 return (0);
  930         }
  931 
  932         /* Private or global ithread? */
  933         it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread;
  934         /*
  935          * If the interrupt thread is already running, then just mark this
  936          * handler as being dead and let the ithread do the actual removal.
  937          *
  938          * During a cold boot while cold is set, msleep() does not sleep,
  939          * so we have to remove the handler here rather than letting the
  940          * thread do it.
  941          */
  942         thread_lock(it->it_thread);
  943         if (!TD_AWAITING_INTR(it->it_thread) && !cold) {
  944                 handler->ih_flags |= IH_DEAD;
  945 
  946                 /*
  947                  * Ensure that the thread will process the handler list
  948                  * again and remove this handler if it has already passed
  949                  * it on the list.
  950                  */
  951                 it->it_need = 1;
  952         } else
  953                 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
  954         thread_unlock(it->it_thread);
  955         while (handler->ih_flags & IH_DEAD)
  956                 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
  957         /* 
  958          * At this point, the handler has been disconnected from the event,
  959          * so we can kill the private ithread if any.
  960          */
  961         if (handler->ih_thread) {
  962                 ithread_destroy(handler->ih_thread);
  963                 handler->ih_thread = NULL;
  964         }
  965         intr_event_update(ie);
  966 #ifdef notyet
  967         /*
  968          * XXX: This could be bad in the case of ppbus(8).  Also, I think
  969          * this could lead to races of stale data when servicing an
  970          * interrupt.
  971          */
  972         dead = 1;
  973         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  974                 if (handler != NULL) {
  975                         dead = 0;
  976                         break;
  977                 }
  978         }
  979         if (dead) {
  980                 ithread_destroy(ie->ie_thread);
  981                 ie->ie_thread = NULL;
  982         }
  983 #endif
  984         mtx_unlock(&ie->ie_lock);
  985         free(handler, M_ITHREAD);
  986         return (0);
  987 }
  988 
  989 static int
  990 intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it)
  991 {
  992         struct intr_entropy entropy;
  993         struct thread *td;
  994         struct thread *ctd;
  995         struct proc *p;
  996 
  997         /*
  998          * If no ithread or no handlers, then we have a stray interrupt.
  999          */
 1000         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL)
 1001                 return (EINVAL);
 1002 
 1003         ctd = curthread;
 1004         td = it->it_thread;
 1005         p = td->td_proc;
 1006 
 1007         /*
 1008          * If any of the handlers for this ithread claim to be good
 1009          * sources of entropy, then gather some.
 1010          */
 1011         if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
 1012                 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
 1013                     p->p_pid, td->td_name);
 1014                 entropy.event = (uintptr_t)ie;
 1015                 entropy.td = ctd;
 1016                 random_harvest(&entropy, sizeof(entropy), 2, 0,
 1017                     RANDOM_INTERRUPT);
 1018         }
 1019 
 1020         KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
 1021 
 1022         /*
 1023          * Set it_need to tell the thread to keep running if it is already
 1024          * running.  Then, lock the thread and see if we actually need to
 1025          * put it on the runqueue.
 1026          */
 1027         it->it_need = 1;
 1028         thread_lock(td);
 1029         if (TD_AWAITING_INTR(td)) {
 1030                 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
 1031                     td->td_name);
 1032                 TD_CLR_IWAIT(td);
 1033                 sched_add(td, SRQ_INTR);
 1034         } else {
 1035                 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
 1036                     __func__, p->p_pid, td->td_name, it->it_need, td->td_state);
 1037         }
 1038         thread_unlock(td);
 1039 
 1040         return (0);
 1041 }
 1042 #endif
 1043 
 1044 /*
 1045  * Allow interrupt event binding for software interrupt handlers -- a no-op,
 1046  * since interrupts are generated in software rather than being directed by
 1047  * a PIC.
 1048  */
 1049 static int
 1050 swi_assign_cpu(void *arg, u_char cpu)
 1051 {
 1052 
 1053         return (0);
 1054 }
 1055 
 1056 /*
 1057  * Add a software interrupt handler to a specified event.  If a given event
 1058  * is not specified, then a new event is created.
 1059  */
 1060 int
 1061 swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
 1062             void *arg, int pri, enum intr_type flags, void **cookiep)
 1063 {
 1064         struct intr_event *ie;
 1065         int error;
 1066 
 1067         if (flags & INTR_ENTROPY)
 1068                 return (EINVAL);
 1069 
 1070         ie = (eventp != NULL) ? *eventp : NULL;
 1071 
 1072         if (ie != NULL) {
 1073                 if (!(ie->ie_flags & IE_SOFT))
 1074                         return (EINVAL);
 1075         } else {
 1076                 error = intr_event_create(&ie, NULL, IE_SOFT, 0,
 1077                     NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri);
 1078                 if (error)
 1079                         return (error);
 1080                 if (eventp != NULL)
 1081                         *eventp = ie;
 1082         }
 1083         error = intr_event_add_handler(ie, name, NULL, handler, arg,
 1084             (pri * RQ_PPQ) + PI_SOFT, flags, cookiep);
 1085         if (error)
 1086                 return (error);
 1087         if (pri == SWI_CLOCK) {
 1088                 struct proc *p;
 1089                 p = ie->ie_thread->it_thread->td_proc;
 1090                 PROC_LOCK(p);
 1091                 p->p_flag |= P_NOLOAD;
 1092                 PROC_UNLOCK(p);
 1093         }
 1094         return (0);
 1095 }
 1096 
 1097 /*
 1098  * Schedule a software interrupt thread.
 1099  */
 1100 void
 1101 swi_sched(void *cookie, int flags)
 1102 {
 1103         struct intr_handler *ih = (struct intr_handler *)cookie;
 1104         struct intr_event *ie = ih->ih_event;
 1105         int error;
 1106 
 1107         CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
 1108             ih->ih_need);
 1109 
 1110         /*
 1111          * Set ih_need for this handler so that if the ithread is already
 1112          * running it will execute this handler on the next pass.  Otherwise,
 1113          * it will execute it the next time it runs.
 1114          */
 1115         atomic_store_rel_int(&ih->ih_need, 1);
 1116 
 1117         if (!(flags & SWI_DELAY)) {
 1118                 PCPU_INC(cnt.v_soft);
 1119 #ifdef INTR_FILTER
 1120                 error = intr_event_schedule_thread(ie, ie->ie_thread);
 1121 #else
 1122                 error = intr_event_schedule_thread(ie);
 1123 #endif
 1124                 KASSERT(error == 0, ("stray software interrupt"));
 1125         }
 1126 }
 1127 
 1128 /*
 1129  * Remove a software interrupt handler.  Currently this code does not
 1130  * remove the associated interrupt event if it becomes empty.  Calling code
 1131  * may do so manually via intr_event_destroy(), but that's not really
 1132  * an optimal interface.
 1133  */
 1134 int
 1135 swi_remove(void *cookie)
 1136 {
 1137 
 1138         return (intr_event_remove_handler(cookie));
 1139 }
 1140 
 1141 #ifdef INTR_FILTER
 1142 static void
 1143 priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih)
 1144 {
 1145         struct intr_event *ie;
 1146 
 1147         ie = ih->ih_event;
 1148         /*
 1149          * If this handler is marked for death, remove it from
 1150          * the list of handlers and wake up the sleeper.
 1151          */
 1152         if (ih->ih_flags & IH_DEAD) {
 1153                 mtx_lock(&ie->ie_lock);
 1154                 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
 1155                 ih->ih_flags &= ~IH_DEAD;
 1156                 wakeup(ih);
 1157                 mtx_unlock(&ie->ie_lock);
 1158                 return;
 1159         }
 1160         
 1161         /* Execute this handler. */
 1162         CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
 1163              __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument,
 1164              ih->ih_name, ih->ih_flags);
 1165         
 1166         if (!(ih->ih_flags & IH_MPSAFE))
 1167                 mtx_lock(&Giant);
 1168         ih->ih_handler(ih->ih_argument);
 1169         if (!(ih->ih_flags & IH_MPSAFE))
 1170                 mtx_unlock(&Giant);
 1171 }
 1172 #endif
 1173 
 1174 /*
 1175  * This is a public function for use by drivers that mux interrupt
 1176  * handlers for child devices from their interrupt handler.
 1177  */
 1178 void
 1179 intr_event_execute_handlers(struct proc *p, struct intr_event *ie)
 1180 {
 1181         struct intr_handler *ih, *ihn;
 1182 
 1183         TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {
 1184                 /*
 1185                  * If this handler is marked for death, remove it from
 1186                  * the list of handlers and wake up the sleeper.
 1187                  */
 1188                 if (ih->ih_flags & IH_DEAD) {
 1189                         mtx_lock(&ie->ie_lock);
 1190                         TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
 1191                         ih->ih_flags &= ~IH_DEAD;
 1192                         wakeup(ih);
 1193                         mtx_unlock(&ie->ie_lock);
 1194                         continue;
 1195                 }
 1196 
 1197                 /* Skip filter only handlers */
 1198                 if (ih->ih_handler == NULL)
 1199                         continue;
 1200 
 1201                 /*
 1202                  * For software interrupt threads, we only execute
 1203                  * handlers that have their need flag set.  Hardware
 1204                  * interrupt threads always invoke all of their handlers.
 1205                  */
 1206                 if (ie->ie_flags & IE_SOFT) {
 1207                         if (!ih->ih_need)
 1208                                 continue;
 1209                         else
 1210                                 atomic_store_rel_int(&ih->ih_need, 0);
 1211                 }
 1212 
 1213                 /* Execute this handler. */
 1214                 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
 1215                     __func__, p->p_pid, (void *)ih->ih_handler, 
 1216                     ih->ih_argument, ih->ih_name, ih->ih_flags);
 1217 
 1218                 if (!(ih->ih_flags & IH_MPSAFE))
 1219                         mtx_lock(&Giant);
 1220                 ih->ih_handler(ih->ih_argument);
 1221                 if (!(ih->ih_flags & IH_MPSAFE))
 1222                         mtx_unlock(&Giant);
 1223         }
 1224 }
 1225 
 1226 static void
 1227 ithread_execute_handlers(struct proc *p, struct intr_event *ie)
 1228 {
 1229 
 1230         /* Interrupt handlers should not sleep. */
 1231         if (!(ie->ie_flags & IE_SOFT))
 1232                 THREAD_NO_SLEEPING();
 1233         intr_event_execute_handlers(p, ie);
 1234         if (!(ie->ie_flags & IE_SOFT))
 1235                 THREAD_SLEEPING_OK();
 1236 
 1237         /*
 1238          * Interrupt storm handling:
 1239          *
 1240          * If this interrupt source is currently storming, then throttle
 1241          * it to only fire the handler once  per clock tick.
 1242          *
 1243          * If this interrupt source is not currently storming, but the
 1244          * number of back to back interrupts exceeds the storm threshold,
 1245          * then enter storming mode.
 1246          */
 1247         if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold &&
 1248             !(ie->ie_flags & IE_SOFT)) {
 1249                 /* Report the message only once every second. */
 1250                 if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {
 1251                         printf(
 1252         "interrupt storm detected on \"%s\"; throttling interrupt source\n",
 1253                             ie->ie_name);
 1254                 }
 1255                 pause("istorm", 1);
 1256         } else
 1257                 ie->ie_count++;
 1258 
 1259         /*
 1260          * Now that all the handlers have had a chance to run, reenable
 1261          * the interrupt source.
 1262          */
 1263         if (ie->ie_post_ithread != NULL)
 1264                 ie->ie_post_ithread(ie->ie_source);
 1265 }
 1266 
 1267 #ifndef INTR_FILTER
 1268 /*
 1269  * This is the main code for interrupt threads.
 1270  */
 1271 static void
 1272 ithread_loop(void *arg)
 1273 {
 1274         struct intr_thread *ithd;
 1275         struct intr_event *ie;
 1276         struct thread *td;
 1277         struct proc *p;
 1278 
 1279         td = curthread;
 1280         p = td->td_proc;
 1281         ithd = (struct intr_thread *)arg;
 1282         KASSERT(ithd->it_thread == td,
 1283             ("%s: ithread and proc linkage out of sync", __func__));
 1284         ie = ithd->it_event;
 1285         ie->ie_count = 0;
 1286 
 1287         /*
 1288          * As long as we have interrupts outstanding, go through the
 1289          * list of handlers, giving each one a go at it.
 1290          */
 1291         for (;;) {
 1292                 /*
 1293                  * If we are an orphaned thread, then just die.
 1294                  */
 1295                 if (ithd->it_flags & IT_DEAD) {
 1296                         CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
 1297                             p->p_pid, td->td_name);
 1298                         free(ithd, M_ITHREAD);
 1299                         kthread_exit();
 1300                 }
 1301 
 1302                 /*
 1303                  * Service interrupts.  If another interrupt arrives while
 1304                  * we are running, it will set it_need to note that we
 1305                  * should make another pass.
 1306                  */
 1307                 while (ithd->it_need) {
 1308                         /*
 1309                          * This might need a full read and write barrier
 1310                          * to make sure that this write posts before any
 1311                          * of the memory or device accesses in the
 1312                          * handlers.
 1313                          */
 1314                         atomic_store_rel_int(&ithd->it_need, 0);
 1315                         ithread_execute_handlers(p, ie);
 1316                 }
 1317                 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
 1318                 mtx_assert(&Giant, MA_NOTOWNED);
 1319 
 1320                 /*
 1321                  * Processed all our interrupts.  Now get the sched
 1322                  * lock.  This may take a while and it_need may get
 1323                  * set again, so we have to check it again.
 1324                  */
 1325                 thread_lock(td);
 1326                 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) {
 1327                         TD_SET_IWAIT(td);
 1328                         ie->ie_count = 0;
 1329                         mi_switch(SW_VOL | SWT_IWAIT, NULL);
 1330                 }
 1331                 thread_unlock(td);
 1332         }
 1333 }
 1334 
 1335 /*
 1336  * Main interrupt handling body.
 1337  *
 1338  * Input:
 1339  * o ie:                        the event connected to this interrupt.
 1340  * o frame:                     some archs (i.e. i386) pass a frame to some.
 1341  *                              handlers as their main argument.
 1342  * Return value:
 1343  * o 0:                         everything ok.
 1344  * o EINVAL:                    stray interrupt.
 1345  */
 1346 int
 1347 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
 1348 {
 1349         struct intr_handler *ih;
 1350         struct thread *td;
 1351         int error, ret, thread;
 1352 
 1353         td = curthread;
 1354 
 1355         /* An interrupt with no event or handlers is a stray interrupt. */
 1356         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
 1357                 return (EINVAL);
 1358 
 1359         /*
 1360          * Execute fast interrupt handlers directly.
 1361          * To support clock handlers, if a handler registers
 1362          * with a NULL argument, then we pass it a pointer to
 1363          * a trapframe as its argument.
 1364          */
 1365         td->td_intr_nesting_level++;
 1366         thread = 0;
 1367         ret = 0;
 1368         critical_enter();
 1369         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
 1370                 if (ih->ih_filter == NULL) {
 1371                         thread = 1;
 1372                         continue;
 1373                 }
 1374                 CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__,
 1375                     ih->ih_filter, ih->ih_argument == NULL ? frame :
 1376                     ih->ih_argument, ih->ih_name);
 1377                 if (ih->ih_argument == NULL)
 1378                         ret = ih->ih_filter(frame);
 1379                 else
 1380                         ret = ih->ih_filter(ih->ih_argument);
 1381                 KASSERT(ret == FILTER_STRAY ||
 1382                     ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
 1383                     (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
 1384                     ("%s: incorrect return value %#x from %s", __func__, ret,
 1385                     ih->ih_name));
 1386 
 1387                 /* 
 1388                  * Wrapper handler special handling:
 1389                  *
 1390                  * in some particular cases (like pccard and pccbb), 
 1391                  * the _real_ device handler is wrapped in a couple of
 1392                  * functions - a filter wrapper and an ithread wrapper.
 1393                  * In this case (and just in this case), the filter wrapper 
 1394                  * could ask the system to schedule the ithread and mask
 1395                  * the interrupt source if the wrapped handler is composed
 1396                  * of just an ithread handler.
 1397                  *
 1398                  * TODO: write a generic wrapper to avoid people rolling 
 1399                  * their own
 1400                  */
 1401                 if (!thread) {
 1402                         if (ret == FILTER_SCHEDULE_THREAD)
 1403                                 thread = 1;
 1404                 }
 1405         }
 1406 
 1407         if (thread) {
 1408                 if (ie->ie_pre_ithread != NULL)
 1409                         ie->ie_pre_ithread(ie->ie_source);
 1410         } else {
 1411                 if (ie->ie_post_filter != NULL)
 1412                         ie->ie_post_filter(ie->ie_source);
 1413         }
 1414         
 1415         /* Schedule the ithread if needed. */
 1416         if (thread) {
 1417                 error = intr_event_schedule_thread(ie);
 1418 #ifndef XEN             
 1419                 KASSERT(error == 0, ("bad stray interrupt"));
 1420 #else
 1421                 if (error != 0)
 1422                         log(LOG_WARNING, "bad stray interrupt");
 1423 #endif          
 1424         }
 1425         critical_exit();
 1426         td->td_intr_nesting_level--;
 1427         return (0);
 1428 }
 1429 #else
 1430 /*
 1431  * This is the main code for interrupt threads.
 1432  */
 1433 static void
 1434 ithread_loop(void *arg)
 1435 {
 1436         struct intr_thread *ithd;
 1437         struct intr_handler *ih;
 1438         struct intr_event *ie;
 1439         struct thread *td;
 1440         struct proc *p;
 1441         int priv;
 1442 
 1443         td = curthread;
 1444         p = td->td_proc;
 1445         ih = (struct intr_handler *)arg;
 1446         priv = (ih->ih_thread != NULL) ? 1 : 0;
 1447         ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread;
 1448         KASSERT(ithd->it_thread == td,
 1449             ("%s: ithread and proc linkage out of sync", __func__));
 1450         ie = ithd->it_event;
 1451         ie->ie_count = 0;
 1452 
 1453         /*
 1454          * As long as we have interrupts outstanding, go through the
 1455          * list of handlers, giving each one a go at it.
 1456          */
 1457         for (;;) {
 1458                 /*
 1459                  * If we are an orphaned thread, then just die.
 1460                  */
 1461                 if (ithd->it_flags & IT_DEAD) {
 1462                         CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
 1463                             p->p_pid, td->td_name);
 1464                         free(ithd, M_ITHREAD);
 1465                         kthread_exit();
 1466                 }
 1467 
 1468                 /*
 1469                  * Service interrupts.  If another interrupt arrives while
 1470                  * we are running, it will set it_need to note that we
 1471                  * should make another pass.
 1472                  */
 1473                 while (ithd->it_need) {
 1474                         /*
 1475                          * This might need a full read and write barrier
 1476                          * to make sure that this write posts before any
 1477                          * of the memory or device accesses in the
 1478                          * handlers.
 1479                          */
 1480                         atomic_store_rel_int(&ithd->it_need, 0);
 1481                         if (priv)
 1482                                 priv_ithread_execute_handler(p, ih);
 1483                         else 
 1484                                 ithread_execute_handlers(p, ie);
 1485                 }
 1486                 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
 1487                 mtx_assert(&Giant, MA_NOTOWNED);
 1488 
 1489                 /*
 1490                  * Processed all our interrupts.  Now get the sched
 1491                  * lock.  This may take a while and it_need may get
 1492                  * set again, so we have to check it again.
 1493                  */
 1494                 thread_lock(td);
 1495                 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) {
 1496                         TD_SET_IWAIT(td);
 1497                         ie->ie_count = 0;
 1498                         mi_switch(SW_VOL | SWT_IWAIT, NULL);
 1499                 }
 1500                 thread_unlock(td);
 1501         }
 1502 }
 1503 
 1504 /* 
 1505  * Main loop for interrupt filter.
 1506  *
 1507  * Some architectures (i386, amd64 and arm) require the optional frame 
 1508  * parameter, and use it as the main argument for fast handler execution
 1509  * when ih_argument == NULL.
 1510  *
 1511  * Return value:
 1512  * o FILTER_STRAY:              No filter recognized the event, and no
 1513  *                              filter-less handler is registered on this 
 1514  *                              line.
 1515  * o FILTER_HANDLED:            A filter claimed the event and served it.
 1516  * o FILTER_SCHEDULE_THREAD:    No filter claimed the event, but there's at
 1517  *                              least one filter-less handler on this line.
 1518  * o FILTER_HANDLED | 
 1519  *   FILTER_SCHEDULE_THREAD:    A filter claimed the event, and asked for
 1520  *                              scheduling the per-handler ithread.
 1521  *
 1522  * In case an ithread has to be scheduled, in *ithd there will be a 
 1523  * pointer to a struct intr_thread containing the thread to be
 1524  * scheduled.
 1525  */
 1526 
 1527 static int
 1528 intr_filter_loop(struct intr_event *ie, struct trapframe *frame, 
 1529                  struct intr_thread **ithd) 
 1530 {
 1531         struct intr_handler *ih;
 1532         void *arg;
 1533         int ret, thread_only;
 1534 
 1535         ret = 0;
 1536         thread_only = 0;
 1537         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
 1538                 /*
 1539                  * Execute fast interrupt handlers directly.
 1540                  * To support clock handlers, if a handler registers
 1541                  * with a NULL argument, then we pass it a pointer to
 1542                  * a trapframe as its argument.
 1543                  */
 1544                 arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument);
 1545                 
 1546                 CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__,
 1547                      ih->ih_filter, ih->ih_handler, arg, ih->ih_name);
 1548 
 1549                 if (ih->ih_filter != NULL)
 1550                         ret = ih->ih_filter(arg);
 1551                 else {
 1552                         thread_only = 1;
 1553                         continue;
 1554                 }
 1555                 KASSERT(ret == FILTER_STRAY ||
 1556                     ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
 1557                     (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
 1558                     ("%s: incorrect return value %#x from %s", __func__, ret,
 1559                     ih->ih_name));
 1560                 if (ret & FILTER_STRAY)
 1561                         continue;
 1562                 else { 
 1563                         *ithd = ih->ih_thread;
 1564                         return (ret);
 1565                 }
 1566         }
 1567 
 1568         /*
 1569          * No filters handled the interrupt and we have at least
 1570          * one handler without a filter.  In this case, we schedule
 1571          * all of the filter-less handlers to run in the ithread.
 1572          */     
 1573         if (thread_only) {
 1574                 *ithd = ie->ie_thread;
 1575                 return (FILTER_SCHEDULE_THREAD);
 1576         }
 1577         return (FILTER_STRAY);
 1578 }
 1579 
 1580 /*
 1581  * Main interrupt handling body.
 1582  *
 1583  * Input:
 1584  * o ie:                        the event connected to this interrupt.
 1585  * o frame:                     some archs (i.e. i386) pass a frame to some.
 1586  *                              handlers as their main argument.
 1587  * Return value:
 1588  * o 0:                         everything ok.
 1589  * o EINVAL:                    stray interrupt.
 1590  */
 1591 int
 1592 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
 1593 {
 1594         struct intr_thread *ithd;
 1595         struct thread *td;
 1596         int thread;
 1597 
 1598         ithd = NULL;
 1599         td = curthread;
 1600 
 1601         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
 1602                 return (EINVAL);
 1603 
 1604         td->td_intr_nesting_level++;
 1605         thread = 0;
 1606         critical_enter();
 1607         thread = intr_filter_loop(ie, frame, &ithd);    
 1608         if (thread & FILTER_HANDLED) {
 1609                 if (ie->ie_post_filter != NULL)
 1610                         ie->ie_post_filter(ie->ie_source);
 1611         } else {
 1612                 if (ie->ie_pre_ithread != NULL)
 1613                         ie->ie_pre_ithread(ie->ie_source);
 1614         }
 1615         critical_exit();
 1616         
 1617         /* Interrupt storm logic */
 1618         if (thread & FILTER_STRAY) {
 1619                 ie->ie_count++;
 1620                 if (ie->ie_count < intr_storm_threshold)
 1621                         printf("Interrupt stray detection not present\n");
 1622         }
 1623 
 1624         /* Schedule an ithread if needed. */
 1625         if (thread & FILTER_SCHEDULE_THREAD) {
 1626                 if (intr_event_schedule_thread(ie, ithd) != 0)
 1627                         panic("%s: impossible stray interrupt", __func__);
 1628         }
 1629         td->td_intr_nesting_level--;
 1630         return (0);
 1631 }
 1632 #endif
 1633 
 1634 #ifdef DDB
 1635 /*
 1636  * Dump details about an interrupt handler
 1637  */
 1638 static void
 1639 db_dump_intrhand(struct intr_handler *ih)
 1640 {
 1641         int comma;
 1642 
 1643         db_printf("\t%-10s ", ih->ih_name);
 1644         switch (ih->ih_pri) {
 1645         case PI_REALTIME:
 1646                 db_printf("CLK ");
 1647                 break;
 1648         case PI_AV:
 1649                 db_printf("AV  ");
 1650                 break;
 1651         case PI_TTYHIGH:
 1652         case PI_TTYLOW:
 1653                 db_printf("TTY ");
 1654                 break;
 1655         case PI_TAPE:
 1656                 db_printf("TAPE");
 1657                 break;
 1658         case PI_NET:
 1659                 db_printf("NET ");
 1660                 break;
 1661         case PI_DISK:
 1662         case PI_DISKLOW:
 1663                 db_printf("DISK");
 1664                 break;
 1665         case PI_DULL:
 1666                 db_printf("DULL");
 1667                 break;
 1668         default:
 1669                 if (ih->ih_pri >= PI_SOFT)
 1670                         db_printf("SWI ");
 1671                 else
 1672                         db_printf("%4u", ih->ih_pri);
 1673                 break;
 1674         }
 1675         db_printf(" ");
 1676         db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC);
 1677         db_printf("(%p)", ih->ih_argument);
 1678         if (ih->ih_need ||
 1679             (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD |
 1680             IH_MPSAFE)) != 0) {
 1681                 db_printf(" {");
 1682                 comma = 0;
 1683                 if (ih->ih_flags & IH_EXCLUSIVE) {
 1684                         if (comma)
 1685                                 db_printf(", ");
 1686                         db_printf("EXCL");
 1687                         comma = 1;
 1688                 }
 1689                 if (ih->ih_flags & IH_ENTROPY) {
 1690                         if (comma)
 1691                                 db_printf(", ");
 1692                         db_printf("ENTROPY");
 1693                         comma = 1;
 1694                 }
 1695                 if (ih->ih_flags & IH_DEAD) {
 1696                         if (comma)
 1697                                 db_printf(", ");
 1698                         db_printf("DEAD");
 1699                         comma = 1;
 1700                 }
 1701                 if (ih->ih_flags & IH_MPSAFE) {
 1702                         if (comma)
 1703                                 db_printf(", ");
 1704                         db_printf("MPSAFE");
 1705                         comma = 1;
 1706                 }
 1707                 if (ih->ih_need) {
 1708                         if (comma)
 1709                                 db_printf(", ");
 1710                         db_printf("NEED");
 1711                 }
 1712                 db_printf("}");
 1713         }
 1714         db_printf("\n");
 1715 }
 1716 
 1717 /*
 1718  * Dump details about a event.
 1719  */
 1720 void
 1721 db_dump_intr_event(struct intr_event *ie, int handlers)
 1722 {
 1723         struct intr_handler *ih;
 1724         struct intr_thread *it;
 1725         int comma;
 1726 
 1727         db_printf("%s ", ie->ie_fullname);
 1728         it = ie->ie_thread;
 1729         if (it != NULL)
 1730                 db_printf("(pid %d)", it->it_thread->td_proc->p_pid);
 1731         else
 1732                 db_printf("(no thread)");
 1733         if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 ||
 1734             (it != NULL && it->it_need)) {
 1735                 db_printf(" {");
 1736                 comma = 0;
 1737                 if (ie->ie_flags & IE_SOFT) {
 1738                         db_printf("SOFT");
 1739                         comma = 1;
 1740                 }
 1741                 if (ie->ie_flags & IE_ENTROPY) {
 1742                         if (comma)
 1743                                 db_printf(", ");
 1744                         db_printf("ENTROPY");
 1745                         comma = 1;
 1746                 }
 1747                 if (ie->ie_flags & IE_ADDING_THREAD) {
 1748                         if (comma)
 1749                                 db_printf(", ");
 1750                         db_printf("ADDING_THREAD");
 1751                         comma = 1;
 1752                 }
 1753                 if (it != NULL && it->it_need) {
 1754                         if (comma)
 1755                                 db_printf(", ");
 1756                         db_printf("NEED");
 1757                 }
 1758                 db_printf("}");
 1759         }
 1760         db_printf("\n");
 1761 
 1762         if (handlers)
 1763                 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
 1764                     db_dump_intrhand(ih);
 1765 }
 1766 
 1767 /*
 1768  * Dump data about interrupt handlers
 1769  */
 1770 DB_SHOW_COMMAND(intr, db_show_intr)
 1771 {
 1772         struct intr_event *ie;
 1773         int all, verbose;
 1774 
 1775         verbose = index(modif, 'v') != NULL;
 1776         all = index(modif, 'a') != NULL;
 1777         TAILQ_FOREACH(ie, &event_list, ie_list) {
 1778                 if (!all && TAILQ_EMPTY(&ie->ie_handlers))
 1779                         continue;
 1780                 db_dump_intr_event(ie, verbose);
 1781                 if (db_pager_quit)
 1782                         break;
 1783         }
 1784 }
 1785 #endif /* DDB */
 1786 
 1787 /*
 1788  * Start standard software interrupt threads
 1789  */
 1790 static void
 1791 start_softintr(void *dummy)
 1792 {
 1793 
 1794         if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
 1795                 panic("died while creating vm swi ithread");
 1796 }
 1797 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr,
 1798     NULL);
 1799 
 1800 /*
 1801  * Sysctls used by systat and others: hw.intrnames and hw.intrcnt.
 1802  * The data for this machine dependent, and the declarations are in machine
 1803  * dependent code.  The layout of intrnames and intrcnt however is machine
 1804  * independent.
 1805  *
 1806  * We do not know the length of intrcnt and intrnames at compile time, so
 1807  * calculate things at run time.
 1808  */
 1809 static int
 1810 sysctl_intrnames(SYSCTL_HANDLER_ARGS)
 1811 {
 1812         return (sysctl_handle_opaque(oidp, intrnames, eintrnames - intrnames,
 1813            req));
 1814 }
 1815 
 1816 SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD,
 1817     NULL, 0, sysctl_intrnames, "", "Interrupt Names");
 1818 
 1819 static int
 1820 sysctl_intrcnt(SYSCTL_HANDLER_ARGS)
 1821 {
 1822         return (sysctl_handle_opaque(oidp, intrcnt,
 1823             (char *)eintrcnt - (char *)intrcnt, req));
 1824 }
 1825 
 1826 SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD,
 1827     NULL, 0, sysctl_intrcnt, "", "Interrupt Counts");
 1828 
 1829 #ifdef DDB
 1830 /*
 1831  * DDB command to dump the interrupt statistics.
 1832  */
 1833 DB_SHOW_COMMAND(intrcnt, db_show_intrcnt)
 1834 {
 1835         u_long *i;
 1836         char *cp;
 1837 
 1838         cp = intrnames;
 1839         for (i = intrcnt; i != eintrcnt && !db_pager_quit; i++) {
 1840                 if (*cp == '\0')
 1841                         break;
 1842                 if (*i != 0)
 1843                         db_printf("%s\t%lu\n", cp, *i);
 1844                 cp += strlen(cp) + 1;
 1845         }
 1846 }
 1847 #endif

Cache object: 9b2eac34cdb5ea897d059b6f15f1b8b7


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.