The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/intr_machdep.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 3. Neither the name of the author nor the names of any co-contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  *
   29  * $FreeBSD$
   30  */
   31 
   32 /*
   33  * Machine dependent interrupt code for amd64.  For amd64, we have to
   34  * deal with different PICs.  Thus, we use the passed in vector to lookup
   35  * an interrupt source associated with that vector.  The interrupt source
   36  * describes which PIC the source belongs to and includes methods to handle
   37  * that source.
   38  */
   39 
   40 #include "opt_atpic.h"
   41 #include "opt_ddb.h"
   42 
   43 #include <sys/param.h>
   44 #include <sys/bus.h>
   45 #include <sys/interrupt.h>
   46 #include <sys/ktr.h>
   47 #include <sys/kernel.h>
   48 #include <sys/lock.h>
   49 #include <sys/mutex.h>
   50 #include <sys/proc.h>
   51 #include <sys/syslog.h>
   52 #include <sys/systm.h>
   53 #include <sys/sx.h>
   54 #include <machine/clock.h>
   55 #include <machine/intr_machdep.h>
   56 #include <machine/smp.h>
   57 #ifdef DDB
   58 #include <ddb/ddb.h>
   59 #endif
   60 
   61 #ifndef DEV_ATPIC
   62 #include <machine/segments.h>
   63 #include <machine/frame.h>
   64 #include <dev/ic/i8259.h>
   65 #include <amd64/isa/icu.h>
   66 #include <amd64/isa/isa.h>
   67 #endif
   68 
   69 #define MAX_STRAY_LOG   5
   70 
   71 typedef void (*mask_fn)(void *);
   72 
   73 static int intrcnt_index;
   74 static struct intsrc *interrupt_sources[NUM_IO_INTS];
   75 static struct sx intr_table_lock;
   76 static struct mtx intrcnt_lock;
   77 static STAILQ_HEAD(, pic) pics;
   78 
   79 #ifdef INTR_FILTER
   80 static void intr_eoi_src(void *arg);
   81 static void intr_disab_eoi_src(void *arg);
   82 static void intr_event_stray(void *cookie);
   83 #endif
   84 
   85 #ifdef SMP
   86 static int assign_cpu;
   87 
   88 static void     intr_assign_next_cpu(struct intsrc *isrc);
   89 #endif
   90 
   91 static void     intr_init(void *__dummy);
   92 static int      intr_pic_registered(struct pic *pic);
   93 static void     intrcnt_setname(const char *name, int index);
   94 static void     intrcnt_updatename(struct intsrc *is);
   95 static void     intrcnt_register(struct intsrc *is);
   96 
   97 static int
   98 intr_pic_registered(struct pic *pic)
   99 {
  100         struct pic *p;
  101 
  102         STAILQ_FOREACH(p, &pics, pics) {
  103                 if (p == pic)
  104                         return (1);
  105         }
  106         return (0);
  107 }
  108 
  109 /*
  110  * Register a new interrupt controller (PIC).  This is to support suspend
  111  * and resume where we suspend/resume controllers rather than individual
  112  * sources.  This also allows controllers with no active sources (such as
  113  * 8259As in a system using the APICs) to participate in suspend and resume.
  114  */
  115 int
  116 intr_register_pic(struct pic *pic)
  117 {
  118         int error;
  119 
  120         sx_xlock(&intr_table_lock);
  121         if (intr_pic_registered(pic))
  122                 error = EBUSY;
  123         else {
  124                 STAILQ_INSERT_TAIL(&pics, pic, pics);
  125                 error = 0;
  126         }
  127         sx_xunlock(&intr_table_lock);
  128         return (error);
  129 }
  130 
  131 /*
  132  * Register a new interrupt source with the global interrupt system.
  133  * The global interrupts need to be disabled when this function is
  134  * called.
  135  */
  136 int
  137 intr_register_source(struct intsrc *isrc)
  138 {
  139         int error, vector;
  140 
  141         KASSERT(intr_pic_registered(isrc->is_pic), ("unregistered PIC"));
  142         vector = isrc->is_pic->pic_vector(isrc);
  143         if (interrupt_sources[vector] != NULL)
  144                 return (EEXIST);
  145 #ifdef INTR_FILTER
  146         error = intr_event_create(&isrc->is_event, isrc, 0,
  147             (mask_fn)isrc->is_pic->pic_enable_source,
  148             intr_eoi_src, intr_disab_eoi_src, "irq%d:", vector);
  149 #else
  150         error = intr_event_create(&isrc->is_event, isrc, 0,
  151             (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector);
  152 #endif
  153         if (error)
  154                 return (error);
  155         sx_xlock(&intr_table_lock);
  156         if (interrupt_sources[vector] != NULL) {
  157                 sx_xunlock(&intr_table_lock);
  158                 intr_event_destroy(isrc->is_event);
  159                 return (EEXIST);
  160         }
  161         intrcnt_register(isrc);
  162         interrupt_sources[vector] = isrc;
  163         isrc->is_handlers = 0;
  164         sx_xunlock(&intr_table_lock);
  165         return (0);
  166 }
  167 
  168 struct intsrc *
  169 intr_lookup_source(int vector)
  170 {
  171 
  172         return (interrupt_sources[vector]);
  173 }
  174 
  175 int
  176 intr_add_handler(const char *name, int vector, driver_filter_t filter, 
  177     driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)    
  178 {
  179         struct intsrc *isrc;
  180         int error;
  181 
  182         isrc = intr_lookup_source(vector);
  183         if (isrc == NULL)
  184                 return (EINVAL);
  185         error = intr_event_add_handler(isrc->is_event, name, filter, handler,
  186             arg, intr_priority(flags), flags, cookiep);
  187         if (error == 0) {
  188                 sx_xlock(&intr_table_lock);
  189                 intrcnt_updatename(isrc);
  190                 isrc->is_handlers++;
  191                 if (isrc->is_handlers == 1) {
  192 #ifdef SMP
  193                         if (assign_cpu)
  194                                 intr_assign_next_cpu(isrc);
  195 #endif
  196                         isrc->is_pic->pic_enable_intr(isrc);
  197                         isrc->is_pic->pic_enable_source(isrc);
  198                 }
  199                 sx_xunlock(&intr_table_lock);
  200         }
  201         return (error);
  202 }
  203 
  204 int
  205 intr_remove_handler(void *cookie)
  206 {
  207         struct intsrc *isrc;
  208         int error;
  209 
  210         isrc = intr_handler_source(cookie);
  211         error = intr_event_remove_handler(cookie);
  212         if (error == 0) {
  213                 sx_xlock(&intr_table_lock);
  214                 isrc->is_handlers--;
  215                 if (isrc->is_handlers == 0) {
  216                         isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI);
  217                         isrc->is_pic->pic_disable_intr(isrc);
  218                 }
  219                 intrcnt_updatename(isrc);
  220                 sx_xunlock(&intr_table_lock);
  221         }
  222         return (error);
  223 }
  224 
  225 int
  226 intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol)
  227 {
  228         struct intsrc *isrc;
  229 
  230         isrc = intr_lookup_source(vector);
  231         if (isrc == NULL)
  232                 return (EINVAL);
  233         return (isrc->is_pic->pic_config_intr(isrc, trig, pol));
  234 }
  235 
  236 #ifdef INTR_FILTER
  237 void
  238 intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame)
  239 {
  240         struct thread *td;
  241         struct intr_event *ie;
  242         int vector;
  243 
  244         td = curthread;
  245 
  246         /*
  247          * We count software interrupts when we process them.  The
  248          * code here follows previous practice, but there's an
  249          * argument for counting hardware interrupts when they're
  250          * processed too.
  251          */
  252         (*isrc->is_count)++;
  253         PCPU_INC(cnt.v_intr);
  254 
  255         ie = isrc->is_event;
  256 
  257         /*
  258          * XXX: We assume that IRQ 0 is only used for the ISA timer
  259          * device (clk).
  260          */
  261         vector = isrc->is_pic->pic_vector(isrc);
  262         if (vector == 0)
  263                 clkintr_pending = 1;
  264 
  265         if (intr_event_handle(ie, frame) != 0)
  266                 intr_event_stray(isrc);
  267 }
  268 
  269 static void
  270 intr_event_stray(void *cookie)
  271 {
  272         struct intsrc *isrc;
  273 
  274         isrc = cookie;
  275         /*
  276          * For stray interrupts, mask and EOI the source, bump the
  277          * stray count, and log the condition.
  278          */
  279         isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
  280         (*isrc->is_straycount)++;
  281         if (*isrc->is_straycount < MAX_STRAY_LOG)
  282                 log(LOG_ERR, "stray irq%d\n", isrc->is_pic->pic_vector(isrc));
  283         else if (*isrc->is_straycount == MAX_STRAY_LOG)
  284                 log(LOG_CRIT,
  285                     "too many stray irq %d's: not logging anymore\n",
  286                     isrc->is_pic->pic_vector(isrc));
  287 }
  288 
  289 static void
  290 intr_eoi_src(void *arg)
  291 {
  292         struct intsrc *isrc;
  293 
  294         isrc = arg;
  295         isrc->is_pic->pic_eoi_source(isrc);
  296 }
  297 
  298 static void
  299 intr_disab_eoi_src(void *arg)
  300 {
  301         struct intsrc *isrc;
  302 
  303         isrc = arg;
  304         isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
  305 }
  306 #else
  307 void
  308 intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame)
  309 {
  310         struct thread *td;
  311         struct intr_event *ie;
  312         struct intr_handler *ih;
  313         int error, vector, thread, ret;
  314 
  315         td = curthread;
  316 
  317         /*
  318          * We count software interrupts when we process them.  The
  319          * code here follows previous practice, but there's an
  320          * argument for counting hardware interrupts when they're
  321          * processed too.
  322          */
  323         (*isrc->is_count)++;
  324         PCPU_INC(cnt.v_intr);
  325 
  326         ie = isrc->is_event;
  327 
  328         /*
  329          * XXX: We assume that IRQ 0 is only used for the ISA timer
  330          * device (clk).
  331          */
  332         vector = isrc->is_pic->pic_vector(isrc);
  333         if (vector == 0)
  334                 clkintr_pending = 1;
  335 
  336         /*
  337          * For stray interrupts, mask and EOI the source, bump the
  338          * stray count, and log the condition.
  339          */
  340         if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) {
  341                 isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
  342                 (*isrc->is_straycount)++;
  343                 if (*isrc->is_straycount < MAX_STRAY_LOG)
  344                         log(LOG_ERR, "stray irq%d\n", vector);
  345                 else if (*isrc->is_straycount == MAX_STRAY_LOG)
  346                         log(LOG_CRIT,
  347                             "too many stray irq %d's: not logging anymore\n",
  348                             vector);
  349                 return;
  350         }
  351 
  352         /*
  353          * Execute fast interrupt handlers directly.
  354          * To support clock handlers, if a handler registers
  355          * with a NULL argument, then we pass it a pointer to
  356          * a trapframe as its argument.
  357          */
  358         td->td_intr_nesting_level++;
  359         ret = 0;
  360         thread = 0;
  361         critical_enter();
  362         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
  363                 if (ih->ih_filter == NULL) {
  364                         thread = 1;
  365                         continue;
  366                 }
  367                 CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__,
  368                     ih->ih_filter, ih->ih_argument == NULL ? frame :
  369                     ih->ih_argument, ih->ih_name);
  370                 if (ih->ih_argument == NULL)
  371                         ret = ih->ih_filter(frame);
  372                 else
  373                         ret = ih->ih_filter(ih->ih_argument);
  374                 /*
  375                  * Wrapper handler special case: see
  376                  * i386/intr_machdep.c::intr_execute_handlers()
  377                  */
  378                 if (!thread) {
  379                         if (ret == FILTER_SCHEDULE_THREAD)
  380                                 thread = 1;
  381                 }
  382         }
  383 
  384         /*
  385          * If there are any threaded handlers that need to run,
  386          * mask the source as well as sending it an EOI.  Otherwise,
  387          * just send it an EOI but leave it unmasked.
  388          */
  389         if (thread)
  390                 isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
  391         else
  392                 isrc->is_pic->pic_eoi_source(isrc);
  393 
  394         /* Schedule the ithread if needed. */
  395         if (thread) {
  396                 error = intr_event_schedule_thread(ie);
  397                 KASSERT(error == 0, ("bad stray interrupt"));
  398         }
  399         critical_exit();
  400         td->td_intr_nesting_level--;
  401 }
  402 #endif
  403 
  404 void
  405 intr_resume(void)
  406 {
  407         struct pic *pic;
  408 
  409 #ifndef DEV_ATPIC
  410         atpic_reset();
  411 #endif
  412         sx_xlock(&intr_table_lock);
  413         STAILQ_FOREACH(pic, &pics, pics) {
  414                 if (pic->pic_resume != NULL)
  415                         pic->pic_resume(pic);
  416         }
  417         sx_xunlock(&intr_table_lock);
  418 }
  419 
  420 void
  421 intr_suspend(void)
  422 {
  423         struct pic *pic;
  424 
  425         sx_xlock(&intr_table_lock);
  426         STAILQ_FOREACH(pic, &pics, pics) {
  427                 if (pic->pic_suspend != NULL)
  428                         pic->pic_suspend(pic);
  429         }
  430         sx_xunlock(&intr_table_lock);
  431 }
  432 
  433 static void
  434 intrcnt_setname(const char *name, int index)
  435 {
  436 
  437         snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s",
  438             MAXCOMLEN, name);
  439 }
  440 
  441 static void
  442 intrcnt_updatename(struct intsrc *is)
  443 {
  444 
  445         intrcnt_setname(is->is_event->ie_fullname, is->is_index);
  446 }
  447 
  448 static void
  449 intrcnt_register(struct intsrc *is)
  450 {
  451         char straystr[MAXCOMLEN + 1];
  452 
  453         KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__));
  454         mtx_lock_spin(&intrcnt_lock);
  455         is->is_index = intrcnt_index;
  456         intrcnt_index += 2;
  457         snprintf(straystr, MAXCOMLEN + 1, "stray irq%d",
  458             is->is_pic->pic_vector(is));
  459         intrcnt_updatename(is);
  460         is->is_count = &intrcnt[is->is_index];
  461         intrcnt_setname(straystr, is->is_index + 1);
  462         is->is_straycount = &intrcnt[is->is_index + 1];
  463         mtx_unlock_spin(&intrcnt_lock);
  464 }
  465 
  466 void
  467 intrcnt_add(const char *name, u_long **countp)
  468 {
  469 
  470         mtx_lock_spin(&intrcnt_lock);
  471         *countp = &intrcnt[intrcnt_index];
  472         intrcnt_setname(name, intrcnt_index);
  473         intrcnt_index++;
  474         mtx_unlock_spin(&intrcnt_lock);
  475 }
  476 
  477 static void
  478 intr_init(void *dummy __unused)
  479 {
  480 
  481         intrcnt_setname("???", 0);
  482         intrcnt_index = 1;
  483         STAILQ_INIT(&pics);
  484         sx_init(&intr_table_lock, "intr sources");
  485         mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
  486 }
  487 SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL)
  488 
  489 #ifndef DEV_ATPIC
  490 /* Initialize the two 8259A's to a known-good shutdown state. */
  491 void
  492 atpic_reset(void)
  493 {
  494 
  495         outb(IO_ICU1, ICW1_RESET | ICW1_IC4);
  496         outb(IO_ICU1 + ICU_IMR_OFFSET, IDT_IO_INTS);
  497         outb(IO_ICU1 + ICU_IMR_OFFSET, 1 << 2);
  498         outb(IO_ICU1 + ICU_IMR_OFFSET, ICW4_8086);
  499         outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff);
  500         outb(IO_ICU1, OCW3_SEL | OCW3_RR);
  501 
  502         outb(IO_ICU2, ICW1_RESET | ICW1_IC4);
  503         outb(IO_ICU2 + ICU_IMR_OFFSET, IDT_IO_INTS + 8);
  504         outb(IO_ICU2 + ICU_IMR_OFFSET, 2);
  505         outb(IO_ICU2 + ICU_IMR_OFFSET, ICW4_8086);
  506         outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff);
  507         outb(IO_ICU2, OCW3_SEL | OCW3_RR);
  508 }
  509 #endif
  510 
  511 #ifdef DDB
  512 /*
  513  * Dump data about interrupt handlers
  514  */
  515 DB_SHOW_COMMAND(irqs, db_show_irqs)
  516 {
  517         struct intsrc **isrc;
  518         int i, verbose;
  519 
  520         if (strcmp(modif, "v") == 0)
  521                 verbose = 1;
  522         else
  523                 verbose = 0;
  524         isrc = interrupt_sources;
  525         for (i = 0; i < NUM_IO_INTS && !db_pager_quit; i++, isrc++)
  526                 if (*isrc != NULL)
  527                         db_dump_intr_event((*isrc)->is_event, verbose);
  528 }
  529 #endif
  530 
  531 #ifdef SMP
  532 /*
  533  * Support for balancing interrupt sources across CPUs.  For now we just
  534  * allocate CPUs round-robin.
  535  */
  536 
  537 /* The BSP is always a valid target. */
  538 static cpumask_t intr_cpus = (1 << 0);
  539 static int current_cpu, num_cpus = 1;
  540 
  541 static void
  542 intr_assign_next_cpu(struct intsrc *isrc)
  543 {
  544         struct pic *pic;
  545         u_int apic_id;
  546 
  547         /*
  548          * Assign this source to a local APIC in a round-robin fashion.
  549          */
  550         pic = isrc->is_pic;
  551         apic_id = cpu_apic_ids[current_cpu];
  552         pic->pic_assign_cpu(isrc, apic_id);
  553         do {
  554                 current_cpu++;
  555                 if (current_cpu >= num_cpus)
  556                         current_cpu = 0;
  557         } while (!(intr_cpus & (1 << current_cpu)));
  558 }
  559 
  560 /*
  561  * Add a CPU to our mask of valid CPUs that can be destinations of
  562  * interrupts.
  563  */
  564 void
  565 intr_add_cpu(u_int cpu)
  566 {
  567 
  568         if (cpu >= MAXCPU)
  569                 panic("%s: Invalid CPU ID", __func__);
  570         if (bootverbose)
  571                 printf("INTR: Adding local APIC %d as a target\n",
  572                     cpu_apic_ids[cpu]);
  573 
  574         intr_cpus |= (1 << cpu);
  575         num_cpus++;
  576 }
  577 
  578 /*
  579  * Distribute all the interrupt sources among the available CPUs once the
  580  * AP's have been launched.
  581  */
  582 static void
  583 intr_shuffle_irqs(void *arg __unused)
  584 {
  585         struct intsrc *isrc;
  586         int i;
  587 
  588         /* Don't bother on UP. */
  589         if (num_cpus <= 1)
  590                 return;
  591 
  592         /* Round-robin assign a CPU to each enabled source. */
  593         sx_xlock(&intr_table_lock);
  594         assign_cpu = 1;
  595         for (i = 0; i < NUM_IO_INTS; i++) {
  596                 isrc = interrupt_sources[i];
  597                 if (isrc != NULL && isrc->is_handlers > 0)
  598                         intr_assign_next_cpu(isrc);
  599         }
  600         sx_xunlock(&intr_table_lock);
  601 }
  602 SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL)
  603 #endif

Cache object: 781e2ac7cb9b4c9f814fce109adf5886


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.