The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/local_apic.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
    3  * Copyright (c) 1996, by Steve Passe
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. The name of the developer may NOT be used to endorse or promote products
   12  *    derived from this software without specific prior written permission.
   13  * 3. Neither the name of the author nor the names of any co-contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 /*
   31  * Local APIC support on Pentium and later processors.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD: releng/6.0/sys/i386/i386/local_apic.c 150931 2005-10-04 15:13:06Z jhb $");
   36 
   37 #include "opt_hwpmc_hooks.h"
   38 
   39 #include <sys/param.h>
   40 #include <sys/systm.h>
   41 #include <sys/bus.h>
   42 #include <sys/kernel.h>
   43 #include <sys/pcpu.h>
   44 #include <sys/smp.h>
   45 
   46 #include <vm/vm.h>
   47 #include <vm/pmap.h>
   48 
   49 #include <machine/apicreg.h>
   50 #include <machine/cputypes.h>
   51 #include <machine/frame.h>
   52 #include <machine/intr_machdep.h>
   53 #include <machine/apicvar.h>
   54 #include <machine/md_var.h>
   55 #include <machine/smp.h>
   56 #include <machine/specialreg.h>
   57 
   58 /*
   59  * We can handle up to 60 APICs via our logical cluster IDs, but currently
   60  * the physical IDs on Intel processors up to the Pentium 4 are limited to
   61  * 16.
   62  */
   63 #define MAX_APICID      16
   64 
   65 /* Sanity checks on IDT vectors. */
   66 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
   67 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
   68 CTASSERT(APIC_LOCAL_INTS == 240);
   69 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
   70 
   71 #define LAPIC_TIMER_HZ_DIVIDER          2
   72 #define LAPIC_TIMER_STATHZ_DIVIDER      15
   73 #define LAPIC_TIMER_PROFHZ_DIVIDER      3
   74 
   75 /*
   76  * Support for local APICs.  Local APICs manage interrupts on each
   77  * individual processor as opposed to I/O APICs which receive interrupts
   78  * from I/O devices and then forward them on to the local APICs.
   79  *
   80  * Local APICs can also send interrupts to each other thus providing the
   81  * mechanism for IPIs.
   82  */
   83 
   84 struct lvt {
   85         u_int lvt_edgetrigger:1;
   86         u_int lvt_activehi:1;
   87         u_int lvt_masked:1;
   88         u_int lvt_active:1;
   89         u_int lvt_mode:16;
   90         u_int lvt_vector:8;
   91 };
   92 
   93 struct lapic {
   94         struct lvt la_lvts[LVT_MAX + 1];
   95         u_int la_id:8;
   96         u_int la_cluster:4;
   97         u_int la_cluster_id:2;
   98         u_int la_present:1;
   99         u_long *la_timer_count;
  100         u_long la_hard_ticks;
  101         u_long la_stat_ticks;
  102         u_long la_prof_ticks;
  103 } static lapics[MAX_APICID];
  104 
  105 /* XXX: should thermal be an NMI? */
  106 
  107 /* Global defaults for local APIC LVT entries. */
  108 static struct lvt lvts[LVT_MAX + 1] = {
  109         { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },  /* LINT0: masked ExtINT */
  110         { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },     /* LINT1: NMI */
  111         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT },      /* Timer */
  112         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT },      /* Error */
  113         { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },     /* PMC */
  114         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT },    /* Thermal */
  115 };
  116 
  117 static inthand_t *ioint_handlers[] = {
  118         NULL,                   /* 0 - 31 */
  119         IDTVEC(apic_isr1),      /* 32 - 63 */
  120         IDTVEC(apic_isr2),      /* 64 - 95 */
  121         IDTVEC(apic_isr3),      /* 96 - 127 */
  122         IDTVEC(apic_isr4),      /* 128 - 159 */
  123         IDTVEC(apic_isr5),      /* 160 - 191 */
  124         IDTVEC(apic_isr6),      /* 192 - 223 */
  125         IDTVEC(apic_isr7),      /* 224 - 255 */
  126 };
  127 
  128 static u_int32_t lapic_timer_divisors[] = { 
  129         APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
  130         APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
  131 };
  132 
  133 volatile lapic_t *lapic;
  134 static u_long lapic_timer_divisor, lapic_timer_period, lapic_timer_hz;
  135 
  136 static void     lapic_enable(void);
  137 static void     lapic_timer_enable_intr(void);
  138 static void     lapic_timer_oneshot(u_int count);
  139 static void     lapic_timer_periodic(u_int count);
  140 static void     lapic_timer_set_divisor(u_int divisor);
  141 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
  142 
  143 static uint32_t
  144 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
  145 {
  146         struct lvt *lvt;
  147 
  148         KASSERT(pin <= LVT_MAX, ("%s: pin %u out of range", __func__, pin));
  149         if (la->la_lvts[pin].lvt_active)
  150                 lvt = &la->la_lvts[pin];
  151         else
  152                 lvt = &lvts[pin];
  153 
  154         value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
  155             APIC_LVT_VECTOR);
  156         if (lvt->lvt_edgetrigger == 0)
  157                 value |= APIC_LVT_TM;
  158         if (lvt->lvt_activehi == 0)
  159                 value |= APIC_LVT_IIPP_INTALO;
  160         if (lvt->lvt_masked)
  161                 value |= APIC_LVT_M;
  162         value |= lvt->lvt_mode;
  163         switch (lvt->lvt_mode) {
  164         case APIC_LVT_DM_NMI:
  165         case APIC_LVT_DM_SMI:
  166         case APIC_LVT_DM_INIT:
  167         case APIC_LVT_DM_EXTINT:
  168                 if (!lvt->lvt_edgetrigger) {
  169                         printf("lapic%u: Forcing LINT%u to edge trigger\n",
  170                             la->la_id, pin);
  171                         value |= APIC_LVT_TM;
  172                 }
  173                 /* Use a vector of 0. */
  174                 break;
  175         case APIC_LVT_DM_FIXED:
  176                 value |= lvt->lvt_vector;
  177                 break;
  178         default:
  179                 panic("bad APIC LVT delivery mode: %#x\n", value);
  180         }
  181         return (value);
  182 }
  183 
  184 /*
  185  * Map the local APIC and setup necessary interrupt vectors.
  186  */
  187 void
  188 lapic_init(uintptr_t addr)
  189 {
  190 
  191         /* Map the local APIC and setup the spurious interrupt handler. */
  192         KASSERT(trunc_page(addr) == addr,
  193             ("local APIC not aligned on a page boundary"));
  194         lapic = (lapic_t *)pmap_mapdev(addr, sizeof(lapic_t));
  195         setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_SYS386IGT, SEL_KPL,
  196             GSEL(GCODE_SEL, SEL_KPL));
  197 
  198         /* Perform basic initialization of the BSP's local APIC. */
  199         lapic_enable();
  200 
  201         /* Set BSP's per-CPU local APIC ID. */
  202         PCPU_SET(apic_id, lapic_id());
  203 
  204         /* Local APIC timer interrupt. */
  205         setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_SYS386IGT, SEL_KPL,
  206             GSEL(GCODE_SEL, SEL_KPL));
  207 
  208         /* XXX: error/thermal interrupts */
  209 }
  210 
  211 /*
  212  * Create a local APIC instance.
  213  */
  214 void
  215 lapic_create(u_int apic_id, int boot_cpu)
  216 {
  217         int i;
  218 
  219         if (apic_id >= MAX_APICID) {
  220                 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
  221                 if (boot_cpu)
  222                         panic("Can't ignore BSP");
  223                 return;
  224         }
  225         KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
  226             apic_id));
  227 
  228         /*
  229          * Assume no local LVT overrides and a cluster of 0 and
  230          * intra-cluster ID of 0.
  231          */
  232         lapics[apic_id].la_present = 1;
  233         lapics[apic_id].la_id = apic_id;
  234         for (i = 0; i < LVT_MAX; i++) {
  235                 lapics[apic_id].la_lvts[i] = lvts[i];
  236                 lapics[apic_id].la_lvts[i].lvt_active = 0;
  237         }
  238 
  239 #ifdef SMP
  240         cpu_add(apic_id, boot_cpu);
  241 #endif
  242 }
  243 
  244 /*
  245  * Dump contents of local APIC registers
  246  */
  247 void
  248 lapic_dump(const char* str)
  249 {
  250 
  251         printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
  252         printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n",
  253             lapic->id, lapic->version, lapic->ldr, lapic->dfr);
  254         printf("  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
  255             lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr);
  256         printf("  timer: 0x%08x therm: 0x%08x err: 0x%08x pcm: 0x%08x\n",
  257             lapic->lvt_timer, lapic->lvt_thermal, lapic->lvt_error,
  258             lapic->lvt_pcint);
  259 }
  260 
  261 void
  262 lapic_enable_intr(u_int irq)
  263 {
  264         u_int vector;
  265 
  266         vector = apic_irq_to_idt(irq);
  267         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
  268         KASSERT(ioint_handlers[vector / 32] != NULL,
  269             ("No ISR handler for IRQ %u", irq));
  270         setidt(vector, ioint_handlers[vector / 32], SDT_SYS386IGT, SEL_KPL,
  271             GSEL(GCODE_SEL, SEL_KPL));
  272 }
  273 
  274 void
  275 lapic_setup(void)
  276 {
  277         struct lapic *la;
  278         u_int32_t value, maxlvt;
  279         register_t eflags;
  280         char buf[MAXCOMLEN + 1];
  281 
  282         la = &lapics[lapic_id()];
  283         KASSERT(la->la_present, ("missing APIC structure"));
  284         eflags = intr_disable();
  285         maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
  286 
  287         /* Initialize the TPR to allow all interrupts. */
  288         lapic_set_tpr(0);
  289 
  290         /* Use the cluster model for logical IDs. */
  291         value = lapic->dfr;
  292         value &= ~APIC_DFR_MODEL_MASK;
  293         value |= APIC_DFR_MODEL_CLUSTER;
  294         lapic->dfr = value;
  295 
  296         /* Set this APIC's logical ID. */
  297         value = lapic->ldr;
  298         value &= ~APIC_ID_MASK;
  299         value |= (la->la_cluster << APIC_ID_CLUSTER_SHIFT |
  300             1 << la->la_cluster_id) << APIC_ID_SHIFT;
  301         lapic->ldr = value;
  302 
  303         /* Setup spurious vector and enable the local APIC. */
  304         lapic_enable();
  305 
  306         /* Program LINT[01] LVT entries. */
  307         lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0);
  308         lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1);
  309 #ifdef  HWPMC_HOOKS
  310         /* Program the PMC LVT entry if present. */
  311         if (maxlvt >= LVT_PMC)
  312                 lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint);
  313 #endif
  314 
  315         /* Program timer LVT and setup handler. */
  316         lapic->lvt_timer = lvt_mode(la, LVT_TIMER, lapic->lvt_timer);
  317         snprintf(buf, sizeof(buf), "cpu%d: timer", PCPU_GET(cpuid));
  318         intrcnt_add(buf, &la->la_timer_count);
  319         if (PCPU_GET(cpuid) != 0) {
  320                 KASSERT(lapic_timer_period != 0, ("lapic%u: zero divisor",
  321                     lapic_id()));
  322                 lapic_timer_set_divisor(lapic_timer_divisor);
  323                 lapic_timer_periodic(lapic_timer_period);
  324                 lapic_timer_enable_intr();
  325         }
  326 
  327         /* XXX: Error and thermal LVTs */
  328 
  329         intr_restore(eflags);
  330 }
  331 
  332 /*
  333  * Called by cpu_initclocks() on the BSP to setup the local APIC timer so
  334  * that it can drive hardclock, statclock, and profclock.  This function
  335  * returns true if it is able to use the local APIC timer to drive the
  336  * clocks and false if it is not able.
  337  */
  338 int
  339 lapic_setup_clock(void)
  340 {
  341         u_long value;
  342 
  343         /* Can't drive the timer without a local APIC. */
  344         if (lapic == NULL)
  345                 return (0);
  346 
  347         /* Start off with a divisor of 2 (power on reset default). */
  348         lapic_timer_divisor = 2;
  349 
  350         /* Try to calibrate the local APIC timer. */
  351         do {
  352                 lapic_timer_set_divisor(lapic_timer_divisor);
  353                 lapic_timer_oneshot(APIC_TIMER_MAX_COUNT);
  354                 DELAY(2000000);
  355                 value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer;
  356                 if (value != APIC_TIMER_MAX_COUNT)
  357                         break;
  358                 lapic_timer_divisor <<= 1;
  359         } while (lapic_timer_divisor <= 128);
  360         if (lapic_timer_divisor > 128)
  361                 panic("lapic: Divisor too big");
  362         value /= 2;
  363         if (bootverbose)
  364                 printf("lapic: Divisor %lu, Frequency %lu hz\n",
  365                     lapic_timer_divisor, value);
  366 
  367         /*
  368          * We will drive the timer at a small multiple of hz and drive
  369          * both of the other timers with similarly small but relatively
  370          * prime divisors.
  371          */
  372         lapic_timer_hz = hz * LAPIC_TIMER_HZ_DIVIDER;
  373         stathz = lapic_timer_hz / LAPIC_TIMER_STATHZ_DIVIDER;
  374         profhz = lapic_timer_hz / LAPIC_TIMER_PROFHZ_DIVIDER;
  375         lapic_timer_period = value / lapic_timer_hz;
  376 
  377         /*
  378          * Start up the timer on the BSP.  The APs will kick off their
  379          * timer during lapic_setup().
  380          */
  381         lapic_timer_periodic(lapic_timer_period);
  382         lapic_timer_enable_intr();
  383         return (1);
  384 }
  385 
  386 void
  387 lapic_disable(void)
  388 {
  389         uint32_t value;
  390 
  391         /* Software disable the local APIC. */
  392         value = lapic->svr;
  393         value &= ~APIC_SVR_SWEN;
  394         lapic->svr = value;
  395 }
  396 
  397 static void
  398 lapic_enable(void)
  399 {
  400         u_int32_t value;
  401 
  402         /* Program the spurious vector to enable the local APIC. */
  403         value = lapic->svr;
  404         value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
  405         value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT);
  406         lapic->svr = value;
  407 }
  408 
  409 int
  410 lapic_id(void)
  411 {
  412 
  413         KASSERT(lapic != NULL, ("local APIC is not mapped"));
  414         return (lapic->id >> APIC_ID_SHIFT);
  415 }
  416 
  417 int
  418 lapic_intr_pending(u_int vector)
  419 {
  420         volatile u_int32_t *irr;
  421 
  422         /*
  423          * The IRR registers are an array of 128-bit registers each of
  424          * which only describes 32 interrupts in the low 32 bits..  Thus,
  425          * we divide the vector by 32 to get the 128-bit index.  We then
  426          * multiply that index by 4 to get the equivalent index from
  427          * treating the IRR as an array of 32-bit registers.  Finally, we
  428          * modulus the vector by 32 to determine the individual bit to
  429          * test.
  430          */
  431         irr = &lapic->irr0;
  432         return (irr[(vector / 32) * 4] & 1 << (vector % 32));
  433 }
  434 
  435 void
  436 lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
  437 {
  438         struct lapic *la;
  439 
  440         KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
  441             __func__, apic_id));
  442         KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
  443             __func__, cluster));
  444         KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
  445             ("%s: intra cluster id %u too big", __func__, cluster_id));
  446         la = &lapics[apic_id];
  447         la->la_cluster = cluster;
  448         la->la_cluster_id = cluster_id;
  449 }
  450 
  451 int
  452 lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
  453 {
  454 
  455         if (pin > LVT_MAX)
  456                 return (EINVAL);
  457         if (apic_id == APIC_ID_ALL) {
  458                 lvts[pin].lvt_masked = masked;
  459                 if (bootverbose)
  460                         printf("lapic:");
  461         } else {
  462                 KASSERT(lapics[apic_id].la_present,
  463                     ("%s: missing APIC %u", __func__, apic_id));
  464                 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
  465                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
  466                 if (bootverbose)
  467                         printf("lapic%u:", apic_id);
  468         }
  469         if (bootverbose)
  470                 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
  471         return (0);
  472 }
  473 
  474 int
  475 lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
  476 {
  477         struct lvt *lvt;
  478 
  479         if (pin > LVT_MAX)
  480                 return (EINVAL);
  481         if (apic_id == APIC_ID_ALL) {
  482                 lvt = &lvts[pin];
  483                 if (bootverbose)
  484                         printf("lapic:");
  485         } else {
  486                 KASSERT(lapics[apic_id].la_present,
  487                     ("%s: missing APIC %u", __func__, apic_id));
  488                 lvt = &lapics[apic_id].la_lvts[pin];
  489                 lvt->lvt_active = 1;
  490                 if (bootverbose)
  491                         printf("lapic%u:", apic_id);
  492         }
  493         lvt->lvt_mode = mode;
  494         switch (mode) {
  495         case APIC_LVT_DM_NMI:
  496         case APIC_LVT_DM_SMI:
  497         case APIC_LVT_DM_INIT:
  498         case APIC_LVT_DM_EXTINT:
  499                 lvt->lvt_edgetrigger = 1;
  500                 lvt->lvt_activehi = 1;
  501                 if (mode == APIC_LVT_DM_EXTINT)
  502                         lvt->lvt_masked = 1;
  503                 else
  504                         lvt->lvt_masked = 0;
  505                 break;
  506         default:
  507                 panic("Unsupported delivery mode: 0x%x\n", mode);
  508         }
  509         if (bootverbose) {
  510                 printf(" Routing ");
  511                 switch (mode) {
  512                 case APIC_LVT_DM_NMI:
  513                         printf("NMI");
  514                         break;
  515                 case APIC_LVT_DM_SMI:
  516                         printf("SMI");
  517                         break;
  518                 case APIC_LVT_DM_INIT:
  519                         printf("INIT");
  520                         break;
  521                 case APIC_LVT_DM_EXTINT:
  522                         printf("ExtINT");
  523                         break;
  524                 }
  525                 printf(" -> LINT%u\n", pin);
  526         }
  527         return (0);
  528 }
  529 
  530 int
  531 lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
  532 {
  533 
  534         if (pin > LVT_MAX || pol == INTR_POLARITY_CONFORM)
  535                 return (EINVAL);
  536         if (apic_id == APIC_ID_ALL) {
  537                 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
  538                 if (bootverbose)
  539                         printf("lapic:");
  540         } else {
  541                 KASSERT(lapics[apic_id].la_present,
  542                     ("%s: missing APIC %u", __func__, apic_id));
  543                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
  544                 lapics[apic_id].la_lvts[pin].lvt_activehi =
  545                     (pol == INTR_POLARITY_HIGH);
  546                 if (bootverbose)
  547                         printf("lapic%u:", apic_id);
  548         }
  549         if (bootverbose)
  550                 printf(" LINT%u polarity: %s\n", pin,
  551                     pol == INTR_POLARITY_HIGH ? "high" : "low");
  552         return (0);
  553 }
  554 
  555 int
  556 lapic_set_lvt_triggermode(u_int apic_id, u_int pin, enum intr_trigger trigger)
  557 {
  558 
  559         if (pin > LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
  560                 return (EINVAL);
  561         if (apic_id == APIC_ID_ALL) {
  562                 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
  563                 if (bootverbose)
  564                         printf("lapic:");
  565         } else {
  566                 KASSERT(lapics[apic_id].la_present,
  567                     ("%s: missing APIC %u", __func__, apic_id));
  568                 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
  569                     (trigger == INTR_TRIGGER_EDGE);
  570                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
  571                 if (bootverbose)
  572                         printf("lapic%u:", apic_id);
  573         }
  574         if (bootverbose)
  575                 printf(" LINT%u trigger: %s\n", pin,
  576                     trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
  577         return (0);
  578 }
  579 
  580 /*
  581  * Adjust the TPR of the current CPU so that it blocks all interrupts below
  582  * the passed in vector.
  583  */
  584 void
  585 lapic_set_tpr(u_int vector)
  586 {
  587 #ifdef CHEAP_TPR
  588         lapic->tpr = vector;
  589 #else
  590         u_int32_t tpr;
  591 
  592         tpr = lapic->tpr & ~APIC_TPR_PRIO;
  593         tpr |= vector;
  594         lapic->tpr = tpr;
  595 #endif
  596 }
  597 
  598 void
  599 lapic_eoi(void)
  600 {
  601 
  602         lapic->eoi = 0;
  603 }
  604 
  605 void
  606 lapic_handle_intr(struct intrframe frame)
  607 {
  608         struct intsrc *isrc;
  609 
  610         if (frame.if_vec == -1)
  611                 panic("Couldn't get vector from ISR!");
  612         isrc = intr_lookup_source(apic_idt_to_irq(frame.if_vec));
  613         intr_execute_handlers(isrc, &frame);
  614 }
  615 
  616 void
  617 lapic_handle_timer(struct clockframe frame)
  618 {
  619         struct lapic *la;
  620 
  621         la = &lapics[PCPU_GET(apic_id)];
  622         (*la->la_timer_count)++;
  623         critical_enter();
  624 
  625         /* Fire hardclock at hz. */
  626         la->la_hard_ticks += hz;
  627         if (la->la_hard_ticks >= lapic_timer_hz) {
  628                 la->la_hard_ticks -= lapic_timer_hz;
  629                 if (PCPU_GET(cpuid) == 0)
  630                         hardclock(&frame);
  631                 else
  632                         hardclock_process(&frame);
  633         }
  634 
  635         /* Fire statclock at stathz. */
  636         la->la_stat_ticks += stathz;
  637         if (la->la_stat_ticks >= lapic_timer_hz) {
  638                 la->la_stat_ticks -= lapic_timer_hz;
  639                 statclock(&frame);
  640         }
  641 
  642         /* Fire profclock at profhz, but only when needed. */
  643         la->la_prof_ticks += profhz;
  644         if (la->la_prof_ticks >= lapic_timer_hz) {
  645                 la->la_prof_ticks -= lapic_timer_hz;
  646                 if (profprocs != 0)
  647                         profclock(&frame);
  648         }
  649         critical_exit();
  650 }
  651 
  652 static void
  653 lapic_timer_set_divisor(u_int divisor)
  654 {
  655 
  656         KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
  657         KASSERT(ffs(divisor) <= sizeof(lapic_timer_divisors) /
  658             sizeof(u_int32_t), ("lapic: invalid divisor %u", divisor));
  659         lapic->dcr_timer = lapic_timer_divisors[ffs(divisor) - 1];
  660 }
  661 
  662 static void
  663 lapic_timer_oneshot(u_int count)
  664 {
  665         u_int32_t value;
  666 
  667         value = lapic->lvt_timer;
  668         value &= ~APIC_LVTT_TM;
  669         value |= APIC_LVTT_TM_ONE_SHOT;
  670         lapic->lvt_timer = value;
  671         lapic->icr_timer = count;
  672 }
  673 
  674 static void
  675 lapic_timer_periodic(u_int count)
  676 {
  677         u_int32_t value;
  678 
  679         value = lapic->lvt_timer;
  680         value &= ~APIC_LVTT_TM;
  681         value |= APIC_LVTT_TM_PERIODIC;
  682         lapic->lvt_timer = value;
  683         lapic->icr_timer = count;
  684 }
  685 
  686 static void
  687 lapic_timer_enable_intr(void)
  688 {
  689         u_int32_t value;
  690 
  691         value = lapic->lvt_timer;
  692         value &= ~APIC_LVT_M;
  693         lapic->lvt_timer = value;
  694 }
  695 
  696 /* Translate between IDT vectors and IRQ vectors. */
  697 u_int
  698 apic_irq_to_idt(u_int irq)
  699 {
  700         u_int vector;
  701 
  702         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
  703         vector = irq + APIC_IO_INTS;
  704         if (vector >= IDT_SYSCALL)
  705                 vector++;
  706         return (vector);
  707 }
  708 
  709 u_int
  710 apic_idt_to_irq(u_int vector)
  711 {
  712 
  713         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
  714             vector <= APIC_IO_INTS + NUM_IO_INTS,
  715             ("Vector %u does not map to an IRQ line", vector));
  716         if (vector > IDT_SYSCALL)
  717                 vector--;
  718         return (vector - APIC_IO_INTS);
  719 }
  720 
  721 /*
  722  * APIC probing support code.  This includes code to manage enumerators.
  723  */
  724 
  725 static SLIST_HEAD(, apic_enumerator) enumerators =
  726         SLIST_HEAD_INITIALIZER(enumerators);
  727 static struct apic_enumerator *best_enum;
  728         
  729 void
  730 apic_register_enumerator(struct apic_enumerator *enumerator)
  731 {
  732 #ifdef INVARIANTS
  733         struct apic_enumerator *apic_enum;
  734 
  735         SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
  736                 if (apic_enum == enumerator)
  737                         panic("%s: Duplicate register of %s", __func__,
  738                             enumerator->apic_name);
  739         }
  740 #endif
  741         SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
  742 }
  743 
  744 /*
  745  * Probe the APIC enumerators, enumerate CPUs, and initialize the
  746  * local APIC.
  747  */
  748 static void
  749 apic_init(void *dummy __unused)
  750 {
  751         struct apic_enumerator *enumerator;
  752         uint64_t apic_base;
  753         int retval, best;
  754 
  755         /* We only support built in local APICs. */
  756         if (!(cpu_feature & CPUID_APIC))
  757                 return;
  758 
  759         /* Don't probe if APIC mode is disabled. */
  760         if (resource_disabled("apic", 0))
  761                 return;
  762 
  763         /* First, probe all the enumerators to find the best match. */
  764         best_enum = NULL;
  765         best = 0;
  766         SLIST_FOREACH(enumerator, &enumerators, apic_next) {
  767                 retval = enumerator->apic_probe();
  768                 if (retval > 0)
  769                         continue;
  770                 if (best_enum == NULL || best < retval) {
  771                         best_enum = enumerator;
  772                         best = retval;
  773                 }
  774         }
  775         if (best_enum == NULL) {
  776                 if (bootverbose)
  777                         printf("APIC: Could not find any APICs.\n");
  778                 return;
  779         }
  780 
  781         if (bootverbose)
  782                 printf("APIC: Using the %s enumerator.\n",
  783                     best_enum->apic_name);
  784 
  785         /*
  786          * To work around an errata, we disable the local APIC on some
  787          * CPUs during early startup.  We need to turn the local APIC back
  788          * on on such CPUs now.
  789          */
  790         if (cpu == CPU_686 && strcmp(cpu_vendor, "GenuineIntel") == 0 &&
  791             (cpu_id & 0xff0) == 0x610) {
  792                 apic_base = rdmsr(MSR_APICBASE);
  793                 apic_base |= APICBASE_ENABLED;
  794                 wrmsr(MSR_APICBASE, apic_base);
  795         }
  796 
  797         /* Second, probe the CPU's in the system. */
  798         retval = best_enum->apic_probe_cpus();
  799         if (retval != 0)
  800                 printf("%s: Failed to probe CPUs: returned %d\n",
  801                     best_enum->apic_name, retval);
  802 
  803         /* Third, initialize the local APIC. */
  804         retval = best_enum->apic_setup_local();
  805         if (retval != 0)
  806                 printf("%s: Failed to setup the local APIC: returned %d\n",
  807                     best_enum->apic_name, retval);
  808 #ifdef SMP
  809         /* Last, setup the cpu topology now that we have probed CPUs */
  810         mp_topology();
  811 #endif
  812 }
  813 SYSINIT(apic_init, SI_SUB_CPU, SI_ORDER_FIRST, apic_init, NULL)
  814 
  815 /*
  816  * Setup the I/O APICs.
  817  */
  818 static void
  819 apic_setup_io(void *dummy __unused)
  820 {
  821         int retval;
  822 
  823         if (best_enum == NULL)
  824                 return;
  825         retval = best_enum->apic_setup_io();
  826         if (retval != 0)
  827                 printf("%s: Failed to setup I/O APICs: returned %d\n",
  828                     best_enum->apic_name, retval);
  829 
  830         /*
  831          * Finish setting up the local APIC on the BSP once we know how to
  832          * properly program the LINT pins.
  833          */
  834         lapic_setup();
  835         if (bootverbose)
  836                 lapic_dump("BSP");
  837 }
  838 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_SECOND, apic_setup_io, NULL)
  839 
  840 #ifdef SMP
  841 /*
  842  * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
  843  * private to the sys/i386 code.  The public interface for the rest of the
  844  * kernel is defined in mp_machdep.c.
  845  */
  846 int
  847 lapic_ipi_wait(int delay)
  848 {
  849         int x, incr;
  850 
  851         /*
  852          * Wait delay loops for IPI to be sent.  This is highly bogus
  853          * since this is sensitive to CPU clock speed.  If delay is
  854          * -1, we wait forever.
  855          */
  856         if (delay == -1) {
  857                 incr = 0;
  858                 delay = 1;
  859         } else
  860                 incr = 1;
  861         for (x = 0; x < delay; x += incr) {
  862                 if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE)
  863                         return (1);
  864                 ia32_pause();
  865         }
  866         return (0);
  867 }
  868 
  869 void
  870 lapic_ipi_raw(register_t icrlo, u_int dest)
  871 {
  872         register_t value, eflags;
  873 
  874         /* XXX: Need more sanity checking of icrlo? */
  875         KASSERT(lapic != NULL, ("%s called too early", __func__));
  876         KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
  877             ("%s: invalid dest field", __func__));
  878         KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
  879             ("%s: reserved bits set in ICR LO register", __func__));
  880 
  881         /* Set destination in ICR HI register if it is being used. */
  882         eflags = intr_disable();
  883         if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
  884                 value = lapic->icr_hi;
  885                 value &= ~APIC_ID_MASK;
  886                 value |= dest << APIC_ID_SHIFT;
  887                 lapic->icr_hi = value;
  888         }
  889 
  890         /* Program the contents of the IPI and dispatch it. */
  891         value = lapic->icr_lo;
  892         value &= APIC_ICRLO_RESV_MASK;
  893         value |= icrlo;
  894         lapic->icr_lo = value;
  895         intr_restore(eflags);
  896 }
  897 
  898 #define BEFORE_SPIN     1000000
  899 #ifdef DETECT_DEADLOCK
  900 #define AFTER_SPIN      1000
  901 #endif
  902 
  903 void
  904 lapic_ipi_vectored(u_int vector, int dest)
  905 {
  906         register_t icrlo, destfield;
  907 
  908         KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
  909             ("%s: invalid vector %d", __func__, vector));
  910 
  911         icrlo = vector | APIC_DELMODE_FIXED | APIC_DESTMODE_PHY |
  912             APIC_LEVEL_DEASSERT | APIC_TRIGMOD_EDGE;
  913         destfield = 0;
  914         switch (dest) {
  915         case APIC_IPI_DEST_SELF:
  916                 icrlo |= APIC_DEST_SELF;
  917                 break;
  918         case APIC_IPI_DEST_ALL:
  919                 icrlo |= APIC_DEST_ALLISELF;
  920                 break;
  921         case APIC_IPI_DEST_OTHERS:
  922                 icrlo |= APIC_DEST_ALLESELF;
  923                 break;
  924         default:
  925                 KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
  926                     ("%s: invalid destination 0x%x", __func__, dest));
  927                 destfield = dest;
  928         }
  929 
  930         /* Wait for an earlier IPI to finish. */
  931         if (!lapic_ipi_wait(BEFORE_SPIN)) {
  932                 if (panicstr != NULL)
  933                         return;
  934                 else
  935                         panic("APIC: Previous IPI is stuck");
  936         }
  937 
  938         lapic_ipi_raw(icrlo, destfield);
  939 
  940 #ifdef DETECT_DEADLOCK
  941         /* Wait for IPI to be delivered. */
  942         if (!lapic_ipi_wait(AFTER_SPIN)) {
  943 #ifdef needsattention
  944                 /*
  945                  * XXX FIXME:
  946                  *
  947                  * The above function waits for the message to actually be
  948                  * delivered.  It breaks out after an arbitrary timeout
  949                  * since the message should eventually be delivered (at
  950                  * least in theory) and that if it wasn't we would catch
  951                  * the failure with the check above when the next IPI is
  952                  * sent.
  953                  *
  954                  * We could skip this wait entirely, EXCEPT it probably
  955                  * protects us from other routines that assume that the
  956                  * message was delivered and acted upon when this function
  957                  * returns.
  958                  */
  959                 printf("APIC: IPI might be stuck\n");
  960 #else /* !needsattention */
  961                 /* Wait until mesage is sent without a timeout. */
  962                 while (lapic->icr_lo & APIC_DELSTAT_PEND)
  963                         ia32_pause();
  964 #endif /* needsattention */
  965         }
  966 #endif /* DETECT_DEADLOCK */
  967 }
  968 #endif /* SMP */

Cache object: dc079307ce51ba5852373e1612b86bf1


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.