Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/osfmk/i386/mp.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * The contents of this file constitute Original Code as defined in and 7 * are subject to the Apple Public Source License Version 1.1 (the 8 * "License"). You may not use this file except in compliance with the 9 * License. Please obtain a copy of the License at 10 * http://www.apple.com/publicsource and read it before using this file. 11 * 12 * This Original Code and all software distributed under the License are 13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER 14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the 17 * License for the specific language governing rights and limitations 18 * under the License. 19 * 20 * @APPLE_LICENSE_HEADER_END@ 21 */ 22 /* 23 * @OSF_COPYRIGHT@ 24 */ 25 26 #include <mach_rt.h> 27 #include <mach_kdb.h> 28 #include <mach_kdp.h> 29 #include <mach_ldebug.h> 30 #include <gprof.h> 31 32 #include <mach/mach_types.h> 33 #include <mach/kern_return.h> 34 35 #include <kern/kern_types.h> 36 #include <kern/startup.h> 37 #include <kern/processor.h> 38 #include <kern/cpu_number.h> 39 #include <kern/cpu_data.h> 40 #include <kern/assert.h> 41 #include <kern/machine.h> 42 43 #include <vm/vm_map.h> 44 #include <vm/vm_kern.h> 45 46 #include <profiling/profile-mk.h> 47 48 #include <i386/mp.h> 49 #include <i386/mp_events.h> 50 #include <i386/mp_slave_boot.h> 51 #include <i386/apic.h> 52 #include <i386/ipl.h> 53 #include <i386/fpu.h> 54 #include <i386/pio.h> 55 #include <i386/cpuid.h> 56 #include <i386/proc_reg.h> 57 #include <i386/machine_cpu.h> 58 #include <i386/misc_protos.h> 59 #include <i386/mtrr.h> 60 #include <i386/postcode.h> 61 #include <i386/perfmon.h> 62 #include <i386/cpu_threads.h> 63 #include <i386/mp_desc.h> 64 65 #if MP_DEBUG 66 #define PAUSE delay(1000000) 67 #define DBG(x...) kprintf(x) 68 #else 69 #define DBG(x...) 70 #define PAUSE 71 #endif /* MP_DEBUG */ 72 73 /* 74 * By default, use high vectors to leave vector space for systems 75 * with multiple I/O APIC's. However some systems that boot with 76 * local APIC disabled will hang in SMM when vectors greater than 77 * 0x5F are used. Those systems are not expected to have I/O APIC 78 * so 16 (0x50 - 0x40) vectors for legacy PIC support is perfect. 79 */ 80 #define LAPIC_DEFAULT_INTERRUPT_BASE 0xD0 81 #define LAPIC_REDUCED_INTERRUPT_BASE 0x50 82 /* 83 * Specific lapic interrupts are relative to this base: 84 */ 85 #define LAPIC_PERFCNT_INTERRUPT 0xB 86 #define LAPIC_TIMER_INTERRUPT 0xC 87 #define LAPIC_SPURIOUS_INTERRUPT 0xD 88 #define LAPIC_INTERPROCESSOR_INTERRUPT 0xE 89 #define LAPIC_ERROR_INTERRUPT 0xF 90 91 /* Initialize lapic_id so cpu_number() works on non SMP systems */ 92 unsigned long lapic_id_initdata = 0; 93 unsigned long lapic_id = (unsigned long)&lapic_id_initdata; 94 vm_offset_t lapic_start; 95 96 static i386_intr_func_t lapic_timer_func; 97 static i386_intr_func_t lapic_pmi_func; 98 99 /* TRUE if local APIC was enabled by the OS not by the BIOS */ 100 static boolean_t lapic_os_enabled = FALSE; 101 102 /* Base vector for local APIC interrupt sources */ 103 int lapic_interrupt_base = LAPIC_DEFAULT_INTERRUPT_BASE; 104 105 void slave_boot_init(void); 106 107 static void mp_kdp_wait(void); 108 static void mp_rendezvous_action(void); 109 110 boolean_t smp_initialized = FALSE; 111 112 decl_simple_lock_data(,mp_kdp_lock); 113 114 decl_mutex_data(static, mp_cpu_boot_lock); 115 116 /* Variables needed for MP rendezvous. */ 117 static void (*mp_rv_setup_func)(void *arg); 118 static void (*mp_rv_action_func)(void *arg); 119 static void (*mp_rv_teardown_func)(void *arg); 120 static void *mp_rv_func_arg; 121 static int mp_rv_ncpus; 122 static long mp_rv_waiters[2]; 123 decl_simple_lock_data(,mp_rv_lock); 124 125 int lapic_to_cpu[MAX_CPUS]; 126 int cpu_to_lapic[MAX_CPUS]; 127 128 static void 129 lapic_cpu_map_init(void) 130 { 131 int i; 132 133 for (i = 0; i < MAX_CPUS; i++) { 134 lapic_to_cpu[i] = -1; 135 cpu_to_lapic[i] = -1; 136 } 137 } 138 139 void 140 lapic_cpu_map(int apic_id, int cpu) 141 { 142 cpu_to_lapic[cpu] = apic_id; 143 lapic_to_cpu[apic_id] = cpu; 144 } 145 146 #ifdef MP_DEBUG 147 static void 148 lapic_cpu_map_dump(void) 149 { 150 int i; 151 152 for (i = 0; i < MAX_CPUS; i++) { 153 if (cpu_to_lapic[i] == -1) 154 continue; 155 kprintf("cpu_to_lapic[%d]: %d\n", 156 i, cpu_to_lapic[i]); 157 } 158 for (i = 0; i < MAX_CPUS; i++) { 159 if (lapic_to_cpu[i] == -1) 160 continue; 161 kprintf("lapic_to_cpu[%d]: %d\n", 162 i, lapic_to_cpu[i]); 163 } 164 } 165 #define LAPIC_CPU_MAP_DUMP() lapic_cpu_map_dump() 166 #define LAPIC_DUMP() lapic_dump() 167 #else 168 #define LAPIC_CPU_MAP_DUMP() 169 #define LAPIC_DUMP() 170 #endif /* MP_DEBUG */ 171 172 #define LAPIC_REG(reg) \ 173 (*((volatile int *)(lapic_start + LAPIC_##reg))) 174 #define LAPIC_REG_OFFSET(reg,off) \ 175 (*((volatile int *)(lapic_start + LAPIC_##reg + (off)))) 176 177 #define LAPIC_VECTOR(src) \ 178 (lapic_interrupt_base + LAPIC_##src##_INTERRUPT) 179 180 #define LAPIC_ISR_IS_SET(base,src) \ 181 (LAPIC_REG_OFFSET(ISR_BASE,((base+LAPIC_##src##_INTERRUPT)/32)*0x10) & \ 182 (1 <<((base + LAPIC_##src##_INTERRUPT)%32))) 183 184 #if GPROF 185 /* 186 * Initialize dummy structs for profiling. These aren't used but 187 * allows hertz_tick() to be built with GPROF defined. 188 */ 189 struct profile_vars _profile_vars; 190 struct profile_vars *_profile_vars_cpus[MAX_CPUS] = { &_profile_vars }; 191 #define GPROF_INIT() \ 192 { \ 193 int i; \ 194 \ 195 /* Hack to initialize pointers to unused profiling structs */ \ 196 for (i = 1; i < MAX_CPUS; i++) \ 197 _profile_vars_cpus[i] = &_profile_vars; \ 198 } 199 #else 200 #define GPROF_INIT() 201 #endif /* GPROF */ 202 203 extern void master_up(void); 204 205 void 206 smp_init(void) 207 { 208 int result; 209 vm_map_entry_t entry; 210 uint32_t lo; 211 uint32_t hi; 212 boolean_t is_boot_processor; 213 boolean_t is_lapic_enabled; 214 vm_offset_t lapic_base; 215 216 simple_lock_init(&mp_kdp_lock, 0); 217 simple_lock_init(&mp_rv_lock, 0); 218 mutex_init(&mp_cpu_boot_lock, 0); 219 console_init(); 220 221 /* Local APIC? */ 222 if (!lapic_probe()) 223 return; 224 225 /* Examine the local APIC state */ 226 rdmsr(MSR_IA32_APIC_BASE, lo, hi); 227 is_boot_processor = (lo & MSR_IA32_APIC_BASE_BSP) != 0; 228 is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0; 229 lapic_base = (lo & MSR_IA32_APIC_BASE_BASE); 230 kprintf("MSR_IA32_APIC_BASE 0x%x %s %s\n", lapic_base, 231 is_lapic_enabled ? "enabled" : "disabled", 232 is_boot_processor ? "BSP" : "AP"); 233 if (!is_boot_processor || !is_lapic_enabled) 234 panic("Unexpected local APIC state\n"); 235 236 /* Establish a map to the local apic */ 237 lapic_start = vm_map_min(kernel_map); 238 result = vm_map_find_space(kernel_map, &lapic_start, 239 round_page(LAPIC_SIZE), 0, &entry); 240 if (result != KERN_SUCCESS) { 241 panic("smp_init: vm_map_find_entry FAILED (err=%d)", result); 242 } 243 vm_map_unlock(kernel_map); 244 pmap_enter(pmap_kernel(), 245 lapic_start, 246 (ppnum_t) i386_btop(lapic_base), 247 VM_PROT_READ|VM_PROT_WRITE, 248 VM_WIMG_USE_DEFAULT, 249 TRUE); 250 lapic_id = (unsigned long)(lapic_start + LAPIC_ID); 251 252 if ((LAPIC_REG(VERSION)&LAPIC_VERSION_MASK) != 0x14) { 253 printf("Local APIC version not 0x14 as expected\n"); 254 } 255 256 /* Set up the lapic_id <-> cpu_number map and add this boot processor */ 257 lapic_cpu_map_init(); 258 lapic_cpu_map((LAPIC_REG(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 0); 259 260 lapic_init(); 261 262 cpu_thread_init(); 263 264 if (pmc_init() != KERN_SUCCESS) 265 printf("Performance counters not available\n"); 266 267 GPROF_INIT(); 268 DBGLOG_CPU_INIT(master_cpu); 269 270 slave_boot_init(); 271 master_up(); 272 273 smp_initialized = TRUE; 274 275 return; 276 } 277 278 279 static int 280 lapic_esr_read(void) 281 { 282 /* write-read register */ 283 LAPIC_REG(ERROR_STATUS) = 0; 284 return LAPIC_REG(ERROR_STATUS); 285 } 286 287 static void 288 lapic_esr_clear(void) 289 { 290 LAPIC_REG(ERROR_STATUS) = 0; 291 LAPIC_REG(ERROR_STATUS) = 0; 292 } 293 294 static const char *DM[8] = { 295 "Fixed", 296 "Lowest Priority", 297 "Invalid", 298 "Invalid", 299 "NMI", 300 "Reset", 301 "Invalid", 302 "ExtINT"}; 303 304 void 305 lapic_dump(void) 306 { 307 int i; 308 309 #define BOOL(a) ((a)?' ':'!') 310 311 kprintf("LAPIC %d at 0x%x version 0x%x\n", 312 (LAPIC_REG(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 313 lapic_start, 314 LAPIC_REG(VERSION)&LAPIC_VERSION_MASK); 315 kprintf("Priorities: Task 0x%x Arbitration 0x%x Processor 0x%x\n", 316 LAPIC_REG(TPR)&LAPIC_TPR_MASK, 317 LAPIC_REG(APR)&LAPIC_APR_MASK, 318 LAPIC_REG(PPR)&LAPIC_PPR_MASK); 319 kprintf("Destination Format 0x%x Logical Destination 0x%x\n", 320 LAPIC_REG(DFR)>>LAPIC_DFR_SHIFT, 321 LAPIC_REG(LDR)>>LAPIC_LDR_SHIFT); 322 kprintf("%cEnabled %cFocusChecking SV 0x%x\n", 323 BOOL(LAPIC_REG(SVR)&LAPIC_SVR_ENABLE), 324 BOOL(!(LAPIC_REG(SVR)&LAPIC_SVR_FOCUS_OFF)), 325 LAPIC_REG(SVR) & LAPIC_SVR_MASK); 326 kprintf("LVT_TIMER: Vector 0x%02x %s %cmasked %s\n", 327 LAPIC_REG(LVT_TIMER)&LAPIC_LVT_VECTOR_MASK, 328 (LAPIC_REG(LVT_TIMER)&LAPIC_LVT_DS_PENDING)?"SendPending":"Idle", 329 BOOL(LAPIC_REG(LVT_TIMER)&LAPIC_LVT_MASKED), 330 (LAPIC_REG(LVT_TIMER)&LAPIC_LVT_PERIODIC)?"Periodic":"OneShot"); 331 kprintf(" Initial Count: 0x%08x \n", LAPIC_REG(TIMER_INITIAL_COUNT)); 332 kprintf(" Current Count: 0x%08x \n", LAPIC_REG(TIMER_CURRENT_COUNT)); 333 kprintf(" Divide Config: 0x%08x \n", LAPIC_REG(TIMER_DIVIDE_CONFIG)); 334 kprintf("LVT_PERFCNT: Vector 0x%02x [%s] %s %cmasked\n", 335 LAPIC_REG(LVT_PERFCNT)&LAPIC_LVT_VECTOR_MASK, 336 DM[(LAPIC_REG(LVT_PERFCNT)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK], 337 (LAPIC_REG(LVT_PERFCNT)&LAPIC_LVT_DS_PENDING)?"SendPending":"Idle", 338 BOOL(LAPIC_REG(LVT_PERFCNT)&LAPIC_LVT_MASKED)); 339 kprintf("LVT_LINT0: Vector 0x%02x [%s][%s][%s] %s %cmasked\n", 340 LAPIC_REG(LVT_LINT0)&LAPIC_LVT_VECTOR_MASK, 341 DM[(LAPIC_REG(LVT_LINT0)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK], 342 (LAPIC_REG(LVT_LINT0)&LAPIC_LVT_TM_LEVEL)?"Level":"Edge ", 343 (LAPIC_REG(LVT_LINT0)&LAPIC_LVT_IP_PLRITY_LOW)?"Low ":"High", 344 (LAPIC_REG(LVT_LINT0)&LAPIC_LVT_DS_PENDING)?"SendPending":"Idle", 345 BOOL(LAPIC_REG(LVT_LINT0)&LAPIC_LVT_MASKED)); 346 kprintf("LVT_LINT1: Vector 0x%02x [%s][%s][%s] %s %cmasked\n", 347 LAPIC_REG(LVT_LINT1)&LAPIC_LVT_VECTOR_MASK, 348 DM[(LAPIC_REG(LVT_LINT1)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK], 349 (LAPIC_REG(LVT_LINT1)&LAPIC_LVT_TM_LEVEL)?"Level":"Edge ", 350 (LAPIC_REG(LVT_LINT1)&LAPIC_LVT_IP_PLRITY_LOW)?"Low ":"High", 351 (LAPIC_REG(LVT_LINT1)&LAPIC_LVT_DS_PENDING)?"SendPending":"Idle", 352 BOOL(LAPIC_REG(LVT_LINT1)&LAPIC_LVT_MASKED)); 353 kprintf("LVT_ERROR: Vector 0x%02x %s %cmasked\n", 354 LAPIC_REG(LVT_ERROR)&LAPIC_LVT_VECTOR_MASK, 355 (LAPIC_REG(LVT_ERROR)&LAPIC_LVT_DS_PENDING)?"SendPending":"Idle", 356 BOOL(LAPIC_REG(LVT_ERROR)&LAPIC_LVT_MASKED)); 357 kprintf("ESR: %08x \n", lapic_esr_read()); 358 kprintf(" "); 359 for(i=0xf; i>=0; i--) 360 kprintf("%x%x%x%x",i,i,i,i); 361 kprintf("\n"); 362 kprintf("TMR: 0x"); 363 for(i=7; i>=0; i--) 364 kprintf("%08x",LAPIC_REG_OFFSET(TMR_BASE, i*0x10)); 365 kprintf("\n"); 366 kprintf("IRR: 0x"); 367 for(i=7; i>=0; i--) 368 kprintf("%08x",LAPIC_REG_OFFSET(IRR_BASE, i*0x10)); 369 kprintf("\n"); 370 kprintf("ISR: 0x"); 371 for(i=7; i >= 0; i--) 372 kprintf("%08x",LAPIC_REG_OFFSET(ISR_BASE, i*0x10)); 373 kprintf("\n"); 374 } 375 376 boolean_t 377 lapic_probe(void) 378 { 379 uint32_t lo; 380 uint32_t hi; 381 382 if (cpuid_features() & CPUID_FEATURE_APIC) 383 return TRUE; 384 385 if (cpuid_family() == 6 || cpuid_family() == 15) { 386 /* 387 * Mobile Pentiums: 388 * There may be a local APIC which wasn't enabled by BIOS. 389 * So we try to enable it explicitly. 390 */ 391 rdmsr(MSR_IA32_APIC_BASE, lo, hi); 392 lo &= ~MSR_IA32_APIC_BASE_BASE; 393 lo |= MSR_IA32_APIC_BASE_ENABLE | LAPIC_START; 394 lo |= MSR_IA32_APIC_BASE_ENABLE; 395 wrmsr(MSR_IA32_APIC_BASE, lo, hi); 396 397 /* 398 * Re-initialize cpu features info and re-check. 399 */ 400 set_cpu_model(); 401 if (cpuid_features() & CPUID_FEATURE_APIC) { 402 printf("Local APIC discovered and enabled\n"); 403 lapic_os_enabled = TRUE; 404 lapic_interrupt_base = LAPIC_REDUCED_INTERRUPT_BASE; 405 return TRUE; 406 } 407 } 408 409 return FALSE; 410 } 411 412 void 413 lapic_shutdown(void) 414 { 415 uint32_t lo; 416 uint32_t hi; 417 uint32_t value; 418 419 /* Shutdown if local APIC was enabled by OS */ 420 if (lapic_os_enabled == FALSE) 421 return; 422 423 mp_disable_preemption(); 424 425 /* ExtINT: masked */ 426 if (get_cpu_number() == master_cpu) { 427 value = LAPIC_REG(LVT_LINT0); 428 value |= LAPIC_LVT_MASKED; 429 LAPIC_REG(LVT_LINT0) = value; 430 } 431 432 /* Timer: masked */ 433 LAPIC_REG(LVT_TIMER) |= LAPIC_LVT_MASKED; 434 435 /* Perfmon: masked */ 436 LAPIC_REG(LVT_PERFCNT) |= LAPIC_LVT_MASKED; 437 438 /* Error: masked */ 439 LAPIC_REG(LVT_ERROR) |= LAPIC_LVT_MASKED; 440 441 /* APIC software disabled */ 442 LAPIC_REG(SVR) &= ~LAPIC_SVR_ENABLE; 443 444 /* Bypass the APIC completely and update cpu features */ 445 rdmsr(MSR_IA32_APIC_BASE, lo, hi); 446 lo &= ~MSR_IA32_APIC_BASE_ENABLE; 447 wrmsr(MSR_IA32_APIC_BASE, lo, hi); 448 set_cpu_model(); 449 450 mp_enable_preemption(); 451 } 452 453 void 454 lapic_init(void) 455 { 456 int value; 457 458 /* Set flat delivery model, logical processor id */ 459 LAPIC_REG(DFR) = LAPIC_DFR_FLAT; 460 LAPIC_REG(LDR) = (get_cpu_number()) << LAPIC_LDR_SHIFT; 461 462 /* Accept all */ 463 LAPIC_REG(TPR) = 0; 464 465 LAPIC_REG(SVR) = LAPIC_VECTOR(SPURIOUS) | LAPIC_SVR_ENABLE; 466 467 /* ExtINT */ 468 if (get_cpu_number() == master_cpu) { 469 value = LAPIC_REG(LVT_LINT0); 470 value &= ~LAPIC_LVT_MASKED; 471 value |= LAPIC_LVT_DM_EXTINT; 472 LAPIC_REG(LVT_LINT0) = value; 473 } 474 475 /* Timer: unmasked, one-shot */ 476 LAPIC_REG(LVT_TIMER) = LAPIC_VECTOR(TIMER); 477 478 /* Perfmon: unmasked */ 479 LAPIC_REG(LVT_PERFCNT) = LAPIC_VECTOR(PERFCNT); 480 481 lapic_esr_clear(); 482 483 LAPIC_REG(LVT_ERROR) = LAPIC_VECTOR(ERROR); 484 485 } 486 487 void 488 lapic_set_timer_func(i386_intr_func_t func) 489 { 490 lapic_timer_func = func; 491 } 492 493 void 494 lapic_set_timer( 495 boolean_t interrupt, 496 lapic_timer_mode_t mode, 497 lapic_timer_divide_t divisor, 498 lapic_timer_count_t initial_count) 499 { 500 boolean_t state; 501 uint32_t timer_vector; 502 503 state = ml_set_interrupts_enabled(FALSE); 504 timer_vector = LAPIC_REG(LVT_TIMER); 505 timer_vector &= ~(LAPIC_LVT_MASKED|LAPIC_LVT_PERIODIC);; 506 timer_vector |= interrupt ? 0 : LAPIC_LVT_MASKED; 507 timer_vector |= (mode == periodic) ? LAPIC_LVT_PERIODIC : 0; 508 LAPIC_REG(LVT_TIMER) = timer_vector; 509 LAPIC_REG(TIMER_DIVIDE_CONFIG) = divisor; 510 LAPIC_REG(TIMER_INITIAL_COUNT) = initial_count; 511 ml_set_interrupts_enabled(state); 512 } 513 514 void 515 lapic_get_timer( 516 lapic_timer_mode_t *mode, 517 lapic_timer_divide_t *divisor, 518 lapic_timer_count_t *initial_count, 519 lapic_timer_count_t *current_count) 520 { 521 boolean_t state; 522 523 state = ml_set_interrupts_enabled(FALSE); 524 if (mode) 525 *mode = (LAPIC_REG(LVT_TIMER) & LAPIC_LVT_PERIODIC) ? 526 periodic : one_shot; 527 if (divisor) 528 *divisor = LAPIC_REG(TIMER_DIVIDE_CONFIG) & LAPIC_TIMER_DIVIDE_MASK; 529 if (initial_count) 530 *initial_count = LAPIC_REG(TIMER_INITIAL_COUNT); 531 if (current_count) 532 *current_count = LAPIC_REG(TIMER_CURRENT_COUNT); 533 ml_set_interrupts_enabled(state); 534 } 535 536 void 537 lapic_set_pmi_func(i386_intr_func_t func) 538 { 539 lapic_pmi_func = func; 540 } 541 542 static inline void 543 _lapic_end_of_interrupt(void) 544 { 545 LAPIC_REG(EOI) = 0; 546 } 547 548 void 549 lapic_end_of_interrupt(void) 550 { 551 _lapic_end_of_interrupt(); 552 } 553 554 int 555 lapic_interrupt(int interrupt, void *state) 556 { 557 interrupt -= lapic_interrupt_base; 558 if (interrupt < 0) 559 return 0; 560 561 switch(interrupt) { 562 case LAPIC_PERFCNT_INTERRUPT: 563 if (lapic_pmi_func != NULL) 564 (*lapic_pmi_func)( 565 (struct i386_interrupt_state *) state); 566 /* Clear interrupt masked */ 567 LAPIC_REG(LVT_PERFCNT) = LAPIC_VECTOR(PERFCNT); 568 _lapic_end_of_interrupt(); 569 return 1; 570 case LAPIC_TIMER_INTERRUPT: 571 _lapic_end_of_interrupt(); 572 if (lapic_timer_func != NULL) 573 (*lapic_timer_func)( 574 (struct i386_interrupt_state *) state); 575 return 1; 576 case LAPIC_ERROR_INTERRUPT: 577 lapic_dump(); 578 panic("Local APIC error\n"); 579 _lapic_end_of_interrupt(); 580 return 1; 581 case LAPIC_SPURIOUS_INTERRUPT: 582 kprintf("SPIV\n"); 583 /* No EOI required here */ 584 return 1; 585 case LAPIC_INTERPROCESSOR_INTERRUPT: 586 cpu_signal_handler((struct i386_interrupt_state *) state); 587 _lapic_end_of_interrupt(); 588 return 1; 589 } 590 return 0; 591 } 592 593 void 594 lapic_smm_restore(void) 595 { 596 boolean_t state; 597 598 if (lapic_os_enabled == FALSE) 599 return; 600 601 state = ml_set_interrupts_enabled(FALSE); 602 603 if (LAPIC_ISR_IS_SET(LAPIC_REDUCED_INTERRUPT_BASE, TIMER)) { 604 /* 605 * Bogus SMI handler enables interrupts but does not know about 606 * local APIC interrupt sources. When APIC timer counts down to 607 * zero while in SMM, local APIC will end up waiting for an EOI 608 * but no interrupt was delivered to the OS. 609 */ 610 _lapic_end_of_interrupt(); 611 612 /* 613 * timer is one-shot, trigger another quick countdown to trigger 614 * another timer interrupt. 615 */ 616 if (LAPIC_REG(TIMER_CURRENT_COUNT) == 0) { 617 LAPIC_REG(TIMER_INITIAL_COUNT) = 1; 618 } 619 620 kprintf("lapic_smm_restore\n"); 621 } 622 623 ml_set_interrupts_enabled(state); 624 } 625 626 kern_return_t 627 intel_startCPU( 628 int slot_num) 629 { 630 631 int i = 1000; 632 int lapic = cpu_to_lapic[slot_num]; 633 634 assert(lapic != -1); 635 636 DBGLOG_CPU_INIT(slot_num); 637 638 DBG("intel_startCPU(%d) lapic_id=%d\n", slot_num, lapic); 639 DBG("IdlePTD(%p): 0x%x\n", &IdlePTD, (int) IdlePTD); 640 641 /* Initialize (or re-initialize) the descriptor tables for this cpu. */ 642 mp_desc_init(cpu_datap(slot_num), FALSE); 643 644 /* Serialize use of the slave boot stack. */ 645 mutex_lock(&mp_cpu_boot_lock); 646 647 mp_disable_preemption(); 648 if (slot_num == get_cpu_number()) { 649 mp_enable_preemption(); 650 mutex_unlock(&mp_cpu_boot_lock); 651 return KERN_SUCCESS; 652 } 653 654 LAPIC_REG(ICRD) = lapic << LAPIC_ICRD_DEST_SHIFT; 655 LAPIC_REG(ICR) = LAPIC_ICR_DM_INIT; 656 delay(10000); 657 658 LAPIC_REG(ICRD) = lapic << LAPIC_ICRD_DEST_SHIFT; 659 LAPIC_REG(ICR) = LAPIC_ICR_DM_STARTUP|(MP_BOOT>>12); 660 delay(200); 661 662 LAPIC_REG(ICRD) = lapic << LAPIC_ICRD_DEST_SHIFT; 663 LAPIC_REG(ICR) = LAPIC_ICR_DM_STARTUP|(MP_BOOT>>12); 664 delay(200); 665 666 #ifdef POSTCODE_DELAY 667 /* Wait much longer if postcodes are displayed for a delay period. */ 668 i *= 10000; 669 #endif 670 while(i-- > 0) { 671 if (cpu_datap(slot_num)->cpu_running) 672 break; 673 delay(10000); 674 } 675 676 mp_enable_preemption(); 677 mutex_unlock(&mp_cpu_boot_lock); 678 679 if (!cpu_datap(slot_num)->cpu_running) { 680 DBG("Failed to start CPU %02d\n", slot_num); 681 printf("Failed to start CPU %02d, rebooting...\n", slot_num); 682 delay(1000000); 683 cpu_shutdown(); 684 return KERN_SUCCESS; 685 } else { 686 DBG("Started CPU %02d\n", slot_num); 687 printf("Started CPU %02d\n", slot_num); 688 return KERN_SUCCESS; 689 } 690 } 691 692 extern char slave_boot_base[]; 693 extern char slave_boot_end[]; 694 extern void pstart(void); 695 696 void 697 slave_boot_init(void) 698 { 699 DBG("V(slave_boot_base)=%p P(slave_boot_base)=%p MP_BOOT=%p sz=0x%x\n", 700 slave_boot_base, 701 kvtophys((vm_offset_t) slave_boot_base), 702 MP_BOOT, 703 slave_boot_end-slave_boot_base); 704 705 /* 706 * Copy the boot entry code to the real-mode vector area MP_BOOT. 707 * This is in page 1 which has been reserved for this purpose by 708 * machine_startup() from the boot processor. 709 * The slave boot code is responsible for switching to protected 710 * mode and then jumping to the common startup, _start(). 711 */ 712 bcopy_phys((addr64_t) kvtophys((vm_offset_t) slave_boot_base), 713 (addr64_t) MP_BOOT, 714 slave_boot_end-slave_boot_base); 715 716 /* 717 * Zero a stack area above the boot code. 718 */ 719 DBG("bzero_phys 0x%x sz 0x%x\n",MP_BOOTSTACK+MP_BOOT-0x400, 0x400); 720 bzero_phys((addr64_t)MP_BOOTSTACK+MP_BOOT-0x400, 0x400); 721 722 /* 723 * Set the location at the base of the stack to point to the 724 * common startup entry. 725 */ 726 DBG("writing 0x%x at phys 0x%x\n", 727 kvtophys((vm_offset_t) &pstart), MP_MACH_START+MP_BOOT); 728 ml_phys_write_word(MP_MACH_START+MP_BOOT, 729 kvtophys((vm_offset_t) &pstart)); 730 731 /* Flush caches */ 732 __asm__("wbinvd"); 733 } 734 735 #if MP_DEBUG 736 cpu_signal_event_log_t *cpu_signal[MAX_CPUS]; 737 cpu_signal_event_log_t *cpu_handle[MAX_CPUS]; 738 739 MP_EVENT_NAME_DECL(); 740 741 #endif /* MP_DEBUG */ 742 743 void 744 cpu_signal_handler(__unused struct i386_interrupt_state *regs) 745 { 746 int my_cpu; 747 volatile int *my_word; 748 #if MACH_KDB && MACH_ASSERT 749 int i=100; 750 #endif /* MACH_KDB && MACH_ASSERT */ 751 752 mp_disable_preemption(); 753 754 my_cpu = cpu_number(); 755 my_word = ¤t_cpu_datap()->cpu_signals; 756 757 do { 758 #if MACH_KDB && MACH_ASSERT 759 if (i-- <= 0) 760 Debugger("cpu_signal_handler"); 761 #endif /* MACH_KDB && MACH_ASSERT */ 762 #if MACH_KDP 763 if (i_bit(MP_KDP, my_word)) { 764 DBGLOG(cpu_handle,my_cpu,MP_KDP); 765 i_bit_clear(MP_KDP, my_word); 766 mp_kdp_wait(); 767 } else 768 #endif /* MACH_KDP */ 769 if (i_bit(MP_TLB_FLUSH, my_word)) { 770 DBGLOG(cpu_handle,my_cpu,MP_TLB_FLUSH); 771 i_bit_clear(MP_TLB_FLUSH, my_word); 772 pmap_update_interrupt(); 773 } else if (i_bit(MP_AST, my_word)) { 774 DBGLOG(cpu_handle,my_cpu,MP_AST); 775 i_bit_clear(MP_AST, my_word); 776 ast_check(cpu_to_processor(my_cpu)); 777 #if MACH_KDB 778 } else if (i_bit(MP_KDB, my_word)) { 779 extern kdb_is_slave[]; 780 781 i_bit_clear(MP_KDB, my_word); 782 kdb_is_slave[my_cpu]++; 783 kdb_kintr(); 784 #endif /* MACH_KDB */ 785 } else if (i_bit(MP_RENDEZVOUS, my_word)) { 786 DBGLOG(cpu_handle,my_cpu,MP_RENDEZVOUS); 787 i_bit_clear(MP_RENDEZVOUS, my_word); 788 mp_rendezvous_action(); 789 } 790 } while (*my_word); 791 792 mp_enable_preemption(); 793 794 } 795 796 #ifdef MP_DEBUG 797 extern int max_lock_loops; 798 #endif /* MP_DEBUG */ 799 void 800 cpu_interrupt(int cpu) 801 { 802 boolean_t state; 803 804 if (smp_initialized) { 805 806 /* Wait for previous interrupt to be delivered... */ 807 #ifdef MP_DEBUG 808 int pending_busy_count = 0; 809 while (LAPIC_REG(ICR) & LAPIC_ICR_DS_PENDING) { 810 if (++pending_busy_count > max_lock_loops) 811 panic("cpus_interrupt() deadlock\n"); 812 #else 813 while (LAPIC_REG(ICR) & LAPIC_ICR_DS_PENDING) { 814 #endif /* MP_DEBUG */ 815 cpu_pause(); 816 } 817 818 state = ml_set_interrupts_enabled(FALSE); 819 LAPIC_REG(ICRD) = 820 cpu_to_lapic[cpu] << LAPIC_ICRD_DEST_SHIFT; 821 LAPIC_REG(ICR) = 822 LAPIC_VECTOR(INTERPROCESSOR) | LAPIC_ICR_DM_FIXED; 823 (void) ml_set_interrupts_enabled(state); 824 } 825 826 } 827 828 void 829 i386_signal_cpu(int cpu, mp_event_t event, mp_sync_t mode) 830 { 831 volatile int *signals = &cpu_datap(cpu)->cpu_signals; 832 uint64_t tsc_timeout; 833 834 835 if (!cpu_datap(cpu)->cpu_running) 836 return; 837 838 DBGLOG(cpu_signal, cpu, event); 839 840 i_bit_set(event, signals); 841 cpu_interrupt(cpu); 842 if (mode == SYNC) { 843 again: 844 tsc_timeout = rdtsc64() + (1000*1000*1000); 845 while (i_bit(event, signals) && rdtsc64() < tsc_timeout) { 846 cpu_pause(); 847 } 848 if (i_bit(event, signals)) { 849 DBG("i386_signal_cpu(%d, 0x%x, SYNC) timed out\n", 850 cpu, event); 851 goto again; 852 } 853 } 854 } 855 856 void 857 i386_signal_cpus(mp_event_t event, mp_sync_t mode) 858 { 859 unsigned int cpu; 860 unsigned int my_cpu = cpu_number(); 861 862 for (cpu = 0; cpu < real_ncpus; cpu++) { 863 if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running) 864 continue; 865 i386_signal_cpu(cpu, event, mode); 866 } 867 } 868 869 int 870 i386_active_cpus(void) 871 { 872 unsigned int cpu; 873 unsigned int ncpus = 0; 874 875 for (cpu = 0; cpu < real_ncpus; cpu++) { 876 if (cpu_datap(cpu)->cpu_running) 877 ncpus++; 878 } 879 return(ncpus); 880 } 881 882 /* 883 * All-CPU rendezvous: 884 * - CPUs are signalled, 885 * - all execute the setup function (if specified), 886 * - rendezvous (i.e. all cpus reach a barrier), 887 * - all execute the action function (if specified), 888 * - rendezvous again, 889 * - execute the teardown function (if specified), and then 890 * - resume. 891 * 892 * Note that the supplied external functions _must_ be reentrant and aware 893 * that they are running in parallel and in an unknown lock context. 894 */ 895 896 static void 897 mp_rendezvous_action(void) 898 { 899 900 /* setup function */ 901 if (mp_rv_setup_func != NULL) 902 mp_rv_setup_func(mp_rv_func_arg); 903 /* spin on entry rendezvous */ 904 atomic_incl(&mp_rv_waiters[0], 1); 905 while (*((volatile long *) &mp_rv_waiters[0]) < mp_rv_ncpus) 906 cpu_pause(); 907 /* action function */ 908 if (mp_rv_action_func != NULL) 909 mp_rv_action_func(mp_rv_func_arg); 910 /* spin on exit rendezvous */ 911 atomic_incl(&mp_rv_waiters[1], 1); 912 while (*((volatile long *) &mp_rv_waiters[1]) < mp_rv_ncpus) 913 cpu_pause(); 914 /* teardown function */ 915 if (mp_rv_teardown_func != NULL) 916 mp_rv_teardown_func(mp_rv_func_arg); 917 } 918 919 void 920 mp_rendezvous(void (*setup_func)(void *), 921 void (*action_func)(void *), 922 void (*teardown_func)(void *), 923 void *arg) 924 { 925 926 if (!smp_initialized) { 927 if (setup_func != NULL) 928 setup_func(arg); 929 if (action_func != NULL) 930 action_func(arg); 931 if (teardown_func != NULL) 932 teardown_func(arg); 933 return; 934 } 935 936 /* obtain rendezvous lock */ 937 simple_lock(&mp_rv_lock); 938 939 /* set static function pointers */ 940 mp_rv_setup_func = setup_func; 941 mp_rv_action_func = action_func; 942 mp_rv_teardown_func = teardown_func; 943 mp_rv_func_arg = arg; 944 945 mp_rv_waiters[0] = 0; /* entry rendezvous count */ 946 mp_rv_waiters[1] = 0; /* exit rendezvous count */ 947 mp_rv_ncpus = i386_active_cpus(); 948 949 /* 950 * signal other processors, which will call mp_rendezvous_action() 951 * with interrupts disabled 952 */ 953 i386_signal_cpus(MP_RENDEZVOUS, ASYNC); 954 955 /* call executor function on this cpu */ 956 mp_rendezvous_action(); 957 958 /* release lock */ 959 simple_unlock(&mp_rv_lock); 960 } 961 962 #if MACH_KDP 963 volatile boolean_t mp_kdp_trap = FALSE; 964 long mp_kdp_ncpus; 965 boolean_t mp_kdp_state; 966 967 968 void 969 mp_kdp_enter(void) 970 { 971 unsigned int cpu; 972 unsigned int ncpus; 973 unsigned int my_cpu = cpu_number(); 974 uint64_t tsc_timeout; 975 976 DBG("mp_kdp_enter()\n"); 977 978 /* 979 * Here to enter the debugger. 980 * In case of races, only one cpu is allowed to enter kdp after 981 * stopping others. 982 */ 983 mp_kdp_state = ml_set_interrupts_enabled(FALSE); 984 simple_lock(&mp_kdp_lock); 985 while (mp_kdp_trap) { 986 simple_unlock(&mp_kdp_lock); 987 DBG("mp_kdp_enter() race lost\n"); 988 mp_kdp_wait(); 989 simple_lock(&mp_kdp_lock); 990 } 991 mp_kdp_ncpus = 1; /* self */ 992 mp_kdp_trap = TRUE; 993 simple_unlock(&mp_kdp_lock); 994 995 /* Deliver a nudge to other cpus, counting how many */ 996 DBG("mp_kdp_enter() signaling other processors\n"); 997 for (ncpus = 1, cpu = 0; cpu < real_ncpus; cpu++) { 998 if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running) 999 continue; 1000 ncpus++; 1001 i386_signal_cpu(cpu, MP_KDP, ASYNC); 1002 } 1003 1004 /* Wait other processors to spin. */ 1005 DBG("mp_kdp_enter() waiting for (%d) processors to suspend\n", ncpus); 1006 tsc_timeout = rdtsc64() + (1000*1000*1000); 1007 while (*((volatile unsigned int *) &mp_kdp_ncpus) != ncpus 1008 && rdtsc64() < tsc_timeout) { 1009 cpu_pause(); 1010 } 1011 DBG("mp_kdp_enter() %d processors done %s\n", 1012 mp_kdp_ncpus, (mp_kdp_ncpus == ncpus) ? "OK" : "timed out"); 1013 postcode(MP_KDP_ENTER); 1014 } 1015 1016 static void 1017 mp_kdp_wait(void) 1018 { 1019 boolean_t state; 1020 1021 state = ml_set_interrupts_enabled(TRUE); 1022 DBG("mp_kdp_wait()\n"); 1023 atomic_incl(&mp_kdp_ncpus, 1); 1024 while (mp_kdp_trap) { 1025 cpu_pause(); 1026 } 1027 atomic_decl(&mp_kdp_ncpus, 1); 1028 DBG("mp_kdp_wait() done\n"); 1029 (void) ml_set_interrupts_enabled(state); 1030 } 1031 1032 void 1033 mp_kdp_exit(void) 1034 { 1035 DBG("mp_kdp_exit()\n"); 1036 atomic_decl(&mp_kdp_ncpus, 1); 1037 mp_kdp_trap = FALSE; 1038 1039 /* Wait other processors to stop spinning. XXX needs timeout */ 1040 DBG("mp_kdp_exit() waiting for processors to resume\n"); 1041 while (*((volatile long *) &mp_kdp_ncpus) > 0) { 1042 cpu_pause(); 1043 } 1044 DBG("mp_kdp_exit() done\n"); 1045 (void) ml_set_interrupts_enabled(mp_kdp_state); 1046 postcode(0); 1047 } 1048 #endif /* MACH_KDP */ 1049 1050 /*ARGSUSED*/ 1051 void 1052 init_ast_check( 1053 __unused processor_t processor) 1054 { 1055 } 1056 1057 void 1058 cause_ast_check( 1059 processor_t processor) 1060 { 1061 int cpu = PROCESSOR_DATA(processor, slot_num); 1062 1063 if (cpu != cpu_number()) { 1064 i386_signal_cpu(cpu, MP_AST, ASYNC); 1065 } 1066 } 1067 1068 /* 1069 * invoke kdb on slave processors 1070 */ 1071 1072 void 1073 remote_kdb(void) 1074 { 1075 unsigned int my_cpu = cpu_number(); 1076 unsigned int cpu; 1077 1078 mp_disable_preemption(); 1079 for (cpu = 0; cpu < real_ncpus; cpu++) { 1080 if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running) 1081 continue; 1082 i386_signal_cpu(cpu, MP_KDB, SYNC); 1083 } 1084 mp_enable_preemption(); 1085 } 1086 1087 /* 1088 * Clear kdb interrupt 1089 */ 1090 1091 void 1092 clear_kdb_intr(void) 1093 { 1094 mp_disable_preemption(); 1095 i_bit_clear(MP_KDB, ¤t_cpu_datap()->cpu_signals); 1096 mp_enable_preemption(); 1097 } 1098 1099 /* 1100 * i386_init_slave() is called from pstart. 1101 * We're in the cpu's interrupt stack with interrupts disabled. 1102 */ 1103 void 1104 i386_init_slave(void) 1105 { 1106 postcode(I386_INIT_SLAVE); 1107 1108 /* Ensure that caching and write-through are enabled */ 1109 set_cr0(get_cr0() & ~(CR0_NW|CR0_CD)); 1110 1111 DBG("i386_init_slave() CPU%d: phys (%d) active.\n", 1112 get_cpu_number(), get_cpu_phys_number()); 1113 1114 lapic_init(); 1115 1116 LAPIC_DUMP(); 1117 LAPIC_CPU_MAP_DUMP(); 1118 1119 mtrr_update_cpu(); 1120 1121 pat_init(); 1122 1123 cpu_init(); 1124 1125 slave_main(); 1126 1127 panic("i386_init_slave() returned from slave_main()"); 1128 } 1129 1130 void 1131 slave_machine_init(void) 1132 { 1133 /* 1134 * Here in process context. 1135 */ 1136 DBG("slave_machine_init() CPU%d\n", get_cpu_number()); 1137 1138 init_fpu(); 1139 1140 cpu_thread_init(); 1141 1142 pmc_init(); 1143 1144 cpu_machine_init(); 1145 1146 clock_init(); 1147 } 1148 1149 #undef cpu_number() 1150 int cpu_number(void) 1151 { 1152 return get_cpu_number(); 1153 } 1154 1155 #if MACH_KDB 1156 #include <ddb/db_output.h> 1157 1158 #define TRAP_DEBUG 0 /* Must match interrupt.s and spl.s */ 1159 1160 1161 #if TRAP_DEBUG 1162 #define MTRAPS 100 1163 struct mp_trap_hist_struct { 1164 unsigned char type; 1165 unsigned char data[5]; 1166 } trap_hist[MTRAPS], *cur_trap_hist = trap_hist, 1167 *max_trap_hist = &trap_hist[MTRAPS]; 1168 1169 void db_trap_hist(void); 1170 1171 /* 1172 * SPL: 1173 * 1: new spl 1174 * 2: old spl 1175 * 3: new tpr 1176 * 4: old tpr 1177 * INT: 1178 * 1: int vec 1179 * 2: old spl 1180 * 3: new spl 1181 * 4: post eoi tpr 1182 * 5: exit tpr 1183 */ 1184 1185 void 1186 db_trap_hist(void) 1187 { 1188 int i,j; 1189 for(i=0;i<MTRAPS;i++) 1190 if (trap_hist[i].type == 1 || trap_hist[i].type == 2) { 1191 db_printf("%s%s", 1192 (&trap_hist[i]>=cur_trap_hist)?"*":" ", 1193 (trap_hist[i].type == 1)?"SPL":"INT"); 1194 for(j=0;j<5;j++) 1195 db_printf(" %02x", trap_hist[i].data[j]); 1196 db_printf("\n"); 1197 } 1198 1199 } 1200 #endif /* TRAP_DEBUG */ 1201 1202 void db_lapic(int cpu); 1203 unsigned int db_remote_read(int cpu, int reg); 1204 void db_ioapic(unsigned int); 1205 void kdb_console(void); 1206 1207 void 1208 kdb_console(void) 1209 { 1210 } 1211 1212 #define BOOLP(a) ((a)?' ':'!') 1213 1214 static char *DM[8] = { 1215 "Fixed", 1216 "Lowest Priority", 1217 "Invalid", 1218 "Invalid", 1219 "NMI", 1220 "Reset", 1221 "Invalid", 1222 "ExtINT"}; 1223 1224 unsigned int 1225 db_remote_read(int cpu, int reg) 1226 { 1227 return -1; 1228 } 1229 1230 void 1231 db_lapic(int cpu) 1232 { 1233 } 1234 1235 void 1236 db_ioapic(unsigned int ind) 1237 { 1238 } 1239 1240 #endif /* MACH_KDB */ 1241
Cache object: bc593401b06506c2390d86576a7e53f3
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]