Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/mp_machdep.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. The name of the developer may NOT be used to endorse or promote products 11 * derived from this software without specific prior written permission. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include <sys/cdefs.h> 27 __FBSDID("$FreeBSD: releng/7.3/sys/i386/i386/mp_machdep.c 198589 2009-10-29 14:34:02Z jhb $"); 28 29 #include "opt_apic.h" 30 #include "opt_cpu.h" 31 #include "opt_kstack_pages.h" 32 #include "opt_mp_watchdog.h" 33 #include "opt_sched.h" 34 #include "opt_smp.h" 35 36 #if !defined(lint) 37 #if !defined(SMP) 38 #error How did you get here? 39 #endif 40 41 #ifndef DEV_APIC 42 #error The apic device is required for SMP, add "device apic" to your config file. 43 #endif 44 #if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT) 45 #error SMP not supported with CPU_DISABLE_CMPXCHG 46 #endif 47 #endif /* not lint */ 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/bus.h> 52 #include <sys/cons.h> /* cngetc() */ 53 #ifdef GPROF 54 #include <sys/gmon.h> 55 #endif 56 #include <sys/kernel.h> 57 #include <sys/ktr.h> 58 #include <sys/lock.h> 59 #include <sys/malloc.h> 60 #include <sys/memrange.h> 61 #include <sys/mutex.h> 62 #include <sys/pcpu.h> 63 #include <sys/proc.h> 64 #include <sys/sched.h> 65 #include <sys/smp.h> 66 #include <sys/sysctl.h> 67 68 #include <vm/vm.h> 69 #include <vm/vm_param.h> 70 #include <vm/pmap.h> 71 #include <vm/vm_kern.h> 72 #include <vm/vm_extern.h> 73 74 #include <machine/apicreg.h> 75 #include <machine/cputypes.h> 76 #include <machine/mca.h> 77 #include <machine/md_var.h> 78 #include <machine/mp_watchdog.h> 79 #include <machine/pcb.h> 80 #include <machine/psl.h> 81 #include <machine/smp.h> 82 #include <machine/specialreg.h> 83 84 #define WARMBOOT_TARGET 0 85 #define WARMBOOT_OFF (KERNBASE + 0x0467) 86 #define WARMBOOT_SEG (KERNBASE + 0x0469) 87 88 #define CMOS_REG (0x70) 89 #define CMOS_DATA (0x71) 90 #define BIOS_RESET (0x0f) 91 #define BIOS_WARM (0x0a) 92 93 /* 94 * this code MUST be enabled here and in mpboot.s. 95 * it follows the very early stages of AP boot by placing values in CMOS ram. 96 * it NORMALLY will never be needed and thus the primitive method for enabling. 97 * 98 #define CHECK_POINTS 99 */ 100 101 #if defined(CHECK_POINTS) && !defined(PC98) 102 #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) 103 #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) 104 105 #define CHECK_INIT(D); \ 106 CHECK_WRITE(0x34, (D)); \ 107 CHECK_WRITE(0x35, (D)); \ 108 CHECK_WRITE(0x36, (D)); \ 109 CHECK_WRITE(0x37, (D)); \ 110 CHECK_WRITE(0x38, (D)); \ 111 CHECK_WRITE(0x39, (D)); 112 113 #define CHECK_PRINT(S); \ 114 printf("%s: %d, %d, %d, %d, %d, %d\n", \ 115 (S), \ 116 CHECK_READ(0x34), \ 117 CHECK_READ(0x35), \ 118 CHECK_READ(0x36), \ 119 CHECK_READ(0x37), \ 120 CHECK_READ(0x38), \ 121 CHECK_READ(0x39)); 122 123 #else /* CHECK_POINTS */ 124 125 #define CHECK_INIT(D) 126 #define CHECK_PRINT(S) 127 #define CHECK_WRITE(A, D) 128 129 #endif /* CHECK_POINTS */ 130 131 /* lock region used by kernel profiling */ 132 int mcount_lock; 133 134 int mp_naps; /* # of Applications processors */ 135 int boot_cpu_id = -1; /* designated BSP */ 136 137 extern struct pcpu __pcpu[]; 138 139 /* 140 * CPU topology map datastructures for HTT. 141 */ 142 static struct cpu_group mp_groups[MAXCPU]; 143 static struct cpu_top mp_top; 144 145 /* AP uses this during bootstrap. Do not staticize. */ 146 char *bootSTK; 147 static int bootAP; 148 149 /* Free these after use */ 150 void *bootstacks[MAXCPU]; 151 152 /* Hotwire a 0->4MB V==P mapping */ 153 extern pt_entry_t *KPTphys; 154 155 struct pcb stoppcbs[MAXCPU]; 156 157 /* Variables needed for SMP tlb shootdown. */ 158 vm_offset_t smp_tlb_addr1; 159 vm_offset_t smp_tlb_addr2; 160 volatile int smp_tlb_wait; 161 162 #ifdef STOP_NMI 163 volatile cpumask_t ipi_nmi_pending; 164 165 static void ipi_nmi_selected(u_int32_t cpus); 166 #endif 167 168 #ifdef COUNT_IPIS 169 /* Interrupt counts. */ 170 static u_long *ipi_preempt_counts[MAXCPU]; 171 static u_long *ipi_ast_counts[MAXCPU]; 172 u_long *ipi_invltlb_counts[MAXCPU]; 173 u_long *ipi_invlrng_counts[MAXCPU]; 174 u_long *ipi_invlpg_counts[MAXCPU]; 175 u_long *ipi_invlcache_counts[MAXCPU]; 176 u_long *ipi_rendezvous_counts[MAXCPU]; 177 u_long *ipi_lazypmap_counts[MAXCPU]; 178 #endif 179 180 /* 181 * Local data and functions. 182 */ 183 184 #ifdef STOP_NMI 185 /* 186 * Provide an alternate method of stopping other CPUs. If another CPU has 187 * disabled interrupts the conventional STOP IPI will be blocked. This 188 * NMI-based stop should get through in that case. 189 */ 190 static int stop_cpus_with_nmi = 1; 191 SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW, 192 &stop_cpus_with_nmi, 0, ""); 193 TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi); 194 #else 195 #define stop_cpus_with_nmi 0 196 #endif 197 198 static u_int logical_cpus; 199 200 /* used to hold the AP's until we are ready to release them */ 201 static struct mtx ap_boot_mtx; 202 203 /* Set to 1 once we're ready to let the APs out of the pen. */ 204 static volatile int aps_ready = 0; 205 206 /* 207 * Store data from cpu_add() until later in the boot when we actually setup 208 * the APs. 209 */ 210 struct cpu_info { 211 int cpu_present:1; 212 int cpu_bsp:1; 213 int cpu_disabled:1; 214 int cpu_hyperthread:1; 215 } static cpu_info[MAX_APIC_ID + 1]; 216 int cpu_apic_ids[MAXCPU]; 217 218 /* Holds pending bitmap based IPIs per CPU */ 219 static volatile u_int cpu_ipi_pending[MAXCPU]; 220 221 static u_int boot_address; 222 223 static void assign_cpu_ids(void); 224 static void install_ap_tramp(void); 225 static void set_interrupt_apic_ids(void); 226 static int start_all_aps(void); 227 static int start_ap(int apic_id); 228 static void release_aps(void *dummy); 229 230 static int hlt_logical_cpus; 231 static u_int hyperthreading_cpus; 232 static cpumask_t hyperthreading_cpus_mask; 233 static int hyperthreading_allowed = 1; 234 static struct sysctl_ctx_list logical_cpu_clist; 235 236 static void 237 mem_range_AP_init(void) 238 { 239 if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP) 240 mem_range_softc.mr_op->initAP(&mem_range_softc); 241 } 242 243 void 244 mp_topology(void) 245 { 246 struct cpu_group *group; 247 int apic_id; 248 int groups; 249 int cpu; 250 251 /* Build the smp_topology map. */ 252 /* Nothing to do if there is no HTT support. */ 253 if (hyperthreading_cpus <= 1) 254 return; 255 group = &mp_groups[0]; 256 groups = 1; 257 for (cpu = 0, apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) { 258 if (!cpu_info[apic_id].cpu_present) 259 continue; 260 /* 261 * If the current group has members and we're not a logical 262 * cpu, create a new group. 263 */ 264 if (group->cg_count != 0 && 265 (apic_id % hyperthreading_cpus) == 0) { 266 group++; 267 groups++; 268 } 269 group->cg_count++; 270 group->cg_mask |= 1 << cpu; 271 cpu++; 272 } 273 274 mp_top.ct_count = groups; 275 mp_top.ct_group = mp_groups; 276 smp_topology = &mp_top; 277 } 278 279 280 /* 281 * Calculate usable address in base memory for AP trampoline code. 282 */ 283 u_int 284 mp_bootaddress(u_int basemem) 285 { 286 287 boot_address = trunc_page(basemem); /* round down to 4k boundary */ 288 if ((basemem - boot_address) < bootMP_size) 289 boot_address -= PAGE_SIZE; /* not enough, lower by 4k */ 290 291 return boot_address; 292 } 293 294 void 295 cpu_add(u_int apic_id, char boot_cpu) 296 { 297 298 if (apic_id > MAX_APIC_ID) { 299 panic("SMP: APIC ID %d too high", apic_id); 300 return; 301 } 302 KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice", 303 apic_id)); 304 cpu_info[apic_id].cpu_present = 1; 305 if (boot_cpu) { 306 KASSERT(boot_cpu_id == -1, 307 ("CPU %d claims to be BSP, but CPU %d already is", apic_id, 308 boot_cpu_id)); 309 boot_cpu_id = apic_id; 310 cpu_info[apic_id].cpu_bsp = 1; 311 } 312 if (mp_ncpus < MAXCPU) 313 mp_ncpus++; 314 if (bootverbose) 315 printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" : 316 "AP"); 317 } 318 319 void 320 cpu_mp_setmaxid(void) 321 { 322 323 mp_maxid = MAXCPU - 1; 324 } 325 326 int 327 cpu_mp_probe(void) 328 { 329 330 /* 331 * Always record BSP in CPU map so that the mbuf init code works 332 * correctly. 333 */ 334 all_cpus = 1; 335 if (mp_ncpus == 0) { 336 /* 337 * No CPUs were found, so this must be a UP system. Setup 338 * the variables to represent a system with a single CPU 339 * with an id of 0. 340 */ 341 mp_ncpus = 1; 342 return (0); 343 } 344 345 /* At least one CPU was found. */ 346 if (mp_ncpus == 1) { 347 /* 348 * One CPU was found, so this must be a UP system with 349 * an I/O APIC. 350 */ 351 return (0); 352 } 353 354 /* At least two CPUs were found. */ 355 return (1); 356 } 357 358 /* 359 * Initialize the IPI handlers and start up the AP's. 360 */ 361 void 362 cpu_mp_start(void) 363 { 364 int i; 365 u_int threads_per_cache, p[4]; 366 367 /* Initialize the logical ID to APIC ID table. */ 368 for (i = 0; i < MAXCPU; i++) { 369 cpu_apic_ids[i] = -1; 370 cpu_ipi_pending[i] = 0; 371 } 372 373 /* Install an inter-CPU IPI for TLB invalidation */ 374 setidt(IPI_INVLTLB, IDTVEC(invltlb), 375 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 376 setidt(IPI_INVLPG, IDTVEC(invlpg), 377 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 378 setidt(IPI_INVLRNG, IDTVEC(invlrng), 379 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 380 381 /* Install an inter-CPU IPI for cache invalidation. */ 382 setidt(IPI_INVLCACHE, IDTVEC(invlcache), 383 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 384 385 /* Install an inter-CPU IPI for lazy pmap release */ 386 setidt(IPI_LAZYPMAP, IDTVEC(lazypmap), 387 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 388 389 /* Install an inter-CPU IPI for all-CPU rendezvous */ 390 setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), 391 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 392 393 /* Install generic inter-CPU IPI handler */ 394 setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler), 395 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 396 397 /* Install an inter-CPU IPI for CPU stop/restart */ 398 setidt(IPI_STOP, IDTVEC(cpustop), 399 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 400 401 402 /* Set boot_cpu_id if needed. */ 403 if (boot_cpu_id == -1) { 404 boot_cpu_id = PCPU_GET(apic_id); 405 cpu_info[boot_cpu_id].cpu_bsp = 1; 406 } else 407 KASSERT(boot_cpu_id == PCPU_GET(apic_id), 408 ("BSP's APIC ID doesn't match boot_cpu_id")); 409 410 /* Setup the initial logical CPUs info. */ 411 logical_cpus = logical_cpus_mask = 0; 412 if (cpu_feature & CPUID_HTT) 413 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; 414 415 /* 416 * Work out if hyperthreading is *really* enabled. This 417 * is made really ugly by the fact that processors lie: Dual 418 * core processors claim to be hyperthreaded even when they're 419 * not, presumably because they want to be treated the same 420 * way as HTT with respect to per-cpu software licensing. 421 * At the time of writing (May 12, 2005) the only hyperthreaded 422 * cpus are from Intel, and Intel's dual-core processors can be 423 * identified via the "deterministic cache parameters" cpuid 424 * calls. 425 */ 426 /* 427 * First determine if this is an Intel processor which claims 428 * to have hyperthreading support. 429 */ 430 if ((cpu_feature & CPUID_HTT) && cpu_vendor_id == CPU_VENDOR_INTEL) { 431 /* 432 * If the "deterministic cache parameters" cpuid calls 433 * are available, use them. 434 */ 435 if (cpu_high >= 4) { 436 /* Ask the processor about the L1 cache. */ 437 for (i = 0; i < 1; i++) { 438 cpuid_count(4, i, p); 439 threads_per_cache = ((p[0] & 0x3ffc000) >> 14) + 1; 440 if (hyperthreading_cpus < threads_per_cache) 441 hyperthreading_cpus = threads_per_cache; 442 if ((p[0] & 0x1f) == 0) 443 break; 444 } 445 } 446 447 /* 448 * If the deterministic cache parameters are not 449 * available, or if no caches were reported to exist, 450 * just accept what the HTT flag indicated. 451 */ 452 if (hyperthreading_cpus == 0) 453 hyperthreading_cpus = logical_cpus; 454 } 455 456 assign_cpu_ids(); 457 458 /* Start each Application Processor */ 459 start_all_aps(); 460 461 set_interrupt_apic_ids(); 462 463 /* Last, setup the cpu topology now that we have probed CPUs */ 464 mp_topology(); 465 } 466 467 468 /* 469 * Print various information about the SMP system hardware and setup. 470 */ 471 void 472 cpu_mp_announce(void) 473 { 474 const char *hyperthread; 475 int i; 476 477 /* List active CPUs first. */ 478 printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id); 479 for (i = 1; i < mp_ncpus; i++) { 480 if (cpu_info[cpu_apic_ids[i]].cpu_hyperthread) 481 hyperthread = "/HT"; 482 else 483 hyperthread = ""; 484 printf(" cpu%d (AP%s): APIC ID: %2d\n", i, hyperthread, 485 cpu_apic_ids[i]); 486 } 487 488 /* List disabled CPUs last. */ 489 for (i = 0; i <= MAX_APIC_ID; i++) { 490 if (!cpu_info[i].cpu_present || !cpu_info[i].cpu_disabled) 491 continue; 492 if (cpu_info[i].cpu_hyperthread) 493 hyperthread = "/HT"; 494 else 495 hyperthread = ""; 496 printf(" cpu (AP%s): APIC ID: %2d (disabled)\n", hyperthread, 497 i); 498 } 499 } 500 501 /* 502 * AP CPU's call this to initialize themselves. 503 */ 504 void 505 init_secondary(void) 506 { 507 struct pcpu *pc; 508 vm_offset_t addr; 509 int gsel_tss; 510 int x, myid; 511 u_int cr0; 512 513 /* bootAP is set in start_ap() to our ID. */ 514 myid = bootAP; 515 516 /* Get per-cpu data */ 517 pc = &__pcpu[myid]; 518 519 /* prime data page for it to use */ 520 pcpu_init(pc, myid, sizeof(struct pcpu)); 521 pc->pc_apic_id = cpu_apic_ids[myid]; 522 pc->pc_prvspace = pc; 523 pc->pc_curthread = 0; 524 525 gdt_segs[GPRIV_SEL].ssd_base = (int) pc; 526 gdt_segs[GPROC0_SEL].ssd_base = (int) &pc->pc_common_tss; 527 528 for (x = 0; x < NGDT; x++) { 529 ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); 530 } 531 532 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; 533 r_gdt.rd_base = (int) &gdt[myid * NGDT]; 534 lgdt(&r_gdt); /* does magic intra-segment return */ 535 536 lidt(&r_idt); 537 538 lldt(_default_ldt); 539 PCPU_SET(currentldt, _default_ldt); 540 541 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); 542 gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; 543 PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */ 544 PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); 545 PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16); 546 PCPU_SET(tss_gdt, &gdt[myid * NGDT + GPROC0_SEL].sd); 547 PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); 548 ltr(gsel_tss); 549 550 PCPU_SET(fsgs_gdt, &gdt[myid * NGDT + GUFS_SEL].sd); 551 552 /* 553 * Set to a known state: 554 * Set by mpboot.s: CR0_PG, CR0_PE 555 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM 556 */ 557 cr0 = rcr0(); 558 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); 559 load_cr0(cr0); 560 CHECK_WRITE(0x38, 5); 561 562 /* Disable local APIC just to be sure. */ 563 lapic_disable(); 564 565 /* signal our startup to the BSP. */ 566 mp_naps++; 567 CHECK_WRITE(0x39, 6); 568 569 /* Spin until the BSP releases the AP's. */ 570 while (!aps_ready) 571 ia32_pause(); 572 573 /* BSP may have changed PTD while we were waiting */ 574 invltlb(); 575 for (addr = 0; addr < NKPT * NBPDR - 1; addr += PAGE_SIZE) 576 invlpg(addr); 577 578 #if defined(I586_CPU) && !defined(NO_F00F_HACK) 579 lidt(&r_idt); 580 #endif 581 582 /* Initialize the PAT MSR if present. */ 583 pmap_init_pat(); 584 585 /* set up CPU registers and state */ 586 cpu_setregs(); 587 588 /* set up FPU state on the AP */ 589 npxinit(); 590 591 /* set up SSE registers */ 592 enable_sse(); 593 594 #ifdef PAE 595 /* Enable the PTE no-execute bit. */ 596 if ((amd_feature & AMDID_NX) != 0) { 597 uint64_t msr; 598 599 msr = rdmsr(MSR_EFER) | EFER_NXE; 600 wrmsr(MSR_EFER, msr); 601 } 602 #endif 603 604 /* A quick check from sanity claus */ 605 if (PCPU_GET(apic_id) != lapic_id()) { 606 printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); 607 printf("SMP: actual apic_id = %d\n", lapic_id()); 608 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); 609 panic("cpuid mismatch! boom!!"); 610 } 611 612 /* Initialize curthread. */ 613 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 614 PCPU_SET(curthread, PCPU_GET(idlethread)); 615 616 mca_init(); 617 618 mtx_lock_spin(&ap_boot_mtx); 619 620 /* Init local apic for irq's */ 621 lapic_setup(1); 622 623 /* Set memory range attributes for this CPU to match the BSP */ 624 mem_range_AP_init(); 625 626 smp_cpus++; 627 628 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); 629 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); 630 631 /* Determine if we are a logical CPU. */ 632 if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0) 633 logical_cpus_mask |= PCPU_GET(cpumask); 634 635 /* Determine if we are a hyperthread. */ 636 if (hyperthreading_cpus > 1 && 637 PCPU_GET(apic_id) % hyperthreading_cpus != 0) 638 hyperthreading_cpus_mask |= PCPU_GET(cpumask); 639 640 /* Build our map of 'other' CPUs. */ 641 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); 642 643 if (bootverbose) 644 lapic_dump("AP"); 645 646 if (smp_cpus == mp_ncpus) { 647 /* enable IPI's, tlb shootdown, freezes etc */ 648 atomic_store_rel_int(&smp_started, 1); 649 smp_active = 1; /* historic */ 650 } 651 652 mtx_unlock_spin(&ap_boot_mtx); 653 654 /* wait until all the AP's are up */ 655 while (smp_started == 0) 656 ia32_pause(); 657 658 /* enter the scheduler */ 659 sched_throw(NULL); 660 661 panic("scheduler returned us to %s", __func__); 662 /* NOTREACHED */ 663 } 664 665 /******************************************************************* 666 * local functions and data 667 */ 668 669 /* 670 * We tell the I/O APIC code about all the CPUs we want to receive 671 * interrupts. If we don't want certain CPUs to receive IRQs we 672 * can simply not tell the I/O APIC code about them in this function. 673 * We also do not tell it about the BSP since it tells itself about 674 * the BSP internally to work with UP kernels and on UP machines. 675 */ 676 static void 677 set_interrupt_apic_ids(void) 678 { 679 u_int i, apic_id; 680 681 for (i = 0; i < MAXCPU; i++) { 682 apic_id = cpu_apic_ids[i]; 683 if (apic_id == -1) 684 continue; 685 if (cpu_info[apic_id].cpu_bsp) 686 continue; 687 if (cpu_info[apic_id].cpu_disabled) 688 continue; 689 690 /* Don't let hyperthreads service interrupts. */ 691 if (hyperthreading_cpus > 1 && 692 apic_id % hyperthreading_cpus != 0) 693 continue; 694 695 intr_add_cpu(i); 696 } 697 } 698 699 /* 700 * Assign logical CPU IDs to local APICs. 701 */ 702 static void 703 assign_cpu_ids(void) 704 { 705 u_int i; 706 707 TUNABLE_INT_FETCH("machdep.hyperthreading_allowed", 708 &hyperthreading_allowed); 709 710 /* Check for explicitly disabled CPUs. */ 711 for (i = 0; i <= MAX_APIC_ID; i++) { 712 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp) 713 continue; 714 715 if (hyperthreading_cpus > 1 && i % hyperthreading_cpus != 0) { 716 cpu_info[i].cpu_hyperthread = 1; 717 #if defined(SCHED_ULE) 718 /* 719 * Don't use HT CPU if it has been disabled by a 720 * tunable. 721 */ 722 if (hyperthreading_allowed == 0) { 723 cpu_info[i].cpu_disabled = 1; 724 continue; 725 } 726 #endif 727 } 728 729 /* Don't use this CPU if it has been disabled by a tunable. */ 730 if (resource_disabled("lapic", i)) { 731 cpu_info[i].cpu_disabled = 1; 732 continue; 733 } 734 } 735 736 /* 737 * Assign CPU IDs to local APIC IDs and disable any CPUs 738 * beyond MAXCPU. CPU 0 is always assigned to the BSP. 739 * 740 * To minimize confusion for userland, we attempt to number 741 * CPUs such that all threads and cores in a package are 742 * grouped together. For now we assume that the BSP is always 743 * the first thread in a package and just start adding APs 744 * starting with the BSP's APIC ID. 745 */ 746 mp_ncpus = 1; 747 cpu_apic_ids[0] = boot_cpu_id; 748 for (i = boot_cpu_id + 1; i != boot_cpu_id; 749 i == MAX_APIC_ID ? i = 0 : i++) { 750 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp || 751 cpu_info[i].cpu_disabled) 752 continue; 753 754 if (mp_ncpus < MAXCPU) { 755 cpu_apic_ids[mp_ncpus] = i; 756 mp_ncpus++; 757 } else 758 cpu_info[i].cpu_disabled = 1; 759 } 760 KASSERT(mp_maxid >= mp_ncpus - 1, 761 ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid, 762 mp_ncpus)); 763 } 764 765 /* 766 * start each AP in our list 767 */ 768 static int 769 start_all_aps(void) 770 { 771 #ifndef PC98 772 u_char mpbiosreason; 773 #endif 774 uintptr_t kptbase; 775 u_int32_t mpbioswarmvec; 776 int apic_id, cpu, i; 777 778 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 779 780 /* install the AP 1st level boot code */ 781 install_ap_tramp(); 782 783 /* save the current value of the warm-start vector */ 784 mpbioswarmvec = *((u_int32_t *) WARMBOOT_OFF); 785 #ifndef PC98 786 outb(CMOS_REG, BIOS_RESET); 787 mpbiosreason = inb(CMOS_DATA); 788 #endif 789 790 /* set up temporary P==V mapping for AP boot */ 791 /* XXX this is a hack, we should boot the AP on its own stack/PTD */ 792 kptbase = (uintptr_t)(void *)KPTphys; 793 for (i = 0; i < NKPT; i++) 794 PTD[i] = (pd_entry_t)(PG_V | PG_RW | 795 ((kptbase + i * PAGE_SIZE) & PG_FRAME)); 796 invltlb(); 797 798 /* start each AP */ 799 for (cpu = 1; cpu < mp_ncpus; cpu++) { 800 apic_id = cpu_apic_ids[cpu]; 801 802 /* allocate and set up a boot stack data page */ 803 bootstacks[cpu] = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); 804 805 /* setup a vector to our boot code */ 806 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 807 *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4); 808 #ifndef PC98 809 outb(CMOS_REG, BIOS_RESET); 810 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 811 #endif 812 813 bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 4; 814 bootAP = cpu; 815 816 /* attempt to start the Application Processor */ 817 CHECK_INIT(99); /* setup checkpoints */ 818 if (!start_ap(apic_id)) { 819 printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id); 820 CHECK_PRINT("trace"); /* show checkpoints */ 821 /* better panic as the AP may be running loose */ 822 printf("panic y/n? [y] "); 823 if (cngetc() != 'n') 824 panic("bye-bye"); 825 } 826 CHECK_PRINT("trace"); /* show checkpoints */ 827 828 all_cpus |= (1 << cpu); /* record AP in CPU map */ 829 } 830 831 /* build our map of 'other' CPUs */ 832 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); 833 834 /* restore the warmstart vector */ 835 *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; 836 837 #ifndef PC98 838 outb(CMOS_REG, BIOS_RESET); 839 outb(CMOS_DATA, mpbiosreason); 840 #endif 841 842 /* Undo V==P hack from above */ 843 for (i = 0; i < NKPT; i++) 844 PTD[i] = 0; 845 pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); 846 847 /* number of APs actually started */ 848 return mp_naps; 849 } 850 851 /* 852 * load the 1st level AP boot code into base memory. 853 */ 854 855 /* targets for relocation */ 856 extern void bigJump(void); 857 extern void bootCodeSeg(void); 858 extern void bootDataSeg(void); 859 extern void MPentry(void); 860 extern u_int MP_GDT; 861 extern u_int mp_gdtbase; 862 863 static void 864 install_ap_tramp(void) 865 { 866 int x; 867 int size = *(int *) ((u_long) & bootMP_size); 868 vm_offset_t va = boot_address + KERNBASE; 869 u_char *src = (u_char *) ((u_long) bootMP); 870 u_char *dst = (u_char *) va; 871 u_int boot_base = (u_int) bootMP; 872 u_int8_t *dst8; 873 u_int16_t *dst16; 874 u_int32_t *dst32; 875 876 KASSERT (size <= PAGE_SIZE, 877 ("'size' do not fit into PAGE_SIZE, as expected.")); 878 pmap_kenter(va, boot_address); 879 pmap_invalidate_page (kernel_pmap, va); 880 for (x = 0; x < size; ++x) 881 *dst++ = *src++; 882 883 /* 884 * modify addresses in code we just moved to basemem. unfortunately we 885 * need fairly detailed info about mpboot.s for this to work. changes 886 * to mpboot.s might require changes here. 887 */ 888 889 /* boot code is located in KERNEL space */ 890 dst = (u_char *) va; 891 892 /* modify the lgdt arg */ 893 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); 894 *dst32 = boot_address + ((u_int) & MP_GDT - boot_base); 895 896 /* modify the ljmp target for MPentry() */ 897 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); 898 *dst32 = ((u_int) MPentry - KERNBASE); 899 900 /* modify the target for boot code segment */ 901 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); 902 dst8 = (u_int8_t *) (dst16 + 1); 903 *dst16 = (u_int) boot_address & 0xffff; 904 *dst8 = ((u_int) boot_address >> 16) & 0xff; 905 906 /* modify the target for boot data segment */ 907 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); 908 dst8 = (u_int8_t *) (dst16 + 1); 909 *dst16 = (u_int) boot_address & 0xffff; 910 *dst8 = ((u_int) boot_address >> 16) & 0xff; 911 } 912 913 /* 914 * This function starts the AP (application processor) identified 915 * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 916 * to accomplish this. This is necessary because of the nuances 917 * of the different hardware we might encounter. It isn't pretty, 918 * but it seems to work. 919 */ 920 static int 921 start_ap(int apic_id) 922 { 923 int vector, ms; 924 int cpus; 925 926 /* calculate the vector */ 927 vector = (boot_address >> 12) & 0xff; 928 929 /* used as a watchpoint to signal AP startup */ 930 cpus = mp_naps; 931 932 /* 933 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting 934 * and running the target CPU. OR this INIT IPI might be latched (P5 935 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be 936 * ignored. 937 */ 938 939 /* do an INIT IPI: assert RESET */ 940 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 941 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); 942 943 /* wait for pending status end */ 944 lapic_ipi_wait(-1); 945 946 /* do an INIT IPI: deassert RESET */ 947 lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL | 948 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0); 949 950 /* wait for pending status end */ 951 DELAY(10000); /* wait ~10mS */ 952 lapic_ipi_wait(-1); 953 954 /* 955 * next we do a STARTUP IPI: the previous INIT IPI might still be 956 * latched, (P5 bug) this 1st STARTUP would then terminate 957 * immediately, and the previously started INIT IPI would continue. OR 958 * the previous INIT IPI has already run. and this STARTUP IPI will 959 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI 960 * will run. 961 */ 962 963 /* do a STARTUP IPI */ 964 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 965 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | 966 vector, apic_id); 967 lapic_ipi_wait(-1); 968 DELAY(200); /* wait ~200uS */ 969 970 /* 971 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF 972 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR 973 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is 974 * recognized after hardware RESET or INIT IPI. 975 */ 976 977 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 978 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | 979 vector, apic_id); 980 lapic_ipi_wait(-1); 981 DELAY(200); /* wait ~200uS */ 982 983 /* Wait up to 5 seconds for it to start. */ 984 for (ms = 0; ms < 5000; ms++) { 985 if (mp_naps > cpus) 986 return 1; /* return SUCCESS */ 987 DELAY(1000); 988 } 989 return 0; /* return FAILURE */ 990 } 991 992 #ifdef COUNT_XINVLTLB_HITS 993 u_int xhits_gbl[MAXCPU]; 994 u_int xhits_pg[MAXCPU]; 995 u_int xhits_rng[MAXCPU]; 996 SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); 997 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, 998 sizeof(xhits_gbl), "IU", ""); 999 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, 1000 sizeof(xhits_pg), "IU", ""); 1001 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, 1002 sizeof(xhits_rng), "IU", ""); 1003 1004 u_int ipi_global; 1005 u_int ipi_page; 1006 u_int ipi_range; 1007 u_int ipi_range_size; 1008 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); 1009 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); 1010 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); 1011 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, 1012 0, ""); 1013 1014 u_int ipi_masked_global; 1015 u_int ipi_masked_page; 1016 u_int ipi_masked_range; 1017 u_int ipi_masked_range_size; 1018 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, 1019 &ipi_masked_global, 0, ""); 1020 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, 1021 &ipi_masked_page, 0, ""); 1022 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, 1023 &ipi_masked_range, 0, ""); 1024 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, 1025 &ipi_masked_range_size, 0, ""); 1026 #endif /* COUNT_XINVLTLB_HITS */ 1027 1028 /* 1029 * Flush the TLB on all other CPU's 1030 */ 1031 static void 1032 smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) 1033 { 1034 u_int ncpu; 1035 1036 ncpu = mp_ncpus - 1; /* does not shootdown self */ 1037 if (ncpu < 1) 1038 return; /* no other cpus */ 1039 if (!(read_eflags() & PSL_I)) 1040 panic("%s: interrupts disabled", __func__); 1041 mtx_lock_spin(&smp_ipi_mtx); 1042 smp_tlb_addr1 = addr1; 1043 smp_tlb_addr2 = addr2; 1044 atomic_store_rel_int(&smp_tlb_wait, 0); 1045 ipi_all_but_self(vector); 1046 while (smp_tlb_wait < ncpu) 1047 ia32_pause(); 1048 mtx_unlock_spin(&smp_ipi_mtx); 1049 } 1050 1051 static void 1052 smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) 1053 { 1054 int ncpu, othercpus; 1055 1056 othercpus = mp_ncpus - 1; 1057 if (mask == (u_int)-1) { 1058 ncpu = othercpus; 1059 if (ncpu < 1) 1060 return; 1061 } else { 1062 mask &= ~PCPU_GET(cpumask); 1063 if (mask == 0) 1064 return; 1065 ncpu = bitcount32(mask); 1066 if (ncpu > othercpus) { 1067 /* XXX this should be a panic offence */ 1068 printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", 1069 ncpu, othercpus); 1070 ncpu = othercpus; 1071 } 1072 /* XXX should be a panic, implied by mask == 0 above */ 1073 if (ncpu < 1) 1074 return; 1075 } 1076 if (!(read_eflags() & PSL_I)) 1077 panic("%s: interrupts disabled", __func__); 1078 mtx_lock_spin(&smp_ipi_mtx); 1079 smp_tlb_addr1 = addr1; 1080 smp_tlb_addr2 = addr2; 1081 atomic_store_rel_int(&smp_tlb_wait, 0); 1082 if (mask == (u_int)-1) 1083 ipi_all_but_self(vector); 1084 else 1085 ipi_selected(mask, vector); 1086 while (smp_tlb_wait < ncpu) 1087 ia32_pause(); 1088 mtx_unlock_spin(&smp_ipi_mtx); 1089 } 1090 1091 void 1092 smp_cache_flush(void) 1093 { 1094 1095 if (smp_started) 1096 smp_tlb_shootdown(IPI_INVLCACHE, 0, 0); 1097 } 1098 1099 void 1100 smp_invltlb(void) 1101 { 1102 1103 if (smp_started) { 1104 smp_tlb_shootdown(IPI_INVLTLB, 0, 0); 1105 #ifdef COUNT_XINVLTLB_HITS 1106 ipi_global++; 1107 #endif 1108 } 1109 } 1110 1111 void 1112 smp_invlpg(vm_offset_t addr) 1113 { 1114 1115 if (smp_started) { 1116 smp_tlb_shootdown(IPI_INVLPG, addr, 0); 1117 #ifdef COUNT_XINVLTLB_HITS 1118 ipi_page++; 1119 #endif 1120 } 1121 } 1122 1123 void 1124 smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) 1125 { 1126 1127 if (smp_started) { 1128 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); 1129 #ifdef COUNT_XINVLTLB_HITS 1130 ipi_range++; 1131 ipi_range_size += (addr2 - addr1) / PAGE_SIZE; 1132 #endif 1133 } 1134 } 1135 1136 void 1137 smp_masked_invltlb(u_int mask) 1138 { 1139 1140 if (smp_started) { 1141 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); 1142 #ifdef COUNT_XINVLTLB_HITS 1143 ipi_masked_global++; 1144 #endif 1145 } 1146 } 1147 1148 void 1149 smp_masked_invlpg(u_int mask, vm_offset_t addr) 1150 { 1151 1152 if (smp_started) { 1153 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); 1154 #ifdef COUNT_XINVLTLB_HITS 1155 ipi_masked_page++; 1156 #endif 1157 } 1158 } 1159 1160 void 1161 smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) 1162 { 1163 1164 if (smp_started) { 1165 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); 1166 #ifdef COUNT_XINVLTLB_HITS 1167 ipi_masked_range++; 1168 ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; 1169 #endif 1170 } 1171 } 1172 1173 void 1174 ipi_bitmap_handler(struct trapframe frame) 1175 { 1176 int cpu = PCPU_GET(cpuid); 1177 u_int ipi_bitmap; 1178 1179 ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]); 1180 1181 if (ipi_bitmap & (1 << IPI_PREEMPT)) { 1182 struct thread *running_thread = curthread; 1183 #ifdef COUNT_IPIS 1184 (*ipi_preempt_counts[cpu])++; 1185 #endif 1186 thread_lock(running_thread); 1187 if (running_thread->td_critnest > 1) 1188 running_thread->td_owepreempt = 1; 1189 else 1190 mi_switch(SW_INVOL | SW_PREEMPT, NULL); 1191 thread_unlock(running_thread); 1192 } 1193 1194 if (ipi_bitmap & (1 << IPI_AST)) { 1195 #ifdef COUNT_IPIS 1196 (*ipi_ast_counts[cpu])++; 1197 #endif 1198 /* Nothing to do for AST */ 1199 } 1200 } 1201 1202 /* 1203 * send an IPI to a set of cpus. 1204 */ 1205 void 1206 ipi_selected(u_int32_t cpus, u_int ipi) 1207 { 1208 int cpu; 1209 u_int bitmap = 0; 1210 u_int old_pending; 1211 u_int new_pending; 1212 1213 if (IPI_IS_BITMAPED(ipi)) { 1214 bitmap = 1 << ipi; 1215 ipi = IPI_BITMAP_VECTOR; 1216 } 1217 1218 #ifdef STOP_NMI 1219 if (ipi == IPI_STOP && stop_cpus_with_nmi) { 1220 ipi_nmi_selected(cpus); 1221 return; 1222 } 1223 #endif 1224 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); 1225 while ((cpu = ffs(cpus)) != 0) { 1226 cpu--; 1227 cpus &= ~(1 << cpu); 1228 1229 KASSERT(cpu_apic_ids[cpu] != -1, 1230 ("IPI to non-existent CPU %d", cpu)); 1231 1232 if (bitmap) { 1233 do { 1234 old_pending = cpu_ipi_pending[cpu]; 1235 new_pending = old_pending | bitmap; 1236 } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending)); 1237 1238 if (old_pending) 1239 continue; 1240 } 1241 1242 lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); 1243 } 1244 1245 } 1246 1247 /* 1248 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself 1249 */ 1250 void 1251 ipi_all(u_int ipi) 1252 { 1253 1254 if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { 1255 ipi_selected(all_cpus, ipi); 1256 return; 1257 } 1258 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1259 lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL); 1260 } 1261 1262 /* 1263 * send an IPI to all CPUs EXCEPT myself 1264 */ 1265 void 1266 ipi_all_but_self(u_int ipi) 1267 { 1268 1269 if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { 1270 ipi_selected(PCPU_GET(other_cpus), ipi); 1271 return; 1272 } 1273 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1274 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); 1275 } 1276 1277 /* 1278 * send an IPI to myself 1279 */ 1280 void 1281 ipi_self(u_int ipi) 1282 { 1283 1284 if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { 1285 ipi_selected(PCPU_GET(cpumask), ipi); 1286 return; 1287 } 1288 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1289 lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); 1290 } 1291 1292 #ifdef STOP_NMI 1293 /* 1294 * send NMI IPI to selected CPUs 1295 */ 1296 1297 #define BEFORE_SPIN 1000000 1298 1299 void 1300 ipi_nmi_selected(u_int32_t cpus) 1301 { 1302 int cpu; 1303 register_t icrlo; 1304 1305 icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT 1306 | APIC_TRIGMOD_EDGE; 1307 1308 CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus); 1309 1310 atomic_set_int(&ipi_nmi_pending, cpus); 1311 1312 while ((cpu = ffs(cpus)) != 0) { 1313 cpu--; 1314 cpus &= ~(1 << cpu); 1315 1316 KASSERT(cpu_apic_ids[cpu] != -1, 1317 ("IPI NMI to non-existent CPU %d", cpu)); 1318 1319 /* Wait for an earlier IPI to finish. */ 1320 if (!lapic_ipi_wait(BEFORE_SPIN)) 1321 panic("ipi_nmi_selected: previous IPI has not cleared"); 1322 1323 lapic_ipi_raw(icrlo, cpu_apic_ids[cpu]); 1324 } 1325 } 1326 1327 int 1328 ipi_nmi_handler(void) 1329 { 1330 int cpumask = PCPU_GET(cpumask); 1331 1332 if (!(ipi_nmi_pending & cpumask)) 1333 return 1; 1334 1335 atomic_clear_int(&ipi_nmi_pending, cpumask); 1336 cpustop_handler(); 1337 return 0; 1338 } 1339 1340 #endif /* STOP_NMI */ 1341 1342 /* 1343 * Handle an IPI_STOP by saving our current context and spinning until we 1344 * are resumed. 1345 */ 1346 void 1347 cpustop_handler(void) 1348 { 1349 int cpu = PCPU_GET(cpuid); 1350 int cpumask = PCPU_GET(cpumask); 1351 1352 savectx(&stoppcbs[cpu]); 1353 1354 /* Indicate that we are stopped */ 1355 atomic_set_int(&stopped_cpus, cpumask); 1356 1357 /* Wait for restart */ 1358 while (!(started_cpus & cpumask)) 1359 ia32_pause(); 1360 1361 atomic_clear_int(&started_cpus, cpumask); 1362 atomic_clear_int(&stopped_cpus, cpumask); 1363 1364 if (cpu == 0 && cpustop_restartfunc != NULL) { 1365 cpustop_restartfunc(); 1366 cpustop_restartfunc = NULL; 1367 } 1368 } 1369 1370 /* 1371 * This is called once the rest of the system is up and running and we're 1372 * ready to let the AP's out of the pen. 1373 */ 1374 static void 1375 release_aps(void *dummy __unused) 1376 { 1377 1378 if (mp_ncpus == 1) 1379 return; 1380 atomic_store_rel_int(&aps_ready, 1); 1381 while (smp_started == 0) 1382 ia32_pause(); 1383 } 1384 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 1385 1386 static int 1387 sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS) 1388 { 1389 u_int mask; 1390 int error; 1391 1392 mask = hlt_cpus_mask; 1393 error = sysctl_handle_int(oidp, &mask, 0, req); 1394 if (error || !req->newptr) 1395 return (error); 1396 1397 if (logical_cpus_mask != 0 && 1398 (mask & logical_cpus_mask) == logical_cpus_mask) 1399 hlt_logical_cpus = 1; 1400 else 1401 hlt_logical_cpus = 0; 1402 1403 if (! hyperthreading_allowed) 1404 mask |= hyperthreading_cpus_mask; 1405 1406 if ((mask & all_cpus) == all_cpus) 1407 mask &= ~(1<<0); 1408 hlt_cpus_mask = mask; 1409 return (error); 1410 } 1411 SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW, 1412 0, 0, sysctl_hlt_cpus, "IU", 1413 "Bitmap of CPUs to halt. 101 (binary) will halt CPUs 0 and 2."); 1414 1415 static int 1416 sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS) 1417 { 1418 int disable, error; 1419 1420 disable = hlt_logical_cpus; 1421 error = sysctl_handle_int(oidp, &disable, 0, req); 1422 if (error || !req->newptr) 1423 return (error); 1424 1425 if (disable) 1426 hlt_cpus_mask |= logical_cpus_mask; 1427 else 1428 hlt_cpus_mask &= ~logical_cpus_mask; 1429 1430 if (! hyperthreading_allowed) 1431 hlt_cpus_mask |= hyperthreading_cpus_mask; 1432 1433 if ((hlt_cpus_mask & all_cpus) == all_cpus) 1434 hlt_cpus_mask &= ~(1<<0); 1435 1436 hlt_logical_cpus = disable; 1437 return (error); 1438 } 1439 1440 static int 1441 sysctl_hyperthreading_allowed(SYSCTL_HANDLER_ARGS) 1442 { 1443 int allowed, error; 1444 1445 allowed = hyperthreading_allowed; 1446 error = sysctl_handle_int(oidp, &allowed, 0, req); 1447 if (error || !req->newptr) 1448 return (error); 1449 1450 #ifdef SCHED_ULE 1451 /* 1452 * SCHED_ULE doesn't allow enabling/disabling HT cores at 1453 * run-time. 1454 */ 1455 if (allowed != hyperthreading_allowed) 1456 return (ENOTSUP); 1457 return (error); 1458 #endif 1459 1460 if (allowed) 1461 hlt_cpus_mask &= ~hyperthreading_cpus_mask; 1462 else 1463 hlt_cpus_mask |= hyperthreading_cpus_mask; 1464 1465 if (logical_cpus_mask != 0 && 1466 (hlt_cpus_mask & logical_cpus_mask) == logical_cpus_mask) 1467 hlt_logical_cpus = 1; 1468 else 1469 hlt_logical_cpus = 0; 1470 1471 if ((hlt_cpus_mask & all_cpus) == all_cpus) 1472 hlt_cpus_mask &= ~(1<<0); 1473 1474 hyperthreading_allowed = allowed; 1475 return (error); 1476 } 1477 1478 static void 1479 cpu_hlt_setup(void *dummy __unused) 1480 { 1481 1482 if (logical_cpus_mask != 0) { 1483 TUNABLE_INT_FETCH("machdep.hlt_logical_cpus", 1484 &hlt_logical_cpus); 1485 sysctl_ctx_init(&logical_cpu_clist); 1486 SYSCTL_ADD_PROC(&logical_cpu_clist, 1487 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, 1488 "hlt_logical_cpus", CTLTYPE_INT|CTLFLAG_RW, 0, 0, 1489 sysctl_hlt_logical_cpus, "IU", ""); 1490 SYSCTL_ADD_UINT(&logical_cpu_clist, 1491 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, 1492 "logical_cpus_mask", CTLTYPE_INT|CTLFLAG_RD, 1493 &logical_cpus_mask, 0, ""); 1494 1495 if (hlt_logical_cpus) 1496 hlt_cpus_mask |= logical_cpus_mask; 1497 1498 /* 1499 * If necessary for security purposes, force 1500 * hyperthreading off, regardless of the value 1501 * of hlt_logical_cpus. 1502 */ 1503 if (hyperthreading_cpus_mask) { 1504 SYSCTL_ADD_PROC(&logical_cpu_clist, 1505 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, 1506 "hyperthreading_allowed", CTLTYPE_INT|CTLFLAG_RW, 1507 0, 0, sysctl_hyperthreading_allowed, "IU", ""); 1508 if (! hyperthreading_allowed) 1509 hlt_cpus_mask |= hyperthreading_cpus_mask; 1510 } 1511 } 1512 } 1513 SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_ANY, cpu_hlt_setup, NULL); 1514 1515 int 1516 mp_grab_cpu_hlt(void) 1517 { 1518 u_int mask = PCPU_GET(cpumask); 1519 #ifdef MP_WATCHDOG 1520 u_int cpuid = PCPU_GET(cpuid); 1521 #endif 1522 int retval; 1523 1524 #ifdef MP_WATCHDOG 1525 ap_watchdog(cpuid); 1526 #endif 1527 1528 retval = mask & hlt_cpus_mask; 1529 while (mask & hlt_cpus_mask) 1530 __asm __volatile("sti; hlt" : : : "memory"); 1531 return (retval); 1532 } 1533 1534 #ifdef COUNT_IPIS 1535 /* 1536 * Setup interrupt counters for IPI handlers. 1537 */ 1538 static void 1539 mp_ipi_intrcnt(void *dummy) 1540 { 1541 char buf[64]; 1542 int i; 1543 1544 for (i = 0; i < mp_maxid; i++) { 1545 if (CPU_ABSENT(i)) 1546 continue; 1547 snprintf(buf, sizeof(buf), "cpu%d: invltlb", i); 1548 intrcnt_add(buf, &ipi_invltlb_counts[i]); 1549 snprintf(buf, sizeof(buf), "cpu%d: invlrng", i); 1550 intrcnt_add(buf, &ipi_invlrng_counts[i]); 1551 snprintf(buf, sizeof(buf), "cpu%d: invlpg", i); 1552 intrcnt_add(buf, &ipi_invlpg_counts[i]); 1553 snprintf(buf, sizeof(buf), "cpu%d: preempt", i); 1554 intrcnt_add(buf, &ipi_preempt_counts[i]); 1555 snprintf(buf, sizeof(buf), "cpu%d: ast", i); 1556 intrcnt_add(buf, &ipi_ast_counts[i]); 1557 snprintf(buf, sizeof(buf), "cpu%d: rendezvous", i); 1558 intrcnt_add(buf, &ipi_rendezvous_counts[i]); 1559 snprintf(buf, sizeof(buf), "cpu%d: lazypmap", i); 1560 intrcnt_add(buf, &ipi_lazypmap_counts[i]); 1561 } 1562 } 1563 SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); 1564 #endif
Cache object: 10310e95488be3e3d0030f320ae2da73
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]