Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/mp_machdep.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1996, by Steve Passe 3 * Copyright (c) 2003, by Peter Wemm 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. The name of the developer may NOT be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "opt_cpu.h" 31 #include "opt_ddb.h" 32 #include "opt_kstack_pages.h" 33 #include "opt_sched.h" 34 #include "opt_smp.h" 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/bus.h> 39 #include <sys/cpuset.h> 40 #ifdef GPROF 41 #include <sys/gmon.h> 42 #endif 43 #include <sys/kernel.h> 44 #include <sys/ktr.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/memrange.h> 48 #include <sys/mutex.h> 49 #include <sys/pcpu.h> 50 #include <sys/proc.h> 51 #include <sys/sched.h> 52 #include <sys/smp.h> 53 #include <sys/sysctl.h> 54 55 #include <vm/vm.h> 56 #include <vm/vm_param.h> 57 #include <vm/pmap.h> 58 #include <vm/vm_kern.h> 59 #include <vm/vm_extern.h> 60 61 #include <x86/apicreg.h> 62 #include <machine/clock.h> 63 #include <machine/cputypes.h> 64 #include <machine/cpufunc.h> 65 #include <x86/mca.h> 66 #include <machine/md_var.h> 67 #include <machine/pcb.h> 68 #include <machine/psl.h> 69 #include <machine/smp.h> 70 #include <machine/specialreg.h> 71 #include <machine/tss.h> 72 73 #define WARMBOOT_TARGET 0 74 #define WARMBOOT_OFF (KERNBASE + 0x0467) 75 #define WARMBOOT_SEG (KERNBASE + 0x0469) 76 77 #define CMOS_REG (0x70) 78 #define CMOS_DATA (0x71) 79 #define BIOS_RESET (0x0f) 80 #define BIOS_WARM (0x0a) 81 82 /* lock region used by kernel profiling */ 83 int mcount_lock; 84 85 int mp_naps; /* # of Applications processors */ 86 int boot_cpu_id = -1; /* designated BSP */ 87 88 extern struct pcpu __pcpu[]; 89 90 /* AP uses this during bootstrap. Do not staticize. */ 91 char *bootSTK; 92 static int bootAP; 93 94 /* Free these after use */ 95 void *bootstacks[MAXCPU]; 96 97 /* Temporary variables for init_secondary() */ 98 char *doublefault_stack; 99 char *nmi_stack; 100 void *dpcpu; 101 102 struct pcb stoppcbs[MAXCPU]; 103 struct pcb **susppcbs; 104 void **suspfpusave; 105 106 /* Variables needed for SMP tlb shootdown. */ 107 vm_offset_t smp_tlb_addr1; 108 vm_offset_t smp_tlb_addr2; 109 volatile int smp_tlb_wait; 110 111 #ifdef COUNT_IPIS 112 /* Interrupt counts. */ 113 static u_long *ipi_preempt_counts[MAXCPU]; 114 static u_long *ipi_ast_counts[MAXCPU]; 115 u_long *ipi_invltlb_counts[MAXCPU]; 116 u_long *ipi_invlrng_counts[MAXCPU]; 117 u_long *ipi_invlpg_counts[MAXCPU]; 118 u_long *ipi_invlcache_counts[MAXCPU]; 119 u_long *ipi_rendezvous_counts[MAXCPU]; 120 static u_long *ipi_hardclock_counts[MAXCPU]; 121 #endif 122 123 extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); 124 125 /* 126 * Local data and functions. 127 */ 128 129 static volatile cpuset_t ipi_nmi_pending; 130 131 /* used to hold the AP's until we are ready to release them */ 132 static struct mtx ap_boot_mtx; 133 134 /* Set to 1 once we're ready to let the APs out of the pen. */ 135 static volatile int aps_ready = 0; 136 137 /* 138 * Store data from cpu_add() until later in the boot when we actually setup 139 * the APs. 140 */ 141 struct cpu_info { 142 int cpu_present:1; 143 int cpu_bsp:1; 144 int cpu_disabled:1; 145 int cpu_hyperthread:1; 146 } static cpu_info[MAX_APIC_ID + 1]; 147 int cpu_apic_ids[MAXCPU]; 148 int apic_cpuids[MAX_APIC_ID + 1]; 149 150 /* Holds pending bitmap based IPIs per CPU */ 151 static volatile u_int cpu_ipi_pending[MAXCPU]; 152 153 static u_int boot_address; 154 static int cpu_logical; /* logical cpus per core */ 155 static int cpu_cores; /* cores per package */ 156 157 static void assign_cpu_ids(void); 158 static void set_interrupt_apic_ids(void); 159 static int start_all_aps(void); 160 static int start_ap(int apic_id); 161 static void release_aps(void *dummy); 162 163 static u_int hyperthreading_cpus; /* logical cpus sharing L1 cache */ 164 static int hyperthreading_allowed = 1; 165 static u_int bootMP_size; 166 167 static void 168 mem_range_AP_init(void) 169 { 170 if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP) 171 mem_range_softc.mr_op->initAP(&mem_range_softc); 172 } 173 174 static void 175 topo_probe_amd(void) 176 { 177 int core_id_bits; 178 int id; 179 180 /* AMD processors do not support HTT. */ 181 cpu_logical = 1; 182 183 if ((amd_feature2 & AMDID2_CMP) == 0) { 184 cpu_cores = 1; 185 return; 186 } 187 188 core_id_bits = (cpu_procinfo2 & AMDID_COREID_SIZE) >> 189 AMDID_COREID_SIZE_SHIFT; 190 if (core_id_bits == 0) { 191 cpu_cores = (cpu_procinfo2 & AMDID_CMP_CORES) + 1; 192 return; 193 } 194 195 /* Fam 10h and newer should get here. */ 196 for (id = 0; id <= MAX_APIC_ID; id++) { 197 /* Check logical CPU availability. */ 198 if (!cpu_info[id].cpu_present || cpu_info[id].cpu_disabled) 199 continue; 200 /* Check if logical CPU has the same package ID. */ 201 if ((id >> core_id_bits) != (boot_cpu_id >> core_id_bits)) 202 continue; 203 cpu_cores++; 204 } 205 } 206 207 /* 208 * Round up to the next power of two, if necessary, and then 209 * take log2. 210 * Returns -1 if argument is zero. 211 */ 212 static __inline int 213 mask_width(u_int x) 214 { 215 216 return (fls(x << (1 - powerof2(x))) - 1); 217 } 218 219 static void 220 topo_probe_0x4(void) 221 { 222 u_int p[4]; 223 int pkg_id_bits; 224 int core_id_bits; 225 int max_cores; 226 int max_logical; 227 int id; 228 229 /* Both zero and one here mean one logical processor per package. */ 230 max_logical = (cpu_feature & CPUID_HTT) != 0 ? 231 (cpu_procinfo & CPUID_HTT_CORES) >> 16 : 1; 232 if (max_logical <= 1) 233 return; 234 235 /* 236 * Because of uniformity assumption we examine only 237 * those logical processors that belong to the same 238 * package as BSP. Further, we count number of 239 * logical processors that belong to the same core 240 * as BSP thus deducing number of threads per core. 241 */ 242 if (cpu_high >= 0x4) { 243 cpuid_count(0x04, 0, p); 244 max_cores = ((p[0] >> 26) & 0x3f) + 1; 245 } else 246 max_cores = 1; 247 core_id_bits = mask_width(max_logical/max_cores); 248 if (core_id_bits < 0) 249 return; 250 pkg_id_bits = core_id_bits + mask_width(max_cores); 251 252 for (id = 0; id <= MAX_APIC_ID; id++) { 253 /* Check logical CPU availability. */ 254 if (!cpu_info[id].cpu_present || cpu_info[id].cpu_disabled) 255 continue; 256 /* Check if logical CPU has the same package ID. */ 257 if ((id >> pkg_id_bits) != (boot_cpu_id >> pkg_id_bits)) 258 continue; 259 cpu_cores++; 260 /* Check if logical CPU has the same package and core IDs. */ 261 if ((id >> core_id_bits) == (boot_cpu_id >> core_id_bits)) 262 cpu_logical++; 263 } 264 265 KASSERT(cpu_cores >= 1 && cpu_logical >= 1, 266 ("topo_probe_0x4 couldn't find BSP")); 267 268 cpu_cores /= cpu_logical; 269 hyperthreading_cpus = cpu_logical; 270 } 271 272 static void 273 topo_probe_0xb(void) 274 { 275 u_int p[4]; 276 int bits; 277 int cnt; 278 int i; 279 int logical; 280 int type; 281 int x; 282 283 /* We only support three levels for now. */ 284 for (i = 0; i < 3; i++) { 285 cpuid_count(0x0b, i, p); 286 287 /* Fall back if CPU leaf 11 doesn't really exist. */ 288 if (i == 0 && p[1] == 0) { 289 topo_probe_0x4(); 290 return; 291 } 292 293 bits = p[0] & 0x1f; 294 logical = p[1] &= 0xffff; 295 type = (p[2] >> 8) & 0xff; 296 if (type == 0 || logical == 0) 297 break; 298 /* 299 * Because of uniformity assumption we examine only 300 * those logical processors that belong to the same 301 * package as BSP. 302 */ 303 for (cnt = 0, x = 0; x <= MAX_APIC_ID; x++) { 304 if (!cpu_info[x].cpu_present || 305 cpu_info[x].cpu_disabled) 306 continue; 307 if (x >> bits == boot_cpu_id >> bits) 308 cnt++; 309 } 310 if (type == CPUID_TYPE_SMT) 311 cpu_logical = cnt; 312 else if (type == CPUID_TYPE_CORE) 313 cpu_cores = cnt; 314 } 315 if (cpu_logical == 0) 316 cpu_logical = 1; 317 cpu_cores /= cpu_logical; 318 } 319 320 /* 321 * Both topology discovery code and code that consumes topology 322 * information assume top-down uniformity of the topology. 323 * That is, all physical packages must be identical and each 324 * core in a package must have the same number of threads. 325 * Topology information is queried only on BSP, on which this 326 * code runs and for which it can query CPUID information. 327 * Then topology is extrapolated on all packages using the 328 * uniformity assumption. 329 */ 330 static void 331 topo_probe(void) 332 { 333 static int cpu_topo_probed = 0; 334 335 if (cpu_topo_probed) 336 return; 337 338 CPU_ZERO(&logical_cpus_mask); 339 if (mp_ncpus <= 1) 340 cpu_cores = cpu_logical = 1; 341 else if (cpu_vendor_id == CPU_VENDOR_AMD) 342 topo_probe_amd(); 343 else if (cpu_vendor_id == CPU_VENDOR_INTEL) { 344 /* 345 * See Intel(R) 64 Architecture Processor 346 * Topology Enumeration article for details. 347 * 348 * Note that 0x1 <= cpu_high < 4 case should be 349 * compatible with topo_probe_0x4() logic when 350 * CPUID.1:EBX[23:16] > 0 (cpu_cores will be 1) 351 * or it should trigger the fallback otherwise. 352 */ 353 if (cpu_high >= 0xb) 354 topo_probe_0xb(); 355 else if (cpu_high >= 0x1) 356 topo_probe_0x4(); 357 } 358 359 /* 360 * Fallback: assume each logical CPU is in separate 361 * physical package. That is, no multi-core, no SMT. 362 */ 363 if (cpu_cores == 0 || cpu_logical == 0) 364 cpu_cores = cpu_logical = 1; 365 cpu_topo_probed = 1; 366 } 367 368 struct cpu_group * 369 cpu_topo(void) 370 { 371 int cg_flags; 372 373 /* 374 * Determine whether any threading flags are 375 * necessry. 376 */ 377 topo_probe(); 378 if (cpu_logical > 1 && hyperthreading_cpus) 379 cg_flags = CG_FLAG_HTT; 380 else if (cpu_logical > 1) 381 cg_flags = CG_FLAG_SMT; 382 else 383 cg_flags = 0; 384 if (mp_ncpus % (cpu_cores * cpu_logical) != 0) { 385 printf("WARNING: Non-uniform processors.\n"); 386 printf("WARNING: Using suboptimal topology.\n"); 387 return (smp_topo_none()); 388 } 389 /* 390 * No multi-core or hyper-threaded. 391 */ 392 if (cpu_logical * cpu_cores == 1) 393 return (smp_topo_none()); 394 /* 395 * Only HTT no multi-core. 396 */ 397 if (cpu_logical > 1 && cpu_cores == 1) 398 return (smp_topo_1level(CG_SHARE_L1, cpu_logical, cg_flags)); 399 /* 400 * Only multi-core no HTT. 401 */ 402 if (cpu_cores > 1 && cpu_logical == 1) 403 return (smp_topo_1level(CG_SHARE_L2, cpu_cores, cg_flags)); 404 /* 405 * Both HTT and multi-core. 406 */ 407 return (smp_topo_2level(CG_SHARE_L2, cpu_cores, 408 CG_SHARE_L1, cpu_logical, cg_flags)); 409 } 410 411 /* 412 * Calculate usable address in base memory for AP trampoline code. 413 */ 414 u_int 415 mp_bootaddress(u_int basemem) 416 { 417 418 bootMP_size = mptramp_end - mptramp_start; 419 boot_address = trunc_page(basemem * 1024); /* round down to 4k boundary */ 420 if (((basemem * 1024) - boot_address) < bootMP_size) 421 boot_address -= PAGE_SIZE; /* not enough, lower by 4k */ 422 /* 3 levels of page table pages */ 423 mptramp_pagetables = boot_address - (PAGE_SIZE * 3); 424 425 return mptramp_pagetables; 426 } 427 428 void 429 cpu_add(u_int apic_id, char boot_cpu) 430 { 431 432 if (apic_id > MAX_APIC_ID) { 433 panic("SMP: APIC ID %d too high", apic_id); 434 return; 435 } 436 KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice", 437 apic_id)); 438 cpu_info[apic_id].cpu_present = 1; 439 if (boot_cpu) { 440 KASSERT(boot_cpu_id == -1, 441 ("CPU %d claims to be BSP, but CPU %d already is", apic_id, 442 boot_cpu_id)); 443 boot_cpu_id = apic_id; 444 cpu_info[apic_id].cpu_bsp = 1; 445 } 446 if (mp_ncpus < MAXCPU) { 447 mp_ncpus++; 448 mp_maxid = mp_ncpus - 1; 449 } 450 if (bootverbose) 451 printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" : 452 "AP"); 453 } 454 455 void 456 cpu_mp_setmaxid(void) 457 { 458 459 /* 460 * mp_maxid should be already set by calls to cpu_add(). 461 * Just sanity check its value here. 462 */ 463 if (mp_ncpus == 0) 464 KASSERT(mp_maxid == 0, 465 ("%s: mp_ncpus is zero, but mp_maxid is not", __func__)); 466 else if (mp_ncpus == 1) 467 mp_maxid = 0; 468 else 469 KASSERT(mp_maxid >= mp_ncpus - 1, 470 ("%s: counters out of sync: max %d, count %d", __func__, 471 mp_maxid, mp_ncpus)); 472 } 473 474 int 475 cpu_mp_probe(void) 476 { 477 478 /* 479 * Always record BSP in CPU map so that the mbuf init code works 480 * correctly. 481 */ 482 CPU_SETOF(0, &all_cpus); 483 if (mp_ncpus == 0) { 484 /* 485 * No CPUs were found, so this must be a UP system. Setup 486 * the variables to represent a system with a single CPU 487 * with an id of 0. 488 */ 489 mp_ncpus = 1; 490 return (0); 491 } 492 493 /* At least one CPU was found. */ 494 if (mp_ncpus == 1) { 495 /* 496 * One CPU was found, so this must be a UP system with 497 * an I/O APIC. 498 */ 499 mp_maxid = 0; 500 return (0); 501 } 502 503 /* At least two CPUs were found. */ 504 return (1); 505 } 506 507 /* 508 * Initialize the IPI handlers and start up the AP's. 509 */ 510 void 511 cpu_mp_start(void) 512 { 513 int i; 514 515 /* Initialize the logical ID to APIC ID table. */ 516 for (i = 0; i < MAXCPU; i++) { 517 cpu_apic_ids[i] = -1; 518 cpu_ipi_pending[i] = 0; 519 } 520 521 /* Install an inter-CPU IPI for TLB invalidation */ 522 setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYSIGT, SEL_KPL, 0); 523 setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYSIGT, SEL_KPL, 0); 524 setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0); 525 526 /* Install an inter-CPU IPI for cache invalidation. */ 527 setidt(IPI_INVLCACHE, IDTVEC(invlcache), SDT_SYSIGT, SEL_KPL, 0); 528 529 /* Install an inter-CPU IPI for all-CPU rendezvous */ 530 setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0); 531 532 /* Install generic inter-CPU IPI handler */ 533 setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler), 534 SDT_SYSIGT, SEL_KPL, 0); 535 536 /* Install an inter-CPU IPI for CPU stop/restart */ 537 setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0); 538 539 /* Install an inter-CPU IPI for CPU suspend/resume */ 540 setidt(IPI_SUSPEND, IDTVEC(cpususpend), SDT_SYSIGT, SEL_KPL, 0); 541 542 /* Set boot_cpu_id if needed. */ 543 if (boot_cpu_id == -1) { 544 boot_cpu_id = PCPU_GET(apic_id); 545 cpu_info[boot_cpu_id].cpu_bsp = 1; 546 } else 547 KASSERT(boot_cpu_id == PCPU_GET(apic_id), 548 ("BSP's APIC ID doesn't match boot_cpu_id")); 549 550 /* Probe logical/physical core configuration. */ 551 topo_probe(); 552 553 assign_cpu_ids(); 554 555 /* Start each Application Processor */ 556 start_all_aps(); 557 558 set_interrupt_apic_ids(); 559 } 560 561 562 /* 563 * Print various information about the SMP system hardware and setup. 564 */ 565 void 566 cpu_mp_announce(void) 567 { 568 const char *hyperthread; 569 int i; 570 571 printf("FreeBSD/SMP: %d package(s) x %d core(s)", 572 mp_ncpus / (cpu_cores * cpu_logical), cpu_cores); 573 if (hyperthreading_cpus > 1) 574 printf(" x %d HTT threads", cpu_logical); 575 else if (cpu_logical > 1) 576 printf(" x %d SMT threads", cpu_logical); 577 printf("\n"); 578 579 /* List active CPUs first. */ 580 printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id); 581 for (i = 1; i < mp_ncpus; i++) { 582 if (cpu_info[cpu_apic_ids[i]].cpu_hyperthread) 583 hyperthread = "/HT"; 584 else 585 hyperthread = ""; 586 printf(" cpu%d (AP%s): APIC ID: %2d\n", i, hyperthread, 587 cpu_apic_ids[i]); 588 } 589 590 /* List disabled CPUs last. */ 591 for (i = 0; i <= MAX_APIC_ID; i++) { 592 if (!cpu_info[i].cpu_present || !cpu_info[i].cpu_disabled) 593 continue; 594 if (cpu_info[i].cpu_hyperthread) 595 hyperthread = "/HT"; 596 else 597 hyperthread = ""; 598 printf(" cpu (AP%s): APIC ID: %2d (disabled)\n", hyperthread, 599 i); 600 } 601 } 602 603 /* 604 * AP CPU's call this to initialize themselves. 605 */ 606 void 607 init_secondary(void) 608 { 609 struct pcpu *pc; 610 struct nmi_pcpu *np; 611 u_int64_t msr, cr0; 612 u_int cpuid; 613 int cpu, gsel_tss, x; 614 struct region_descriptor ap_gdt; 615 616 /* Set by the startup code for us to use */ 617 cpu = bootAP; 618 619 /* Init tss */ 620 common_tss[cpu] = common_tss[0]; 621 common_tss[cpu].tss_rsp0 = 0; /* not used until after switch */ 622 common_tss[cpu].tss_iobase = sizeof(struct amd64tss) + 623 IOPAGES * PAGE_SIZE; 624 common_tss[cpu].tss_ist1 = (long)&doublefault_stack[PAGE_SIZE]; 625 626 /* The NMI stack runs on IST2. */ 627 np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; 628 common_tss[cpu].tss_ist2 = (long) np; 629 630 /* Prepare private GDT */ 631 gdt_segs[GPROC0_SEL].ssd_base = (long) &common_tss[cpu]; 632 for (x = 0; x < NGDT; x++) { 633 if (x != GPROC0_SEL && x != (GPROC0_SEL + 1) && 634 x != GUSERLDT_SEL && x != (GUSERLDT_SEL + 1)) 635 ssdtosd(&gdt_segs[x], &gdt[NGDT * cpu + x]); 636 } 637 ssdtosyssd(&gdt_segs[GPROC0_SEL], 638 (struct system_segment_descriptor *)&gdt[NGDT * cpu + GPROC0_SEL]); 639 ap_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; 640 ap_gdt.rd_base = (long) &gdt[NGDT * cpu]; 641 lgdt(&ap_gdt); /* does magic intra-segment return */ 642 643 /* Get per-cpu data */ 644 pc = &__pcpu[cpu]; 645 646 /* prime data page for it to use */ 647 pcpu_init(pc, cpu, sizeof(struct pcpu)); 648 dpcpu_init(dpcpu, cpu); 649 pc->pc_apic_id = cpu_apic_ids[cpu]; 650 pc->pc_prvspace = pc; 651 pc->pc_curthread = 0; 652 pc->pc_tssp = &common_tss[cpu]; 653 pc->pc_commontssp = &common_tss[cpu]; 654 pc->pc_rsp0 = 0; 655 pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu + 656 GPROC0_SEL]; 657 pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL]; 658 pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL]; 659 pc->pc_ldt = (struct system_segment_descriptor *)&gdt[NGDT * cpu + 660 GUSERLDT_SEL]; 661 662 /* Save the per-cpu pointer for use by the NMI handler. */ 663 np->np_pcpu = (register_t) pc; 664 665 wrmsr(MSR_FSBASE, 0); /* User value */ 666 wrmsr(MSR_GSBASE, (u_int64_t)pc); 667 wrmsr(MSR_KGSBASE, (u_int64_t)pc); /* XXX User value while we're in the kernel */ 668 669 lidt(&r_idt); 670 671 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); 672 ltr(gsel_tss); 673 674 /* 675 * Set to a known state: 676 * Set by mpboot.s: CR0_PG, CR0_PE 677 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM 678 */ 679 cr0 = rcr0(); 680 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); 681 load_cr0(cr0); 682 683 /* Set up the fast syscall stuff */ 684 msr = rdmsr(MSR_EFER) | EFER_SCE; 685 wrmsr(MSR_EFER, msr); 686 wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall)); 687 wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); 688 msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 689 ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48); 690 wrmsr(MSR_STAR, msr); 691 wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D); 692 693 /* Disable local APIC just to be sure. */ 694 lapic_disable(); 695 696 /* signal our startup to the BSP. */ 697 mp_naps++; 698 699 /* Spin until the BSP releases the AP's. */ 700 while (!aps_ready) 701 ia32_pause(); 702 703 /* Initialize the PAT MSR. */ 704 pmap_init_pat(); 705 706 /* set up CPU registers and state */ 707 cpu_setregs(); 708 709 /* set up SSE/NX registers */ 710 initializecpu(); 711 712 /* set up FPU state on the AP */ 713 fpuinit(); 714 715 /* A quick check from sanity claus */ 716 cpuid = PCPU_GET(cpuid); 717 if (PCPU_GET(apic_id) != lapic_id()) { 718 printf("SMP: cpuid = %d\n", cpuid); 719 printf("SMP: actual apic_id = %d\n", lapic_id()); 720 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); 721 panic("cpuid mismatch! boom!!"); 722 } 723 724 /* Initialize curthread. */ 725 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 726 PCPU_SET(curthread, PCPU_GET(idlethread)); 727 728 mca_init(); 729 730 mtx_lock_spin(&ap_boot_mtx); 731 732 /* Init local apic for irq's */ 733 lapic_setup(1); 734 735 /* Set memory range attributes for this CPU to match the BSP */ 736 mem_range_AP_init(); 737 738 smp_cpus++; 739 740 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); 741 printf("SMP: AP CPU #%d Launched!\n", cpuid); 742 743 /* Determine if we are a logical CPU. */ 744 /* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */ 745 if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0) 746 CPU_SET(cpuid, &logical_cpus_mask); 747 748 if (bootverbose) 749 lapic_dump("AP"); 750 751 if (smp_cpus == mp_ncpus) { 752 /* enable IPI's, tlb shootdown, freezes etc */ 753 atomic_store_rel_int(&smp_started, 1); 754 smp_active = 1; /* historic */ 755 } 756 757 /* 758 * Enable global pages TLB extension 759 * This also implicitly flushes the TLB 760 */ 761 762 load_cr4(rcr4() | CR4_PGE); 763 load_ds(_udatasel); 764 load_es(_udatasel); 765 load_fs(_ufssel); 766 mtx_unlock_spin(&ap_boot_mtx); 767 768 /* Wait until all the AP's are up. */ 769 while (smp_started == 0) 770 ia32_pause(); 771 772 /* Start per-CPU event timers. */ 773 cpu_initclocks_ap(); 774 775 sched_throw(NULL); 776 777 panic("scheduler returned us to %s", __func__); 778 /* NOTREACHED */ 779 } 780 781 /******************************************************************* 782 * local functions and data 783 */ 784 785 /* 786 * We tell the I/O APIC code about all the CPUs we want to receive 787 * interrupts. If we don't want certain CPUs to receive IRQs we 788 * can simply not tell the I/O APIC code about them in this function. 789 */ 790 static void 791 set_interrupt_apic_ids(void) 792 { 793 u_int i, apic_id; 794 795 for (i = 0; i < MAXCPU; i++) { 796 apic_id = cpu_apic_ids[i]; 797 if (apic_id == -1) 798 continue; 799 if (cpu_info[apic_id].cpu_disabled) 800 continue; 801 802 /* Don't let hyperthreads service interrupts. */ 803 if (hyperthreading_cpus > 1 && 804 apic_id % hyperthreading_cpus != 0) 805 continue; 806 807 intr_add_cpu(i); 808 } 809 } 810 811 /* 812 * Assign logical CPU IDs to local APICs. 813 */ 814 static void 815 assign_cpu_ids(void) 816 { 817 u_int i; 818 819 TUNABLE_INT_FETCH("machdep.hyperthreading_allowed", 820 &hyperthreading_allowed); 821 822 /* Check for explicitly disabled CPUs. */ 823 for (i = 0; i <= MAX_APIC_ID; i++) { 824 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp) 825 continue; 826 827 if (hyperthreading_cpus > 1 && i % hyperthreading_cpus != 0) { 828 cpu_info[i].cpu_hyperthread = 1; 829 830 /* 831 * Don't use HT CPU if it has been disabled by a 832 * tunable. 833 */ 834 if (hyperthreading_allowed == 0) { 835 cpu_info[i].cpu_disabled = 1; 836 continue; 837 } 838 } 839 840 /* Don't use this CPU if it has been disabled by a tunable. */ 841 if (resource_disabled("lapic", i)) { 842 cpu_info[i].cpu_disabled = 1; 843 continue; 844 } 845 } 846 847 if (hyperthreading_allowed == 0 && hyperthreading_cpus > 1) { 848 hyperthreading_cpus = 0; 849 cpu_logical = 1; 850 } 851 852 /* 853 * Assign CPU IDs to local APIC IDs and disable any CPUs 854 * beyond MAXCPU. CPU 0 is always assigned to the BSP. 855 * 856 * To minimize confusion for userland, we attempt to number 857 * CPUs such that all threads and cores in a package are 858 * grouped together. For now we assume that the BSP is always 859 * the first thread in a package and just start adding APs 860 * starting with the BSP's APIC ID. 861 */ 862 mp_ncpus = 1; 863 cpu_apic_ids[0] = boot_cpu_id; 864 apic_cpuids[boot_cpu_id] = 0; 865 for (i = boot_cpu_id + 1; i != boot_cpu_id; 866 i == MAX_APIC_ID ? i = 0 : i++) { 867 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp || 868 cpu_info[i].cpu_disabled) 869 continue; 870 871 if (mp_ncpus < MAXCPU) { 872 cpu_apic_ids[mp_ncpus] = i; 873 apic_cpuids[i] = mp_ncpus; 874 mp_ncpus++; 875 } else 876 cpu_info[i].cpu_disabled = 1; 877 } 878 KASSERT(mp_maxid >= mp_ncpus - 1, 879 ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid, 880 mp_ncpus)); 881 } 882 883 /* 884 * start each AP in our list 885 */ 886 static int 887 start_all_aps(void) 888 { 889 vm_offset_t va = boot_address + KERNBASE; 890 u_int64_t *pt4, *pt3, *pt2; 891 u_int32_t mpbioswarmvec; 892 int apic_id, cpu, i; 893 u_char mpbiosreason; 894 895 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 896 897 /* install the AP 1st level boot code */ 898 pmap_kenter(va, boot_address); 899 pmap_invalidate_page(kernel_pmap, va); 900 bcopy(mptramp_start, (void *)va, bootMP_size); 901 902 /* Locate the page tables, they'll be below the trampoline */ 903 pt4 = (u_int64_t *)(uintptr_t)(mptramp_pagetables + KERNBASE); 904 pt3 = pt4 + (PAGE_SIZE) / sizeof(u_int64_t); 905 pt2 = pt3 + (PAGE_SIZE) / sizeof(u_int64_t); 906 907 /* Create the initial 1GB replicated page tables */ 908 for (i = 0; i < 512; i++) { 909 /* Each slot of the level 4 pages points to the same level 3 page */ 910 pt4[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + PAGE_SIZE); 911 pt4[i] |= PG_V | PG_RW | PG_U; 912 913 /* Each slot of the level 3 pages points to the same level 2 page */ 914 pt3[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + (2 * PAGE_SIZE)); 915 pt3[i] |= PG_V | PG_RW | PG_U; 916 917 /* The level 2 page slots are mapped with 2MB pages for 1GB. */ 918 pt2[i] = i * (2 * 1024 * 1024); 919 pt2[i] |= PG_V | PG_RW | PG_PS | PG_U; 920 } 921 922 /* save the current value of the warm-start vector */ 923 mpbioswarmvec = *((u_int32_t *) WARMBOOT_OFF); 924 outb(CMOS_REG, BIOS_RESET); 925 mpbiosreason = inb(CMOS_DATA); 926 927 /* setup a vector to our boot code */ 928 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 929 *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4); 930 outb(CMOS_REG, BIOS_RESET); 931 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 932 933 /* start each AP */ 934 for (cpu = 1; cpu < mp_ncpus; cpu++) { 935 apic_id = cpu_apic_ids[cpu]; 936 937 /* allocate and set up an idle stack data page */ 938 bootstacks[cpu] = (void *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); 939 doublefault_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE); 940 nmi_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE); 941 dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE); 942 943 bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8; 944 bootAP = cpu; 945 946 /* attempt to start the Application Processor */ 947 if (!start_ap(apic_id)) { 948 /* restore the warmstart vector */ 949 *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; 950 panic("AP #%d (PHY# %d) failed!", cpu, apic_id); 951 } 952 953 CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ 954 } 955 956 /* restore the warmstart vector */ 957 *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; 958 959 outb(CMOS_REG, BIOS_RESET); 960 outb(CMOS_DATA, mpbiosreason); 961 962 /* number of APs actually started */ 963 return mp_naps; 964 } 965 966 967 /* 968 * This function starts the AP (application processor) identified 969 * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 970 * to accomplish this. This is necessary because of the nuances 971 * of the different hardware we might encounter. It isn't pretty, 972 * but it seems to work. 973 */ 974 static int 975 start_ap(int apic_id) 976 { 977 int vector, ms; 978 int cpus; 979 980 /* calculate the vector */ 981 vector = (boot_address >> 12) & 0xff; 982 983 /* used as a watchpoint to signal AP startup */ 984 cpus = mp_naps; 985 986 /* 987 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting 988 * and running the target CPU. OR this INIT IPI might be latched (P5 989 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be 990 * ignored. 991 */ 992 993 /* do an INIT IPI: assert RESET */ 994 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 995 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); 996 997 /* wait for pending status end */ 998 lapic_ipi_wait(-1); 999 1000 /* do an INIT IPI: deassert RESET */ 1001 lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL | 1002 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0); 1003 1004 /* wait for pending status end */ 1005 DELAY(10000); /* wait ~10mS */ 1006 lapic_ipi_wait(-1); 1007 1008 /* 1009 * next we do a STARTUP IPI: the previous INIT IPI might still be 1010 * latched, (P5 bug) this 1st STARTUP would then terminate 1011 * immediately, and the previously started INIT IPI would continue. OR 1012 * the previous INIT IPI has already run. and this STARTUP IPI will 1013 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI 1014 * will run. 1015 */ 1016 1017 /* do a STARTUP IPI */ 1018 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 1019 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | 1020 vector, apic_id); 1021 lapic_ipi_wait(-1); 1022 DELAY(200); /* wait ~200uS */ 1023 1024 /* 1025 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF 1026 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR 1027 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is 1028 * recognized after hardware RESET or INIT IPI. 1029 */ 1030 1031 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 1032 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | 1033 vector, apic_id); 1034 lapic_ipi_wait(-1); 1035 DELAY(200); /* wait ~200uS */ 1036 1037 /* Wait up to 5 seconds for it to start. */ 1038 for (ms = 0; ms < 5000; ms++) { 1039 if (mp_naps > cpus) 1040 return 1; /* return SUCCESS */ 1041 DELAY(1000); 1042 } 1043 return 0; /* return FAILURE */ 1044 } 1045 1046 #ifdef COUNT_XINVLTLB_HITS 1047 u_int xhits_gbl[MAXCPU]; 1048 u_int xhits_pg[MAXCPU]; 1049 u_int xhits_rng[MAXCPU]; 1050 static SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); 1051 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, 1052 sizeof(xhits_gbl), "IU", ""); 1053 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, 1054 sizeof(xhits_pg), "IU", ""); 1055 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, 1056 sizeof(xhits_rng), "IU", ""); 1057 1058 u_int ipi_global; 1059 u_int ipi_page; 1060 u_int ipi_range; 1061 u_int ipi_range_size; 1062 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); 1063 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); 1064 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); 1065 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, 1066 &ipi_range_size, 0, ""); 1067 1068 u_int ipi_masked_global; 1069 u_int ipi_masked_page; 1070 u_int ipi_masked_range; 1071 u_int ipi_masked_range_size; 1072 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, 1073 &ipi_masked_global, 0, ""); 1074 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, 1075 &ipi_masked_page, 0, ""); 1076 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, 1077 &ipi_masked_range, 0, ""); 1078 SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, 1079 &ipi_masked_range_size, 0, ""); 1080 #endif /* COUNT_XINVLTLB_HITS */ 1081 1082 /* 1083 * Send an IPI to specified CPU handling the bitmap logic. 1084 */ 1085 static void 1086 ipi_send_cpu(int cpu, u_int ipi) 1087 { 1088 u_int bitmap, old_pending, new_pending; 1089 1090 KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu)); 1091 1092 if (IPI_IS_BITMAPED(ipi)) { 1093 bitmap = 1 << ipi; 1094 ipi = IPI_BITMAP_VECTOR; 1095 do { 1096 old_pending = cpu_ipi_pending[cpu]; 1097 new_pending = old_pending | bitmap; 1098 } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu], 1099 old_pending, new_pending)); 1100 if (old_pending) 1101 return; 1102 } 1103 lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); 1104 } 1105 1106 /* 1107 * Flush the TLB on all other CPU's 1108 */ 1109 static void 1110 smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) 1111 { 1112 u_int ncpu; 1113 1114 ncpu = mp_ncpus - 1; /* does not shootdown self */ 1115 if (ncpu < 1) 1116 return; /* no other cpus */ 1117 if (!(read_rflags() & PSL_I)) 1118 panic("%s: interrupts disabled", __func__); 1119 mtx_lock_spin(&smp_ipi_mtx); 1120 smp_tlb_addr1 = addr1; 1121 smp_tlb_addr2 = addr2; 1122 atomic_store_rel_int(&smp_tlb_wait, 0); 1123 ipi_all_but_self(vector); 1124 while (smp_tlb_wait < ncpu) 1125 ia32_pause(); 1126 mtx_unlock_spin(&smp_ipi_mtx); 1127 } 1128 1129 static void 1130 smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) 1131 { 1132 int cpu, ncpu, othercpus; 1133 1134 othercpus = mp_ncpus - 1; 1135 if (CPU_ISFULLSET(&mask)) { 1136 if (othercpus < 1) 1137 return; 1138 } else { 1139 CPU_CLR(PCPU_GET(cpuid), &mask); 1140 if (CPU_EMPTY(&mask)) 1141 return; 1142 } 1143 if (!(read_rflags() & PSL_I)) 1144 panic("%s: interrupts disabled", __func__); 1145 mtx_lock_spin(&smp_ipi_mtx); 1146 smp_tlb_addr1 = addr1; 1147 smp_tlb_addr2 = addr2; 1148 atomic_store_rel_int(&smp_tlb_wait, 0); 1149 if (CPU_ISFULLSET(&mask)) { 1150 ncpu = othercpus; 1151 ipi_all_but_self(vector); 1152 } else { 1153 ncpu = 0; 1154 while ((cpu = CPU_FFS(&mask)) != 0) { 1155 cpu--; 1156 CPU_CLR(cpu, &mask); 1157 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, 1158 cpu, vector); 1159 ipi_send_cpu(cpu, vector); 1160 ncpu++; 1161 } 1162 } 1163 while (smp_tlb_wait < ncpu) 1164 ia32_pause(); 1165 mtx_unlock_spin(&smp_ipi_mtx); 1166 } 1167 1168 void 1169 smp_cache_flush(void) 1170 { 1171 1172 if (smp_started) 1173 smp_tlb_shootdown(IPI_INVLCACHE, 0, 0); 1174 } 1175 1176 void 1177 smp_invltlb(void) 1178 { 1179 1180 if (smp_started) { 1181 smp_tlb_shootdown(IPI_INVLTLB, 0, 0); 1182 #ifdef COUNT_XINVLTLB_HITS 1183 ipi_global++; 1184 #endif 1185 } 1186 } 1187 1188 void 1189 smp_invlpg(vm_offset_t addr) 1190 { 1191 1192 if (smp_started) { 1193 smp_tlb_shootdown(IPI_INVLPG, addr, 0); 1194 #ifdef COUNT_XINVLTLB_HITS 1195 ipi_page++; 1196 #endif 1197 } 1198 } 1199 1200 void 1201 smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) 1202 { 1203 1204 if (smp_started) { 1205 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); 1206 #ifdef COUNT_XINVLTLB_HITS 1207 ipi_range++; 1208 ipi_range_size += (addr2 - addr1) / PAGE_SIZE; 1209 #endif 1210 } 1211 } 1212 1213 void 1214 smp_masked_invltlb(cpuset_t mask) 1215 { 1216 1217 if (smp_started) { 1218 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); 1219 #ifdef COUNT_XINVLTLB_HITS 1220 ipi_masked_global++; 1221 #endif 1222 } 1223 } 1224 1225 void 1226 smp_masked_invlpg(cpuset_t mask, vm_offset_t addr) 1227 { 1228 1229 if (smp_started) { 1230 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); 1231 #ifdef COUNT_XINVLTLB_HITS 1232 ipi_masked_page++; 1233 #endif 1234 } 1235 } 1236 1237 void 1238 smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2) 1239 { 1240 1241 if (smp_started) { 1242 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); 1243 #ifdef COUNT_XINVLTLB_HITS 1244 ipi_masked_range++; 1245 ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; 1246 #endif 1247 } 1248 } 1249 1250 void 1251 ipi_bitmap_handler(struct trapframe frame) 1252 { 1253 struct trapframe *oldframe; 1254 struct thread *td; 1255 int cpu = PCPU_GET(cpuid); 1256 u_int ipi_bitmap; 1257 1258 critical_enter(); 1259 td = curthread; 1260 td->td_intr_nesting_level++; 1261 oldframe = td->td_intr_frame; 1262 td->td_intr_frame = &frame; 1263 ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]); 1264 if (ipi_bitmap & (1 << IPI_PREEMPT)) { 1265 #ifdef COUNT_IPIS 1266 (*ipi_preempt_counts[cpu])++; 1267 #endif 1268 sched_preempt(td); 1269 } 1270 if (ipi_bitmap & (1 << IPI_AST)) { 1271 #ifdef COUNT_IPIS 1272 (*ipi_ast_counts[cpu])++; 1273 #endif 1274 /* Nothing to do for AST */ 1275 } 1276 if (ipi_bitmap & (1 << IPI_HARDCLOCK)) { 1277 #ifdef COUNT_IPIS 1278 (*ipi_hardclock_counts[cpu])++; 1279 #endif 1280 hardclockintr(); 1281 } 1282 td->td_intr_frame = oldframe; 1283 td->td_intr_nesting_level--; 1284 critical_exit(); 1285 } 1286 1287 /* 1288 * send an IPI to a set of cpus. 1289 */ 1290 void 1291 ipi_selected(cpuset_t cpus, u_int ipi) 1292 { 1293 int cpu; 1294 1295 /* 1296 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit 1297 * of help in order to understand what is the source. 1298 * Set the mask of receiving CPUs for this purpose. 1299 */ 1300 if (ipi == IPI_STOP_HARD) 1301 CPU_OR_ATOMIC(&ipi_nmi_pending, &cpus); 1302 1303 while ((cpu = CPU_FFS(&cpus)) != 0) { 1304 cpu--; 1305 CPU_CLR(cpu, &cpus); 1306 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); 1307 ipi_send_cpu(cpu, ipi); 1308 } 1309 } 1310 1311 /* 1312 * send an IPI to a specific CPU. 1313 */ 1314 void 1315 ipi_cpu(int cpu, u_int ipi) 1316 { 1317 1318 /* 1319 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit 1320 * of help in order to understand what is the source. 1321 * Set the mask of receiving CPUs for this purpose. 1322 */ 1323 if (ipi == IPI_STOP_HARD) 1324 CPU_SET_ATOMIC(cpu, &ipi_nmi_pending); 1325 1326 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); 1327 ipi_send_cpu(cpu, ipi); 1328 } 1329 1330 /* 1331 * send an IPI to all CPUs EXCEPT myself 1332 */ 1333 void 1334 ipi_all_but_self(u_int ipi) 1335 { 1336 cpuset_t other_cpus; 1337 1338 other_cpus = all_cpus; 1339 CPU_CLR(PCPU_GET(cpuid), &other_cpus); 1340 1341 if (IPI_IS_BITMAPED(ipi)) { 1342 ipi_selected(other_cpus, ipi); 1343 return; 1344 } 1345 1346 /* 1347 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit 1348 * of help in order to understand what is the source. 1349 * Set the mask of receiving CPUs for this purpose. 1350 */ 1351 if (ipi == IPI_STOP_HARD) 1352 CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus); 1353 1354 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1355 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); 1356 } 1357 1358 int 1359 ipi_nmi_handler() 1360 { 1361 u_int cpuid; 1362 1363 /* 1364 * As long as there is not a simple way to know about a NMI's 1365 * source, if the bitmask for the current CPU is present in 1366 * the global pending bitword an IPI_STOP_HARD has been issued 1367 * and should be handled. 1368 */ 1369 cpuid = PCPU_GET(cpuid); 1370 if (!CPU_ISSET(cpuid, &ipi_nmi_pending)) 1371 return (1); 1372 1373 CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending); 1374 cpustop_handler(); 1375 return (0); 1376 } 1377 1378 /* 1379 * Handle an IPI_STOP by saving our current context and spinning until we 1380 * are resumed. 1381 */ 1382 void 1383 cpustop_handler(void) 1384 { 1385 u_int cpu; 1386 1387 cpu = PCPU_GET(cpuid); 1388 1389 savectx(&stoppcbs[cpu]); 1390 1391 /* Indicate that we are stopped */ 1392 CPU_SET_ATOMIC(cpu, &stopped_cpus); 1393 1394 /* Wait for restart */ 1395 while (!CPU_ISSET(cpu, &started_cpus)) 1396 ia32_pause(); 1397 1398 CPU_CLR_ATOMIC(cpu, &started_cpus); 1399 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 1400 1401 #ifdef DDB 1402 amd64_db_resume_dbreg(); 1403 #endif 1404 1405 if (cpu == 0 && cpustop_restartfunc != NULL) { 1406 cpustop_restartfunc(); 1407 cpustop_restartfunc = NULL; 1408 } 1409 } 1410 1411 /* 1412 * Handle an IPI_SUSPEND by saving our current context and spinning until we 1413 * are resumed. 1414 */ 1415 void 1416 cpususpend_handler(void) 1417 { 1418 u_int cpu; 1419 1420 cpu = PCPU_GET(cpuid); 1421 1422 if (savectx(susppcbs[cpu])) { 1423 fpususpend(suspfpusave[cpu]); 1424 wbinvd(); 1425 CPU_SET_ATOMIC(cpu, &stopped_cpus); 1426 } else { 1427 pmap_init_pat(); 1428 load_cr3(susppcbs[cpu]->pcb_cr3); 1429 initializecpu(); 1430 PCPU_SET(switchtime, 0); 1431 PCPU_SET(switchticks, ticks); 1432 } 1433 1434 /* Wait for resume */ 1435 while (!CPU_ISSET(cpu, &started_cpus)) 1436 ia32_pause(); 1437 1438 /* Resume MCA and local APIC */ 1439 mca_resume(); 1440 lapic_setup(0); 1441 1442 CPU_CLR_ATOMIC(cpu, &started_cpus); 1443 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 1444 } 1445 1446 /* 1447 * This is called once the rest of the system is up and running and we're 1448 * ready to let the AP's out of the pen. 1449 */ 1450 static void 1451 release_aps(void *dummy __unused) 1452 { 1453 1454 if (mp_ncpus == 1) 1455 return; 1456 atomic_store_rel_int(&aps_ready, 1); 1457 while (smp_started == 0) 1458 ia32_pause(); 1459 } 1460 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 1461 1462 #ifdef COUNT_IPIS 1463 /* 1464 * Setup interrupt counters for IPI handlers. 1465 */ 1466 static void 1467 mp_ipi_intrcnt(void *dummy) 1468 { 1469 char buf[64]; 1470 int i; 1471 1472 CPU_FOREACH(i) { 1473 snprintf(buf, sizeof(buf), "cpu%d:invltlb", i); 1474 intrcnt_add(buf, &ipi_invltlb_counts[i]); 1475 snprintf(buf, sizeof(buf), "cpu%d:invlrng", i); 1476 intrcnt_add(buf, &ipi_invlrng_counts[i]); 1477 snprintf(buf, sizeof(buf), "cpu%d:invlpg", i); 1478 intrcnt_add(buf, &ipi_invlpg_counts[i]); 1479 snprintf(buf, sizeof(buf), "cpu%d:invlcache", i); 1480 intrcnt_add(buf, &ipi_invlcache_counts[i]); 1481 snprintf(buf, sizeof(buf), "cpu%d:preempt", i); 1482 intrcnt_add(buf, &ipi_preempt_counts[i]); 1483 snprintf(buf, sizeof(buf), "cpu%d:ast", i); 1484 intrcnt_add(buf, &ipi_ast_counts[i]); 1485 snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); 1486 intrcnt_add(buf, &ipi_rendezvous_counts[i]); 1487 snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); 1488 intrcnt_add(buf, &ipi_hardclock_counts[i]); 1489 } 1490 } 1491 SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); 1492 #endif 1493
Cache object: d22ac49b92905356b3d5c8bc61bb1cbd
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]