1 /*-
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 * derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28
29 #include "opt_apic.h"
30 #include "opt_cpu.h"
31 #include "opt_kstack_pages.h"
32 #include "opt_mp_watchdog.h"
33
34 #if !defined(lint)
35 #if !defined(SMP)
36 #error How did you get here?
37 #endif
38
39 #if defined(I386_CPU) && !defined(COMPILING_LINT)
40 #error SMP not supported with I386_CPU
41 #endif
42 #ifndef DEV_APIC
43 #error The apic device is required for SMP, add "device apic" to your config file.
44 #endif
45 #if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT)
46 #error SMP not supported with CPU_DISABLE_CMPXCHG
47 #endif
48 #endif /* not lint */
49
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/bus.h>
53 #include <sys/cons.h> /* cngetc() */
54 #ifdef GPROF
55 #include <sys/gmon.h>
56 #endif
57 #include <sys/kernel.h>
58 #include <sys/ktr.h>
59 #include <sys/lock.h>
60 #include <sys/malloc.h>
61 #include <sys/memrange.h>
62 #include <sys/mutex.h>
63 #include <sys/pcpu.h>
64 #include <sys/proc.h>
65 #include <sys/smp.h>
66 #include <sys/sysctl.h>
67
68 #include <vm/vm.h>
69 #include <vm/vm_param.h>
70 #include <vm/pmap.h>
71 #include <vm/vm_kern.h>
72 #include <vm/vm_extern.h>
73
74 #include <machine/apicreg.h>
75 #include <machine/clock.h>
76 #include <machine/md_var.h>
77 #include <machine/mp_watchdog.h>
78 #include <machine/pcb.h>
79 #include <machine/smp.h>
80 #include <machine/smptests.h> /** COUNT_XINVLTLB_HITS */
81 #include <machine/specialreg.h>
82 #include <machine/privatespace.h>
83
84 #define WARMBOOT_TARGET 0
85 #define WARMBOOT_OFF (KERNBASE + 0x0467)
86 #define WARMBOOT_SEG (KERNBASE + 0x0469)
87
88 #define CMOS_REG (0x70)
89 #define CMOS_DATA (0x71)
90 #define BIOS_RESET (0x0f)
91 #define BIOS_WARM (0x0a)
92
93 /*
94 * this code MUST be enabled here and in mpboot.s.
95 * it follows the very early stages of AP boot by placing values in CMOS ram.
96 * it NORMALLY will never be needed and thus the primitive method for enabling.
97 *
98 #define CHECK_POINTS
99 */
100
101 #if defined(CHECK_POINTS) && !defined(PC98)
102 #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA))
103 #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
104
105 #define CHECK_INIT(D); \
106 CHECK_WRITE(0x34, (D)); \
107 CHECK_WRITE(0x35, (D)); \
108 CHECK_WRITE(0x36, (D)); \
109 CHECK_WRITE(0x37, (D)); \
110 CHECK_WRITE(0x38, (D)); \
111 CHECK_WRITE(0x39, (D));
112
113 #define CHECK_PRINT(S); \
114 printf("%s: %d, %d, %d, %d, %d, %d\n", \
115 (S), \
116 CHECK_READ(0x34), \
117 CHECK_READ(0x35), \
118 CHECK_READ(0x36), \
119 CHECK_READ(0x37), \
120 CHECK_READ(0x38), \
121 CHECK_READ(0x39));
122
123 #else /* CHECK_POINTS */
124
125 #define CHECK_INIT(D)
126 #define CHECK_PRINT(S)
127 #define CHECK_WRITE(A, D)
128
129 #endif /* CHECK_POINTS */
130
131 /*
132 * Values to send to the POST hardware.
133 */
134 #define MP_BOOTADDRESS_POST 0x10
135 #define MP_PROBE_POST 0x11
136 #define MPTABLE_PASS1_POST 0x12
137
138 #define MP_START_POST 0x13
139 #define MP_ENABLE_POST 0x14
140 #define MPTABLE_PASS2_POST 0x15
141
142 #define START_ALL_APS_POST 0x16
143 #define INSTALL_AP_TRAMP_POST 0x17
144 #define START_AP_POST 0x18
145
146 #define MP_ANNOUNCE_POST 0x19
147
148 /* lock region used by kernel profiling */
149 int mcount_lock;
150
151 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
152 int current_postcode;
153
154 int mp_naps; /* # of Applications processors */
155 int boot_cpu_id = -1; /* designated BSP */
156 extern int nkpt;
157
158 /*
159 * CPU topology map datastructures for HTT.
160 */
161 static struct cpu_group mp_groups[MAXCPU];
162 static struct cpu_top mp_top;
163
164 /* AP uses this during bootstrap. Do not staticize. */
165 char *bootSTK;
166 static int bootAP;
167
168 /* Hotwire a 0->4MB V==P mapping */
169 extern pt_entry_t *KPTphys;
170
171 /* SMP page table page */
172 extern pt_entry_t *SMPpt;
173
174 struct pcb stoppcbs[MAXCPU];
175
176 /* Variables needed for SMP tlb shootdown. */
177 vm_offset_t smp_tlb_addr1;
178 vm_offset_t smp_tlb_addr2;
179 volatile int smp_tlb_wait;
180
181 #ifdef KDB_STOP_NMI
182 volatile cpumask_t ipi_nmi_pending;
183 #endif
184
185 /*
186 * Local data and functions.
187 */
188
189 static u_int logical_cpus;
190
191 /* used to hold the AP's until we are ready to release them */
192 static struct mtx ap_boot_mtx;
193
194 /* Set to 1 once we're ready to let the APs out of the pen. */
195 static volatile int aps_ready = 0;
196
197 /*
198 * Store data from cpu_add() until later in the boot when we actually setup
199 * the APs.
200 */
201 struct cpu_info {
202 int cpu_present:1;
203 int cpu_bsp:1;
204 int cpu_disabled:1;
205 } static cpu_info[MAXCPU];
206 static int cpu_apic_ids[MAXCPU];
207
208 /* Holds pending bitmap based IPIs per CPU */
209 static volatile u_int cpu_ipi_pending[MAXCPU];
210
211 static u_int boot_address;
212
213 static void set_logical_apic_ids(void);
214 static int start_all_aps(void);
215 static void install_ap_tramp(void);
216 static int start_ap(int apic_id);
217 static void release_aps(void *dummy);
218
219 static int hlt_logical_cpus;
220 static u_int hyperthreading_cpus;
221 static cpumask_t hyperthreading_cpus_mask;
222 static int hyperthreading_allowed;
223 static struct sysctl_ctx_list logical_cpu_clist;
224
225 static void
226 mem_range_AP_init(void)
227 {
228 if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP)
229 mem_range_softc.mr_op->initAP(&mem_range_softc);
230 }
231
232 void
233 mp_topology(void)
234 {
235 struct cpu_group *group;
236 int logical_cpus;
237 int apic_id;
238 int groups;
239 int cpu;
240
241 /* Build the smp_topology map. */
242 /* Nothing to do if there is no HTT support. */
243 if ((cpu_feature & CPUID_HTT) == 0)
244 return;
245 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16;
246 if (logical_cpus <= 1)
247 return;
248 group = &mp_groups[0];
249 groups = 1;
250 for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) {
251 if (!cpu_info[apic_id].cpu_present)
252 continue;
253 /*
254 * If the current group has members and we're not a logical
255 * cpu, create a new group.
256 */
257 if (group->cg_count != 0 && (apic_id % logical_cpus) == 0) {
258 group++;
259 groups++;
260 }
261 group->cg_count++;
262 group->cg_mask |= 1 << cpu;
263 cpu++;
264 }
265
266 mp_top.ct_count = groups;
267 mp_top.ct_group = mp_groups;
268 smp_topology = &mp_top;
269 }
270
271
272 /*
273 * Calculate usable address in base memory for AP trampoline code.
274 */
275 u_int
276 mp_bootaddress(u_int basemem)
277 {
278 POSTCODE(MP_BOOTADDRESS_POST);
279
280 boot_address = trunc_page(basemem); /* round down to 4k boundary */
281 if ((basemem - boot_address) < bootMP_size)
282 boot_address -= PAGE_SIZE; /* not enough, lower by 4k */
283
284 return boot_address;
285 }
286
287 void
288 cpu_add(u_int apic_id, char boot_cpu)
289 {
290
291 if (apic_id >= MAXCPU) {
292 printf("SMP: CPU %d exceeds maximum CPU %d, ignoring\n",
293 apic_id, MAXCPU - 1);
294 return;
295 }
296 KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice",
297 apic_id));
298 cpu_info[apic_id].cpu_present = 1;
299 if (boot_cpu) {
300 KASSERT(boot_cpu_id == -1,
301 ("CPU %d claims to be BSP, but CPU %d already is", apic_id,
302 boot_cpu_id));
303 boot_cpu_id = apic_id;
304 cpu_info[apic_id].cpu_bsp = 1;
305 }
306 mp_ncpus++;
307 if (bootverbose)
308 printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" :
309 "AP");
310
311 }
312
313 void
314 cpu_mp_setmaxid(void)
315 {
316
317 mp_maxid = MAXCPU - 1;
318 }
319
320 int
321 cpu_mp_probe(void)
322 {
323
324 /*
325 * Always record BSP in CPU map so that the mbuf init code works
326 * correctly.
327 */
328 all_cpus = 1;
329 if (mp_ncpus == 0) {
330 /*
331 * No CPUs were found, so this must be a UP system. Setup
332 * the variables to represent a system with a single CPU
333 * with an id of 0.
334 */
335 mp_ncpus = 1;
336 return (0);
337 }
338
339 /* At least one CPU was found. */
340 if (mp_ncpus == 1) {
341 /*
342 * One CPU was found, so this must be a UP system with
343 * an I/O APIC.
344 */
345 return (0);
346 }
347
348 /* At least two CPUs were found. */
349 return (1);
350 }
351
352 /*
353 * Initialize the IPI handlers and start up the AP's.
354 */
355 void
356 cpu_mp_start(void)
357 {
358 int i;
359 u_int threads_per_cache, p[4];
360
361 POSTCODE(MP_START_POST);
362
363 /* Initialize the logical ID to APIC ID table. */
364 for (i = 0; i < MAXCPU; i++) {
365 cpu_apic_ids[i] = -1;
366 cpu_ipi_pending[i] = 0;
367 }
368
369 /* Install an inter-CPU IPI for TLB invalidation */
370 setidt(IPI_INVLTLB, IDTVEC(invltlb),
371 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
372 setidt(IPI_INVLPG, IDTVEC(invlpg),
373 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
374 setidt(IPI_INVLRNG, IDTVEC(invlrng),
375 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
376
377 /* Install an inter-CPU IPI for lazy pmap release */
378 setidt(IPI_LAZYPMAP, IDTVEC(lazypmap),
379 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
380
381 /* Install an inter-CPU IPI for all-CPU rendezvous */
382 setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous),
383 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
384
385 /* Install generic inter-CPU IPI handler */
386 setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler),
387 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
388
389 /* Install an inter-CPU IPI for CPU stop/restart */
390 setidt(IPI_STOP, IDTVEC(cpustop),
391 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
392
393
394 /* Set boot_cpu_id if needed. */
395 if (boot_cpu_id == -1) {
396 boot_cpu_id = PCPU_GET(apic_id);
397 cpu_info[boot_cpu_id].cpu_bsp = 1;
398 } else
399 KASSERT(boot_cpu_id == PCPU_GET(apic_id),
400 ("BSP's APIC ID doesn't match boot_cpu_id"));
401 cpu_apic_ids[0] = boot_cpu_id;
402
403 /* Start each Application Processor */
404 start_all_aps();
405
406 /* Setup the initial logical CPUs info. */
407 logical_cpus = logical_cpus_mask = 0;
408 if (cpu_feature & CPUID_HTT)
409 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16;
410
411 /*
412 * Work out if hyperthreading is *really* enabled. This
413 * is made really ugly by the fact that processors lie: Dual
414 * core processors claim to be hyperthreaded even when they're
415 * not, presumably because they want to be treated the same
416 * way as HTT with respect to per-cpu software licensing.
417 * At the time of writing (May 12, 2005) the only hyperthreaded
418 * cpus are from Intel, and Intel's dual-core processors can be
419 * identified via the "deterministic cache parameters" cpuid
420 * calls.
421 */
422 /*
423 * First determine if this is an Intel processor which claims
424 * to have hyperthreading support.
425 */
426 if ((cpu_feature & CPUID_HTT) &&
427 (strcmp(cpu_vendor, "GenuineIntel") == 0)) {
428 /*
429 * If the "deterministic cache parameters" cpuid calls
430 * are available, use them.
431 */
432 if (cpu_high >= 4) {
433 /* Ask the processor about the L1 cache. */
434 for (i = 0; i < 1; i++) {
435 cpuid_count(4, i, p);
436 threads_per_cache = ((p[0] & 0x3ffc000) >> 14) + 1;
437 if (hyperthreading_cpus < threads_per_cache)
438 hyperthreading_cpus = threads_per_cache;
439 if ((p[0] & 0x1f) == 0)
440 break;
441 }
442 }
443
444 /*
445 * If the deterministic cache parameters are not
446 * available, or if no caches were reported to exist,
447 * just accept what the HTT flag indicated.
448 */
449 if (hyperthreading_cpus == 0)
450 hyperthreading_cpus = logical_cpus;
451 }
452
453 set_logical_apic_ids();
454 }
455
456
457 /*
458 * Print various information about the SMP system hardware and setup.
459 */
460 void
461 cpu_mp_announce(void)
462 {
463 int i, x;
464
465 POSTCODE(MP_ANNOUNCE_POST);
466
467 /* List CPUs */
468 printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id);
469 for (i = 1, x = 0; x < MAXCPU; x++) {
470 if (!cpu_info[x].cpu_present || cpu_info[x].cpu_bsp)
471 continue;
472 if (cpu_info[x].cpu_disabled)
473 printf(" cpu (AP): APIC ID: %2d (disabled)\n", x);
474 else {
475 KASSERT(i < mp_ncpus,
476 ("mp_ncpus and actual cpus are out of whack"));
477 printf(" cpu%d (AP): APIC ID: %2d\n", i++, x);
478 }
479 }
480 }
481
482 /*
483 * AP CPU's call this to initialize themselves.
484 */
485 void
486 init_secondary(void)
487 {
488 int gsel_tss;
489 int x, myid;
490 u_int cr0;
491
492 /* bootAP is set in start_ap() to our ID. */
493 myid = bootAP;
494 gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
495 gdt_segs[GPROC0_SEL].ssd_base =
496 (int) &SMP_prvspace[myid].pcpu.pc_common_tss;
497 SMP_prvspace[myid].pcpu.pc_prvspace =
498 &SMP_prvspace[myid].pcpu;
499
500 for (x = 0; x < NGDT; x++) {
501 ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd);
502 }
503
504 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
505 r_gdt.rd_base = (int) &gdt[myid * NGDT];
506 lgdt(&r_gdt); /* does magic intra-segment return */
507
508 lidt(&r_idt);
509
510 lldt(_default_ldt);
511 PCPU_SET(currentldt, _default_ldt);
512
513 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
514 gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
515 PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */
516 PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL));
517 PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16);
518 PCPU_SET(tss_gdt, &gdt[myid * NGDT + GPROC0_SEL].sd);
519 PCPU_SET(common_tssd, *PCPU_GET(tss_gdt));
520 ltr(gsel_tss);
521
522 PCPU_SET(fsgs_gdt, &gdt[myid * NGDT + GUFS_SEL].sd);
523
524 /*
525 * Set to a known state:
526 * Set by mpboot.s: CR0_PG, CR0_PE
527 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
528 */
529 cr0 = rcr0();
530 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
531 load_cr0(cr0);
532 CHECK_WRITE(0x38, 5);
533
534 /* Disable local APIC just to be sure. */
535 lapic_disable();
536
537 /* signal our startup to the BSP. */
538 mp_naps++;
539 CHECK_WRITE(0x39, 6);
540
541 /* Spin until the BSP releases the AP's. */
542 while (!aps_ready)
543 ia32_pause();
544
545 /* BSP may have changed PTD while we were waiting */
546 invltlb();
547 pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
548
549 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
550 lidt(&r_idt);
551 #endif
552
553 /* set up CPU registers and state */
554 cpu_setregs();
555
556 /* set up FPU state on the AP */
557 npxinit(__INITIAL_NPXCW__);
558
559 /* set up SSE registers */
560 enable_sse();
561
562 /* A quick check from sanity claus */
563 if (PCPU_GET(apic_id) != lapic_id()) {
564 printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
565 printf("SMP: actual apic_id = %d\n", lapic_id());
566 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
567 printf("PTD[MPPTDI] = %#jx\n", (uintmax_t)PTD[MPPTDI]);
568 panic("cpuid mismatch! boom!!");
569 }
570
571 mtx_lock_spin(&ap_boot_mtx);
572
573 /* Init local apic for irq's */
574 lapic_setup();
575
576 /* Set memory range attributes for this CPU to match the BSP */
577 mem_range_AP_init();
578
579 smp_cpus++;
580
581 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
582 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
583
584 /* Determine if we are a logical CPU. */
585 if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0)
586 logical_cpus_mask |= PCPU_GET(cpumask);
587
588 /* Determine if we are a hyperthread. */
589 if (hyperthreading_cpus > 1 &&
590 PCPU_GET(apic_id) % hyperthreading_cpus != 0)
591 hyperthreading_cpus_mask |= PCPU_GET(cpumask);
592
593 /* Build our map of 'other' CPUs. */
594 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
595
596 if (bootverbose)
597 lapic_dump("AP");
598
599 if (smp_cpus == mp_ncpus) {
600 /* enable IPI's, tlb shootdown, freezes etc */
601 atomic_store_rel_int(&smp_started, 1);
602 smp_active = 1; /* historic */
603 }
604
605 mtx_unlock_spin(&ap_boot_mtx);
606
607 /* wait until all the AP's are up */
608 while (smp_started == 0)
609 ia32_pause();
610
611 /* ok, now grab sched_lock and enter the scheduler */
612 mtx_lock_spin(&sched_lock);
613
614 binuptime(PCPU_PTR(switchtime));
615 PCPU_SET(switchticks, ticks);
616
617 cpu_throw(NULL, choosethread()); /* doesn't return */
618
619 panic("scheduler returned us to %s", __func__);
620 /* NOTREACHED */
621 }
622
623 /*******************************************************************
624 * local functions and data
625 */
626
627 /*
628 * Set the APIC logical IDs.
629 *
630 * We want to cluster logical CPU's within the same APIC ID cluster.
631 * Since logical CPU's are aligned simply filling in the clusters in
632 * APIC ID order works fine. Note that this does not try to balance
633 * the number of CPU's in each cluster. (XXX?)
634 */
635 static void
636 set_logical_apic_ids(void)
637 {
638 u_int apic_id, cluster, cluster_id;
639
640 /* Force us to allocate cluster 0 at the start. */
641 cluster = -1;
642 cluster_id = APIC_MAX_INTRACLUSTER_ID;
643 for (apic_id = 0; apic_id < MAXCPU; apic_id++) {
644 if (!cpu_info[apic_id].cpu_present)
645 continue;
646 if (cluster_id == APIC_MAX_INTRACLUSTER_ID) {
647 cluster = ioapic_next_logical_cluster();
648 cluster_id = 0;
649 } else
650 cluster_id++;
651 if (bootverbose)
652 printf("APIC ID: physical %u, logical %u:%u\n",
653 apic_id, cluster, cluster_id);
654 lapic_set_logical_id(apic_id, cluster, cluster_id);
655 }
656 }
657
658 /*
659 * start each AP in our list
660 */
661 static int
662 start_all_aps(void)
663 {
664 #ifndef PC98
665 u_char mpbiosreason;
666 #endif
667 u_long mpbioswarmvec;
668 struct pcpu *pc;
669 char *stack;
670 uintptr_t kptbase;
671 int i, pg, apic_id, cpu;
672
673 POSTCODE(START_ALL_APS_POST);
674
675 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
676
677 /* install the AP 1st level boot code */
678 install_ap_tramp();
679
680 /* save the current value of the warm-start vector */
681 mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
682 #ifndef PC98
683 outb(CMOS_REG, BIOS_RESET);
684 mpbiosreason = inb(CMOS_DATA);
685 #endif
686
687 /* set up temporary P==V mapping for AP boot */
688 /* XXX this is a hack, we should boot the AP on its own stack/PTD */
689 kptbase = (uintptr_t)(void *)KPTphys;
690 for (i = 0; i < NKPT; i++)
691 PTD[i] = (pd_entry_t)(PG_V | PG_RW |
692 ((kptbase + i * PAGE_SIZE) & PG_FRAME));
693 invltlb();
694
695 /* start each AP */
696 for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) {
697
698 /* Ignore non-existent CPUs and the BSP. */
699 if (!cpu_info[apic_id].cpu_present ||
700 cpu_info[apic_id].cpu_bsp)
701 continue;
702
703 /* Don't use this CPU if it has been disabled by a tunable. */
704 if (resource_disabled("lapic", apic_id)) {
705 cpu_info[apic_id].cpu_disabled = 1;
706 mp_ncpus--;
707 continue;
708 }
709
710 cpu++;
711
712 /* save APIC ID for this logical ID */
713 cpu_apic_ids[cpu] = apic_id;
714
715 /* first page of AP's private space */
716 pg = cpu * i386_btop(sizeof(struct privatespace));
717
718 /* allocate a new private data page */
719 pc = (struct pcpu *)kmem_alloc(kernel_map, PAGE_SIZE);
720
721 /* wire it into the private page table page */
722 SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(pc));
723
724 /* allocate and set up an idle stack data page */
725 stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */
726 for (i = 0; i < KSTACK_PAGES; i++)
727 SMPpt[pg + 1 + i] = (pt_entry_t)
728 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
729
730 /* prime data page for it to use */
731 pcpu_init(pc, cpu, sizeof(struct pcpu));
732 pc->pc_apic_id = apic_id;
733
734 /* setup a vector to our boot code */
735 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
736 *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
737 #ifndef PC98
738 outb(CMOS_REG, BIOS_RESET);
739 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
740 #endif
741
742 bootSTK = &SMP_prvspace[cpu].idlekstack[KSTACK_PAGES *
743 PAGE_SIZE];
744 bootAP = cpu;
745
746 /* attempt to start the Application Processor */
747 CHECK_INIT(99); /* setup checkpoints */
748 if (!start_ap(apic_id)) {
749 printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id);
750 CHECK_PRINT("trace"); /* show checkpoints */
751 /* better panic as the AP may be running loose */
752 printf("panic y/n? [y] ");
753 if (cngetc() != 'n')
754 panic("bye-bye");
755 }
756 CHECK_PRINT("trace"); /* show checkpoints */
757
758 all_cpus |= (1 << cpu); /* record AP in CPU map */
759 }
760
761 /* build our map of 'other' CPUs */
762 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
763
764 /* restore the warmstart vector */
765 *(u_long *) WARMBOOT_OFF = mpbioswarmvec;
766 #ifndef PC98
767 outb(CMOS_REG, BIOS_RESET);
768 outb(CMOS_DATA, mpbiosreason);
769 #endif
770
771 /*
772 * Set up the idle context for the BSP. Similar to above except
773 * that some was done by locore, some by pmap.c and some is implicit
774 * because the BSP is cpu#0 and the page is initially zero and also
775 * because we can refer to variables by name on the BSP..
776 */
777
778 /* Allocate and setup BSP idle stack */
779 stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
780 for (i = 0; i < KSTACK_PAGES; i++)
781 SMPpt[1 + i] = (pt_entry_t)
782 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
783
784 for (i = 0; i < NKPT; i++)
785 PTD[i] = 0;
786 pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
787
788 /* number of APs actually started */
789 return mp_naps;
790 }
791
792 /*
793 * load the 1st level AP boot code into base memory.
794 */
795
796 /* targets for relocation */
797 extern void bigJump(void);
798 extern void bootCodeSeg(void);
799 extern void bootDataSeg(void);
800 extern void MPentry(void);
801 extern u_int MP_GDT;
802 extern u_int mp_gdtbase;
803
804 static void
805 install_ap_tramp(void)
806 {
807 int x;
808 int size = *(int *) ((u_long) & bootMP_size);
809 vm_offset_t va = boot_address + KERNBASE;
810 u_char *src = (u_char *) ((u_long) bootMP);
811 u_char *dst = (u_char *) va;
812 u_int boot_base = (u_int) bootMP;
813 u_int8_t *dst8;
814 u_int16_t *dst16;
815 u_int32_t *dst32;
816
817 POSTCODE(INSTALL_AP_TRAMP_POST);
818
819 KASSERT (size <= PAGE_SIZE,
820 ("'size' do not fit into PAGE_SIZE, as expected."));
821 pmap_kenter(va, boot_address);
822 pmap_invalidate_page (kernel_pmap, va);
823 for (x = 0; x < size; ++x)
824 *dst++ = *src++;
825
826 /*
827 * modify addresses in code we just moved to basemem. unfortunately we
828 * need fairly detailed info about mpboot.s for this to work. changes
829 * to mpboot.s might require changes here.
830 */
831
832 /* boot code is located in KERNEL space */
833 dst = (u_char *) va;
834
835 /* modify the lgdt arg */
836 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
837 *dst32 = boot_address + ((u_int) & MP_GDT - boot_base);
838
839 /* modify the ljmp target for MPentry() */
840 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
841 *dst32 = ((u_int) MPentry - KERNBASE);
842
843 /* modify the target for boot code segment */
844 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
845 dst8 = (u_int8_t *) (dst16 + 1);
846 *dst16 = (u_int) boot_address & 0xffff;
847 *dst8 = ((u_int) boot_address >> 16) & 0xff;
848
849 /* modify the target for boot data segment */
850 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
851 dst8 = (u_int8_t *) (dst16 + 1);
852 *dst16 = (u_int) boot_address & 0xffff;
853 *dst8 = ((u_int) boot_address >> 16) & 0xff;
854 }
855
856 /*
857 * This function starts the AP (application processor) identified
858 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
859 * to accomplish this. This is necessary because of the nuances
860 * of the different hardware we might encounter. It isn't pretty,
861 * but it seems to work.
862 */
863 static int
864 start_ap(int apic_id)
865 {
866 int vector, ms;
867 int cpus;
868
869 POSTCODE(START_AP_POST);
870
871 /* calculate the vector */
872 vector = (boot_address >> 12) & 0xff;
873
874 /* used as a watchpoint to signal AP startup */
875 cpus = mp_naps;
876
877 /*
878 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
879 * and running the target CPU. OR this INIT IPI might be latched (P5
880 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
881 * ignored.
882 */
883
884 /* do an INIT IPI: assert RESET */
885 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
886 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id);
887
888 /* wait for pending status end */
889 lapic_ipi_wait(-1);
890
891 /* do an INIT IPI: deassert RESET */
892 lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL |
893 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0);
894
895 /* wait for pending status end */
896 DELAY(10000); /* wait ~10mS */
897 lapic_ipi_wait(-1);
898
899 /*
900 * next we do a STARTUP IPI: the previous INIT IPI might still be
901 * latched, (P5 bug) this 1st STARTUP would then terminate
902 * immediately, and the previously started INIT IPI would continue. OR
903 * the previous INIT IPI has already run. and this STARTUP IPI will
904 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
905 * will run.
906 */
907
908 /* do a STARTUP IPI */
909 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
910 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
911 vector, apic_id);
912 lapic_ipi_wait(-1);
913 DELAY(200); /* wait ~200uS */
914
915 /*
916 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
917 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
918 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
919 * recognized after hardware RESET or INIT IPI.
920 */
921
922 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
923 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
924 vector, apic_id);
925 lapic_ipi_wait(-1);
926 DELAY(200); /* wait ~200uS */
927
928 /* Wait up to 5 seconds for it to start. */
929 for (ms = 0; ms < 5000; ms++) {
930 if (mp_naps > cpus)
931 return 1; /* return SUCCESS */
932 DELAY(1000);
933 }
934 return 0; /* return FAILURE */
935 }
936
937 #ifdef COUNT_XINVLTLB_HITS
938 u_int xhits_gbl[MAXCPU];
939 u_int xhits_pg[MAXCPU];
940 u_int xhits_rng[MAXCPU];
941 SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
942 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
943 sizeof(xhits_gbl), "IU", "");
944 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
945 sizeof(xhits_pg), "IU", "");
946 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng,
947 sizeof(xhits_rng), "IU", "");
948
949 u_int ipi_global;
950 u_int ipi_page;
951 u_int ipi_range;
952 u_int ipi_range_size;
953 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, "");
954 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, "");
955 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, "");
956 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size,
957 0, "");
958
959 u_int ipi_masked_global;
960 u_int ipi_masked_page;
961 u_int ipi_masked_range;
962 u_int ipi_masked_range_size;
963 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW,
964 &ipi_masked_global, 0, "");
965 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW,
966 &ipi_masked_page, 0, "");
967 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW,
968 &ipi_masked_range, 0, "");
969 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW,
970 &ipi_masked_range_size, 0, "");
971 #endif /* COUNT_XINVLTLB_HITS */
972
973 /*
974 * Flush the TLB on all other CPU's
975 */
976 static void
977 smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
978 {
979 u_int ncpu;
980
981 ncpu = mp_ncpus - 1; /* does not shootdown self */
982 if (ncpu < 1)
983 return; /* no other cpus */
984 mtx_assert(&smp_ipi_mtx, MA_OWNED);
985 smp_tlb_addr1 = addr1;
986 smp_tlb_addr2 = addr2;
987 atomic_store_rel_int(&smp_tlb_wait, 0);
988 ipi_all_but_self(vector);
989 /*
990 * Enable interrupts here to workaround Opteron Errata 106.
991 * The while loop runs entirely out of instruction cache,
992 * which blocks updates to the cache from other CPUs.
993 * Interrupts break the lock, allowing the write to post.
994 */
995 enable_intr();
996 while (smp_tlb_wait < ncpu)
997 ia32_pause();
998 disable_intr();
999 }
1000
1001 /*
1002 * This is about as magic as it gets. fortune(1) has got similar code
1003 * for reversing bits in a word. Who thinks up this stuff??
1004 *
1005 * Yes, it does appear to be consistently faster than:
1006 * while (i = ffs(m)) {
1007 * m >>= i;
1008 * bits++;
1009 * }
1010 * and
1011 * while (lsb = (m & -m)) { // This is magic too
1012 * m &= ~lsb; // or: m ^= lsb
1013 * bits++;
1014 * }
1015 * Both of these latter forms do some very strange things on gcc-3.1 with
1016 * -mcpu=pentiumpro and/or -march=pentiumpro and/or -O or -O2.
1017 * There is probably an SSE or MMX popcnt instruction.
1018 *
1019 * I wonder if this should be in libkern?
1020 *
1021 * XXX Stop the presses! Another one:
1022 * static __inline u_int32_t
1023 * popcnt1(u_int32_t v)
1024 * {
1025 * v -= ((v >> 1) & 0x55555555);
1026 * v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
1027 * v = (v + (v >> 4)) & 0x0F0F0F0F;
1028 * return (v * 0x01010101) >> 24;
1029 * }
1030 * The downside is that it has a multiply. With a pentium3 with
1031 * -mcpu=pentiumpro and -march=pentiumpro then gcc-3.1 will use
1032 * an imull, and in that case it is faster. In most other cases
1033 * it appears slightly slower.
1034 *
1035 * Another variant (also from fortune):
1036 * #define BITCOUNT(x) (((BX_(x)+(BX_(x)>>4)) & 0x0F0F0F0F) % 255)
1037 * #define BX_(x) ((x) - (((x)>>1)&0x77777777) \
1038 * - (((x)>>2)&0x33333333) \
1039 * - (((x)>>3)&0x11111111))
1040 */
1041 static __inline u_int32_t
1042 popcnt(u_int32_t m)
1043 {
1044
1045 m = (m & 0x55555555) + ((m & 0xaaaaaaaa) >> 1);
1046 m = (m & 0x33333333) + ((m & 0xcccccccc) >> 2);
1047 m = (m & 0x0f0f0f0f) + ((m & 0xf0f0f0f0) >> 4);
1048 m = (m & 0x00ff00ff) + ((m & 0xff00ff00) >> 8);
1049 m = (m & 0x0000ffff) + ((m & 0xffff0000) >> 16);
1050 return m;
1051 }
1052
1053 static void
1054 smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
1055 {
1056 int ncpu, othercpus;
1057
1058 othercpus = mp_ncpus - 1;
1059 if (mask == (u_int)-1) {
1060 ncpu = othercpus;
1061 if (ncpu < 1)
1062 return;
1063 } else {
1064 mask &= ~PCPU_GET(cpumask);
1065 if (mask == 0)
1066 return;
1067 ncpu = popcnt(mask);
1068 if (ncpu > othercpus) {
1069 /* XXX this should be a panic offence */
1070 printf("SMP: tlb shootdown to %d other cpus (only have %d)\n",
1071 ncpu, othercpus);
1072 ncpu = othercpus;
1073 }
1074 /* XXX should be a panic, implied by mask == 0 above */
1075 if (ncpu < 1)
1076 return;
1077 }
1078 mtx_assert(&smp_ipi_mtx, MA_OWNED);
1079 smp_tlb_addr1 = addr1;
1080 smp_tlb_addr2 = addr2;
1081 atomic_store_rel_int(&smp_tlb_wait, 0);
1082 if (mask == (u_int)-1)
1083 ipi_all_but_self(vector);
1084 else
1085 ipi_selected(mask, vector);
1086 /*
1087 * Enable interrupts here to workaround Opteron Errata 106.
1088 * The while loop runs entirely out of instruction cache,
1089 * which blocks updates to the cache from other CPUs.
1090 * Interrupts break the lock, allowing the write to post.
1091 */
1092 enable_intr();
1093 while (smp_tlb_wait < ncpu)
1094 ia32_pause();
1095 disable_intr();
1096 }
1097
1098 void
1099 smp_invltlb(void)
1100 {
1101 if (smp_started) {
1102 smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
1103 #ifdef COUNT_XINVLTLB_HITS
1104 ipi_global++;
1105 #endif
1106 }
1107 }
1108
1109 void
1110 smp_invlpg(vm_offset_t addr)
1111 {
1112 if (smp_started) {
1113 smp_tlb_shootdown(IPI_INVLPG, addr, 0);
1114 #ifdef COUNT_XINVLTLB_HITS
1115 ipi_page++;
1116 #endif
1117 }
1118 }
1119
1120 void
1121 smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2)
1122 {
1123 if (smp_started) {
1124 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
1125 #ifdef COUNT_XINVLTLB_HITS
1126 ipi_range++;
1127 ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
1128 #endif
1129 }
1130 }
1131
1132 void
1133 smp_masked_invltlb(u_int mask)
1134 {
1135 if (smp_started) {
1136 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
1137 #ifdef COUNT_XINVLTLB_HITS
1138 ipi_masked_global++;
1139 #endif
1140 }
1141 }
1142
1143 void
1144 smp_masked_invlpg(u_int mask, vm_offset_t addr)
1145 {
1146 if (smp_started) {
1147 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
1148 #ifdef COUNT_XINVLTLB_HITS
1149 ipi_masked_page++;
1150 #endif
1151 }
1152 }
1153
1154 void
1155 smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2)
1156 {
1157 if (smp_started) {
1158 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
1159 #ifdef COUNT_XINVLTLB_HITS
1160 ipi_masked_range++;
1161 ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE;
1162 #endif
1163 }
1164 }
1165
1166
1167 /*
1168 * For statclock, we send an IPI to all CPU's to have them call this
1169 * function.
1170 */
1171
1172 void
1173 forward_statclock(void)
1174 {
1175 int map;
1176
1177 CTR0(KTR_SMP, "forward_statclock");
1178
1179 if (!smp_started || cold || panicstr)
1180 return;
1181
1182 map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask);
1183 if (map != 0)
1184 ipi_selected(map, IPI_STATCLOCK);
1185 }
1186
1187 /*
1188 * For each hardclock(), we send an IPI to all other CPU's to have them
1189 * execute this function. It would be nice to reduce contention on
1190 * sched_lock if we could simply peek at the CPU to determine the user/kernel
1191 * state and call hardclock_process() on the CPU receiving the clock interrupt
1192 * and then just use a simple IPI to handle any ast's if needed.
1193 */
1194
1195 void
1196 forward_hardclock(void)
1197 {
1198 u_int map;
1199
1200 CTR0(KTR_SMP, "forward_hardclock");
1201
1202 if (!smp_started || cold || panicstr)
1203 return;
1204
1205 map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask);
1206 if (map != 0)
1207 ipi_selected(map, IPI_HARDCLOCK);
1208 }
1209
1210 void
1211 ipi_bitmap_handler(struct clockframe frame)
1212 {
1213 int cpu = PCPU_GET(cpuid);
1214 u_int ipi_bitmap;
1215 struct thread *td;
1216
1217 ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
1218
1219 critical_enter();
1220
1221 /* Nothing to do for AST */
1222
1223 if (ipi_bitmap & (1 << IPI_HARDCLOCK)) {
1224 td = curthread;
1225 td->td_intr_nesting_level++;
1226 hardclock_process(&frame);
1227 td->td_intr_nesting_level--;
1228 }
1229
1230 if (ipi_bitmap & (1 << IPI_STATCLOCK)) {
1231 CTR0(KTR_SMP, "forwarded_statclock");
1232
1233 td = curthread;
1234 td->td_intr_nesting_level++;
1235 if (profprocs != 0)
1236 profclock(&frame);
1237 if (pscnt == psdiv)
1238 statclock(&frame);
1239 td->td_intr_nesting_level--;
1240 }
1241
1242 critical_exit();
1243 }
1244
1245 /*
1246 * send an IPI to a set of cpus.
1247 */
1248 void
1249 ipi_selected(u_int32_t cpus, u_int ipi)
1250 {
1251 int cpu;
1252 u_int bitmap = 0;
1253 u_int old_pending;
1254 u_int new_pending;
1255
1256 if (IPI_IS_BITMAPED(ipi)) {
1257 bitmap = 1 << ipi;
1258 ipi = IPI_BITMAP_VECTOR;
1259 }
1260
1261 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
1262 while ((cpu = ffs(cpus)) != 0) {
1263 cpu--;
1264 cpus &= ~(1 << cpu);
1265
1266 KASSERT(cpu_apic_ids[cpu] != -1,
1267 ("IPI to non-existent CPU %d", cpu));
1268
1269 if (bitmap) {
1270 do {
1271 old_pending = cpu_ipi_pending[cpu];
1272 new_pending = old_pending | bitmap;
1273 } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));
1274
1275 if (old_pending)
1276 continue;
1277 }
1278
1279 lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
1280 }
1281
1282 }
1283
1284 /*
1285 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself
1286 */
1287 void
1288 ipi_all(u_int ipi)
1289 {
1290
1291 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1292 lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL);
1293 }
1294
1295 /*
1296 * send an IPI to all CPUs EXCEPT myself
1297 */
1298 void
1299 ipi_all_but_self(u_int ipi)
1300 {
1301
1302 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1303 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
1304 }
1305
1306 /*
1307 * send an IPI to myself
1308 */
1309 void
1310 ipi_self(u_int ipi)
1311 {
1312
1313 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1314 lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
1315 }
1316
1317 #ifdef KDB_STOP_NMI
1318 /*
1319 * send NMI IPI to selected CPUs
1320 */
1321
1322 #define BEFORE_SPIN 1000000
1323
1324 void
1325 ipi_nmi_selected(u_int32_t cpus)
1326 {
1327
1328 int cpu;
1329 register_t icrlo;
1330
1331 icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT
1332 | APIC_TRIGMOD_EDGE;
1333
1334 CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus);
1335
1336
1337 atomic_set_int(&ipi_nmi_pending, cpus);
1338
1339
1340 while ((cpu = ffs(cpus)) != 0) {
1341 cpu--;
1342 cpus &= ~(1 << cpu);
1343
1344 KASSERT(cpu_apic_ids[cpu] != -1,
1345 ("IPI NMI to non-existent CPU %d", cpu));
1346
1347 /* Wait for an earlier IPI to finish. */
1348 if (!lapic_ipi_wait(BEFORE_SPIN))
1349 panic("ipi_nmi_selected: previous IPI has not cleared");
1350
1351 lapic_ipi_raw(icrlo,cpu_apic_ids[cpu]);
1352 }
1353 }
1354
1355
1356 int
1357 ipi_nmi_handler()
1358 {
1359 int cpu = PCPU_GET(cpuid);
1360
1361 if(!(atomic_load_acq_int(&ipi_nmi_pending) & (1 << cpu)))
1362 return 1;
1363
1364 atomic_clear_int(&ipi_nmi_pending,1 << cpu);
1365
1366 savectx(&stoppcbs[cpu]);
1367
1368 /* Indicate that we are stopped */
1369 atomic_set_int(&stopped_cpus,1 << cpu);
1370
1371
1372 /* Wait for restart */
1373 while(!(atomic_load_acq_int(&started_cpus) & (1 << cpu)))
1374 ia32_pause();
1375
1376 atomic_clear_int(&started_cpus,1 << cpu);
1377 atomic_clear_int(&stopped_cpus,1 << cpu);
1378
1379 if(cpu == 0 && cpustop_restartfunc != NULL)
1380 cpustop_restartfunc();
1381
1382 return 0;
1383 }
1384
1385 #endif /* KDB_STOP_NMI */
1386
1387 /*
1388 * This is called once the rest of the system is up and running and we're
1389 * ready to let the AP's out of the pen.
1390 */
1391 static void
1392 release_aps(void *dummy __unused)
1393 {
1394
1395 if (mp_ncpus == 1)
1396 return;
1397 mtx_lock_spin(&sched_lock);
1398 atomic_store_rel_int(&aps_ready, 1);
1399 while (smp_started == 0)
1400 ia32_pause();
1401 mtx_unlock_spin(&sched_lock);
1402 }
1403 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
1404
1405 static int
1406 sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS)
1407 {
1408 u_int mask;
1409 int error;
1410
1411 mask = hlt_cpus_mask;
1412 error = sysctl_handle_int(oidp, &mask, 0, req);
1413 if (error || !req->newptr)
1414 return (error);
1415
1416 if (logical_cpus_mask != 0 &&
1417 (mask & logical_cpus_mask) == logical_cpus_mask)
1418 hlt_logical_cpus = 1;
1419 else
1420 hlt_logical_cpus = 0;
1421
1422 if (! hyperthreading_allowed)
1423 mask |= hyperthreading_cpus_mask;
1424
1425 if ((mask & all_cpus) == all_cpus)
1426 mask &= ~(1<<0);
1427 hlt_cpus_mask = mask;
1428 return (error);
1429 }
1430 SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW,
1431 0, 0, sysctl_hlt_cpus, "IU",
1432 "Bitmap of CPUs to halt. 101 (binary) will halt CPUs 0 and 2.");
1433
1434 static int
1435 sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS)
1436 {
1437 int disable, error;
1438
1439 disable = hlt_logical_cpus;
1440 error = sysctl_handle_int(oidp, &disable, 0, req);
1441 if (error || !req->newptr)
1442 return (error);
1443
1444 if (disable)
1445 hlt_cpus_mask |= logical_cpus_mask;
1446 else
1447 hlt_cpus_mask &= ~logical_cpus_mask;
1448
1449 if (! hyperthreading_allowed)
1450 hlt_cpus_mask |= hyperthreading_cpus_mask;
1451
1452 if ((hlt_cpus_mask & all_cpus) == all_cpus)
1453 hlt_cpus_mask &= ~(1<<0);
1454
1455 hlt_logical_cpus = disable;
1456 return (error);
1457 }
1458
1459 static int
1460 sysctl_hyperthreading_allowed(SYSCTL_HANDLER_ARGS)
1461 {
1462 int allowed, error;
1463
1464 allowed = hyperthreading_allowed;
1465 error = sysctl_handle_int(oidp, &allowed, 0, req);
1466 if (error || !req->newptr)
1467 return (error);
1468
1469 if (allowed)
1470 hlt_cpus_mask &= ~hyperthreading_cpus_mask;
1471 else
1472 hlt_cpus_mask |= hyperthreading_cpus_mask;
1473
1474 if (logical_cpus_mask != 0 &&
1475 (hlt_cpus_mask & logical_cpus_mask) == logical_cpus_mask)
1476 hlt_logical_cpus = 1;
1477 else
1478 hlt_logical_cpus = 0;
1479
1480 if ((hlt_cpus_mask & all_cpus) == all_cpus)
1481 hlt_cpus_mask &= ~(1<<0);
1482
1483 hyperthreading_allowed = allowed;
1484 return (error);
1485 }
1486
1487 static void
1488 cpu_hlt_setup(void *dummy __unused)
1489 {
1490
1491 if (logical_cpus_mask != 0) {
1492 TUNABLE_INT_FETCH("machdep.hlt_logical_cpus",
1493 &hlt_logical_cpus);
1494 sysctl_ctx_init(&logical_cpu_clist);
1495 SYSCTL_ADD_PROC(&logical_cpu_clist,
1496 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO,
1497 "hlt_logical_cpus", CTLTYPE_INT|CTLFLAG_RW, 0, 0,
1498 sysctl_hlt_logical_cpus, "IU", "");
1499 SYSCTL_ADD_UINT(&logical_cpu_clist,
1500 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO,
1501 "logical_cpus_mask", CTLTYPE_INT|CTLFLAG_RD,
1502 &logical_cpus_mask, 0, "");
1503
1504 if (hlt_logical_cpus)
1505 hlt_cpus_mask |= logical_cpus_mask;
1506
1507 /*
1508 * If necessary for security purposes, force
1509 * hyperthreading off, regardless of the value
1510 * of hlt_logical_cpus.
1511 */
1512 if (hyperthreading_cpus_mask) {
1513 TUNABLE_INT_FETCH("machdep.hyperthreading_allowed",
1514 &hyperthreading_allowed);
1515 SYSCTL_ADD_PROC(&logical_cpu_clist,
1516 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO,
1517 "hyperthreading_allowed", CTLTYPE_INT|CTLFLAG_RW,
1518 0, 0, sysctl_hyperthreading_allowed, "IU", "");
1519 if (! hyperthreading_allowed)
1520 hlt_cpus_mask |= hyperthreading_cpus_mask;
1521 }
1522 }
1523 }
1524 SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_ANY, cpu_hlt_setup, NULL);
1525
1526 int
1527 mp_grab_cpu_hlt(void)
1528 {
1529 u_int mask = PCPU_GET(cpumask);
1530 #ifdef MP_WATCHDOG
1531 u_int cpuid = PCPU_GET(cpuid);
1532 #endif
1533 int retval;
1534
1535 #ifdef MP_WATCHDOG
1536 ap_watchdog(cpuid);
1537 #endif
1538
1539 retval = mask & hlt_cpus_mask;
1540 while (mask & hlt_cpus_mask)
1541 __asm __volatile("sti; hlt" : : : "memory");
1542 return (retval);
1543 }
Cache object: deec121a05ad9ec7f8c57fd4480aeb8e
|