The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/xen/mp_machdep.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1996, by Steve Passe
    3  * Copyright (c) 2008, by Kip Macy
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. The name of the developer may NOT be used to endorse or promote products
   12  *    derived from this software without specific prior written permission.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD: releng/10.2/sys/i386/xen/mp_machdep.c 278423 2015-02-08 23:04:32Z marius $");
   29 
   30 #include "opt_apic.h"
   31 #include "opt_cpu.h"
   32 #include "opt_kstack_pages.h"
   33 #include "opt_mp_watchdog.h"
   34 #include "opt_pmap.h"
   35 #include "opt_sched.h"
   36 #include "opt_smp.h"
   37 
   38 #if !defined(lint)
   39 #if !defined(SMP)
   40 #error How did you get here?
   41 #endif
   42 
   43 #ifndef DEV_APIC
   44 #error The apic device is required for SMP, add "device apic" to your config file.
   45 #endif
   46 #if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT)
   47 #error SMP not supported with CPU_DISABLE_CMPXCHG
   48 #endif
   49 #endif /* not lint */
   50 
   51 #include <sys/param.h>
   52 #include <sys/systm.h>
   53 #include <sys/bus.h>
   54 #include <sys/cons.h>   /* cngetc() */
   55 #include <sys/cpuset.h>
   56 #ifdef GPROF 
   57 #include <sys/gmon.h>
   58 #endif
   59 #include <sys/kernel.h>
   60 #include <sys/ktr.h>
   61 #include <sys/lock.h>
   62 #include <sys/malloc.h>
   63 #include <sys/memrange.h>
   64 #include <sys/mutex.h>
   65 #include <sys/pcpu.h>
   66 #include <sys/proc.h>
   67 #include <sys/rwlock.h>
   68 #include <sys/sched.h>
   69 #include <sys/smp.h>
   70 #include <sys/sysctl.h>
   71 
   72 #include <vm/vm.h>
   73 #include <vm/vm_param.h>
   74 #include <vm/pmap.h>
   75 #include <vm/vm_kern.h>
   76 #include <vm/vm_extern.h>
   77 #include <vm/vm_page.h>
   78 
   79 #include <x86/apicreg.h>
   80 #include <machine/md_var.h>
   81 #include <machine/mp_watchdog.h>
   82 #include <machine/pcb.h>
   83 #include <machine/psl.h>
   84 #include <machine/smp.h>
   85 #include <machine/specialreg.h>
   86 #include <machine/pcpu.h>
   87 
   88 #include <xen/xen-os.h>
   89 #include <xen/evtchn.h>
   90 #include <xen/xen_intr.h>
   91 #include <xen/hypervisor.h>
   92 #include <xen/interface/vcpu.h>
   93 
   94 /*---------------------------- Extern Declarations ---------------------------*/
   95 extern  struct pcpu __pcpu[];
   96 
   97 extern void Xhypervisor_callback(void);
   98 extern void failsafe_callback(void);
   99 extern void pmap_lazyfix_action(void);
  100 
  101 /*--------------------------- Forward Declarations ---------------------------*/
  102 static driver_filter_t  smp_reschedule_interrupt;
  103 static driver_filter_t  smp_call_function_interrupt;
  104 static void             assign_cpu_ids(void);
  105 static void             set_interrupt_apic_ids(void);
  106 static int              start_all_aps(void);
  107 static int              start_ap(int apic_id);
  108 static void             release_aps(void *dummy);
  109 
  110 /*---------------------------------- Macros ----------------------------------*/
  111 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
  112 
  113 /*-------------------------------- Local Types -------------------------------*/
  114 typedef void call_data_func_t(uintptr_t , uintptr_t);
  115 
  116 struct cpu_info {
  117         int     cpu_present:1;
  118         int     cpu_bsp:1;
  119         int     cpu_disabled:1;
  120 };
  121 
  122 struct xen_ipi_handler
  123 {
  124         driver_filter_t *filter;
  125         const char      *description;
  126 };
  127 
  128 enum {
  129         RESCHEDULE_VECTOR,
  130         CALL_FUNCTION_VECTOR,
  131 };
  132 
  133 /*-------------------------------- Global Data -------------------------------*/
  134 static u_int    hyperthreading_cpus;
  135 static cpuset_t hyperthreading_cpus_mask;
  136 
  137 int     mp_naps;                /* # of Applications processors */
  138 int     boot_cpu_id = -1;       /* designated BSP */
  139 
  140 static int bootAP;
  141 static union descriptor *bootAPgdt;
  142 
  143 /* Free these after use */
  144 void *bootstacks[MAXCPU];
  145 
  146 struct pcb stoppcbs[MAXCPU];
  147 
  148 /* Variables needed for SMP tlb shootdown. */
  149 vm_offset_t smp_tlb_addr1;
  150 vm_offset_t smp_tlb_addr2;
  151 volatile int smp_tlb_wait;
  152 
  153 static u_int logical_cpus;
  154 static volatile cpuset_t ipi_nmi_pending;
  155 
  156 /* used to hold the AP's until we are ready to release them */
  157 static struct mtx ap_boot_mtx;
  158 
  159 /* Set to 1 once we're ready to let the APs out of the pen. */
  160 static volatile int aps_ready = 0;
  161 
  162 /*
  163  * Store data from cpu_add() until later in the boot when we actually setup
  164  * the APs.
  165  */
  166 static struct cpu_info cpu_info[MAX_APIC_ID + 1];
  167 int cpu_apic_ids[MAXCPU];
  168 int apic_cpuids[MAX_APIC_ID + 1];
  169 
  170 /* Holds pending bitmap based IPIs per CPU */
  171 static volatile u_int cpu_ipi_pending[MAXCPU];
  172 
  173 static int cpu_logical;
  174 static int cpu_cores;
  175 
  176 static const struct xen_ipi_handler xen_ipis[] = 
  177 {
  178         [RESCHEDULE_VECTOR]     = { smp_reschedule_interrupt,   "resched"  },
  179         [CALL_FUNCTION_VECTOR]  = { smp_call_function_interrupt,"callfunc" }
  180 };
  181 
  182 /*------------------------------- Per-CPU Data -------------------------------*/
  183 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
  184 DPCPU_DEFINE(struct vcpu_info *, vcpu_info);
  185 
  186 /*------------------------------ Implementation ------------------------------*/
  187 struct cpu_group *
  188 cpu_topo(void)
  189 {
  190         if (cpu_cores == 0)
  191                 cpu_cores = 1;
  192         if (cpu_logical == 0)
  193                 cpu_logical = 1;
  194         if (mp_ncpus % (cpu_cores * cpu_logical) != 0) {
  195                 printf("WARNING: Non-uniform processors.\n");
  196                 printf("WARNING: Using suboptimal topology.\n");
  197                 return (smp_topo_none());
  198         }
  199         /*
  200          * No multi-core or hyper-threaded.
  201          */
  202         if (cpu_logical * cpu_cores == 1)
  203                 return (smp_topo_none());
  204         /*
  205          * Only HTT no multi-core.
  206          */
  207         if (cpu_logical > 1 && cpu_cores == 1)
  208                 return (smp_topo_1level(CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
  209         /*
  210          * Only multi-core no HTT.
  211          */
  212         if (cpu_cores > 1 && cpu_logical == 1)
  213                 return (smp_topo_1level(CG_SHARE_NONE, cpu_cores, 0));
  214         /*
  215          * Both HTT and multi-core.
  216          */
  217         return (smp_topo_2level(CG_SHARE_NONE, cpu_cores,
  218             CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
  219 }
  220 
  221 /*
  222  * Calculate usable address in base memory for AP trampoline code.
  223  */
  224 u_int
  225 mp_bootaddress(u_int basemem)
  226 {
  227 
  228         return (basemem);
  229 }
  230 
  231 void
  232 cpu_add(u_int apic_id, char boot_cpu)
  233 {
  234 
  235         if (apic_id > MAX_APIC_ID) {
  236                 panic("SMP: APIC ID %d too high", apic_id);
  237                 return;
  238         }
  239         KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice",
  240             apic_id));
  241         cpu_info[apic_id].cpu_present = 1;
  242         if (boot_cpu) {
  243                 KASSERT(boot_cpu_id == -1,
  244                     ("CPU %d claims to be BSP, but CPU %d already is", apic_id,
  245                     boot_cpu_id));
  246                 boot_cpu_id = apic_id;
  247                 cpu_info[apic_id].cpu_bsp = 1;
  248         }
  249         if (mp_ncpus < MAXCPU)
  250                 mp_ncpus++;
  251         if (bootverbose)
  252                 printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" :
  253                     "AP");
  254 }
  255 
  256 void
  257 cpu_mp_setmaxid(void)
  258 {
  259 
  260         mp_maxid = MAXCPU - 1;
  261 }
  262 
  263 int
  264 cpu_mp_probe(void)
  265 {
  266 
  267         /*
  268          * Always record BSP in CPU map so that the mbuf init code works
  269          * correctly.
  270          */
  271         CPU_SETOF(0, &all_cpus);
  272         if (mp_ncpus == 0) {
  273                 /*
  274                  * No CPUs were found, so this must be a UP system.  Setup
  275                  * the variables to represent a system with a single CPU
  276                  * with an id of 0.
  277                  */
  278                 mp_ncpus = 1;
  279                 return (0);
  280         }
  281 
  282         /* At least one CPU was found. */
  283         if (mp_ncpus == 1) {
  284                 /*
  285                  * One CPU was found, so this must be a UP system with
  286                  * an I/O APIC.
  287                  */
  288                 return (0);
  289         }
  290 
  291         /* At least two CPUs were found. */
  292         return (1);
  293 }
  294 
  295 /*
  296  * Initialize the IPI handlers and start up the AP's.
  297  */
  298 void
  299 cpu_mp_start(void)
  300 {
  301         int i;
  302 
  303         /* Initialize the logical ID to APIC ID table. */
  304         for (i = 0; i < MAXCPU; i++) {
  305                 cpu_apic_ids[i] = -1;
  306                 cpu_ipi_pending[i] = 0;
  307         }
  308 
  309         /* Set boot_cpu_id if needed. */
  310         if (boot_cpu_id == -1) {
  311                 boot_cpu_id = PCPU_GET(apic_id);
  312                 cpu_info[boot_cpu_id].cpu_bsp = 1;
  313         } else
  314                 KASSERT(boot_cpu_id == PCPU_GET(apic_id),
  315                     ("BSP's APIC ID doesn't match boot_cpu_id"));
  316         cpu_apic_ids[0] = boot_cpu_id;
  317         apic_cpuids[boot_cpu_id] = 0;
  318 
  319         assign_cpu_ids();
  320 
  321         /* Start each Application Processor */
  322         start_all_aps();
  323 
  324         /* Setup the initial logical CPUs info. */
  325         logical_cpus = 0;
  326         CPU_ZERO(&logical_cpus_mask);
  327         if (cpu_feature & CPUID_HTT)
  328                 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16;
  329 
  330         set_interrupt_apic_ids();
  331 }
  332 
  333 
  334 static void
  335 iv_rendezvous(uintptr_t a, uintptr_t b)
  336 {
  337         smp_rendezvous_action();
  338 }
  339 
  340 static void
  341 iv_invltlb(uintptr_t a, uintptr_t b)
  342 {
  343         xen_tlb_flush();
  344 }
  345 
  346 static void
  347 iv_invlpg(uintptr_t a, uintptr_t b)
  348 {
  349         xen_invlpg(a);
  350 }
  351 
  352 static void
  353 iv_invlrng(uintptr_t a, uintptr_t b)
  354 {
  355         vm_offset_t start = (vm_offset_t)a;
  356         vm_offset_t end = (vm_offset_t)b;
  357 
  358         while (start < end) {
  359                 xen_invlpg(start);
  360                 start += PAGE_SIZE;
  361         }
  362 }
  363 
  364 
  365 static void
  366 iv_invlcache(uintptr_t a, uintptr_t b)
  367 {
  368 
  369         wbinvd();
  370         atomic_add_int(&smp_tlb_wait, 1);
  371 }
  372 
  373 static void
  374 iv_lazypmap(uintptr_t a, uintptr_t b)
  375 {
  376         pmap_lazyfix_action();
  377         atomic_add_int(&smp_tlb_wait, 1);
  378 }
  379 
  380 /*
  381  * These start from "IPI offset" APIC_IPI_INTS
  382  */
  383 static call_data_func_t *ipi_vectors[6] = 
  384 {
  385         iv_rendezvous,
  386         iv_invltlb,
  387         iv_invlpg,
  388         iv_invlrng,
  389         iv_invlcache,
  390         iv_lazypmap,
  391 };
  392 
  393 /*
  394  * Reschedule call back. Nothing to do,
  395  * all the work is done automatically when
  396  * we return from the interrupt.
  397  */
  398 static int
  399 smp_reschedule_interrupt(void *unused)
  400 {
  401         int cpu = PCPU_GET(cpuid);
  402         u_int ipi_bitmap;
  403 
  404         ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
  405 
  406         if (ipi_bitmap & (1 << IPI_PREEMPT)) {
  407 #ifdef COUNT_IPIS
  408                 (*ipi_preempt_counts[cpu])++;
  409 #endif
  410                 sched_preempt(curthread);
  411         }
  412 
  413         if (ipi_bitmap & (1 << IPI_AST)) {
  414 #ifdef COUNT_IPIS
  415                 (*ipi_ast_counts[cpu])++;
  416 #endif
  417                 /* Nothing to do for AST */
  418         }       
  419         return (FILTER_HANDLED);
  420 }
  421 
  422 struct _call_data {
  423         uint16_t func_id;
  424         uint16_t wait;
  425         uintptr_t arg1;
  426         uintptr_t arg2;
  427         atomic_t started;
  428         atomic_t finished;
  429 };
  430 
  431 static struct _call_data *call_data;
  432 
  433 static int
  434 smp_call_function_interrupt(void *unused)
  435 {       
  436         call_data_func_t *func;
  437         uintptr_t arg1 = call_data->arg1;
  438         uintptr_t arg2 = call_data->arg2;
  439         int wait = call_data->wait;
  440         atomic_t *started = &call_data->started;
  441         atomic_t *finished = &call_data->finished;
  442 
  443         /* We only handle function IPIs, not bitmap IPIs */
  444         if (call_data->func_id < APIC_IPI_INTS ||
  445             call_data->func_id > IPI_BITMAP_VECTOR)
  446                 panic("invalid function id %u", call_data->func_id);
  447         
  448         func = ipi_vectors[IPI_TO_IDX(call_data->func_id)];
  449         /*
  450          * Notify initiating CPU that I've grabbed the data and am
  451          * about to execute the function
  452          */
  453         mb();
  454         atomic_inc(started);
  455         /*
  456          * At this point the info structure may be out of scope unless wait==1
  457          */
  458         (*func)(arg1, arg2);
  459 
  460         if (wait) {
  461                 mb();
  462                 atomic_inc(finished);
  463         }
  464         atomic_add_int(&smp_tlb_wait, 1);
  465         return (FILTER_HANDLED);
  466 }
  467 
  468 /*
  469  * Print various information about the SMP system hardware and setup.
  470  */
  471 void
  472 cpu_mp_announce(void)
  473 {
  474         int i, x;
  475 
  476         /* List CPUs */
  477         printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id);
  478         for (i = 1, x = 0; x <= MAX_APIC_ID; x++) {
  479                 if (!cpu_info[x].cpu_present || cpu_info[x].cpu_bsp)
  480                         continue;
  481                 if (cpu_info[x].cpu_disabled)
  482                         printf("  cpu (AP): APIC ID: %2d (disabled)\n", x);
  483                 else {
  484                         KASSERT(i < mp_ncpus,
  485                             ("mp_ncpus and actual cpus are out of whack"));
  486                         printf(" cpu%d (AP): APIC ID: %2d\n", i++, x);
  487                 }
  488         }
  489 }
  490 
  491 static int
  492 xen_smp_cpu_init(unsigned int cpu)
  493 {
  494         xen_intr_handle_t *ipi_handle;
  495         const struct xen_ipi_handler *ipi;
  496         int idx, rc;
  497 
  498         ipi_handle = DPCPU_ID_GET(cpu, ipi_handle);
  499         for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) {
  500 
  501                 /*
  502                  * The PCPU variable pc_device is not initialized on i386 PV,
  503                  * so we have to use the root_bus device in order to setup
  504                  * the IPIs.
  505                  */
  506                 rc = xen_intr_alloc_and_bind_ipi(root_bus, cpu,
  507                     ipi->filter, INTR_TYPE_TTY, &ipi_handle[idx]);
  508                 if (rc != 0) {
  509                         printf("Unable to allocate a XEN IPI port. "
  510                             "Error %d\n", rc);
  511                         break;
  512                 }
  513                 xen_intr_describe(ipi_handle[idx], "%s", ipi->description);
  514         }
  515 
  516         for (;idx < nitems(xen_ipis); idx++)
  517                     ipi_handle[idx] = NULL;
  518 
  519         if (rc == 0)
  520                 return (0);
  521 
  522         /* Either all are successfully mapped, or none at all. */
  523         for (idx = 0; idx < nitems(xen_ipis); idx++) {
  524                 if (ipi_handle[idx] == NULL)
  525                         continue;
  526 
  527                 xen_intr_unbind(ipi_handle[idx]);
  528                 ipi_handle[idx] = NULL;
  529         }
  530 
  531         return (rc);
  532 }
  533 
  534 static void
  535 xen_smp_intr_init_cpus(void *unused)
  536 {
  537         int i;
  538             
  539         for (i = 0; i < mp_ncpus; i++)
  540                 xen_smp_cpu_init(i);
  541 }
  542 
  543 static void
  544 xen_smp_intr_setup_cpus(void *unused)
  545 {
  546         int i;
  547 
  548         for (i = 0; i < mp_ncpus; i++)
  549                 DPCPU_ID_SET(i, vcpu_info,
  550                     &HYPERVISOR_shared_info->vcpu_info[i]);
  551 }
  552 
  553 #define MTOPSIZE (1<<(14 + PAGE_SHIFT))
  554 
  555 /*
  556  * AP CPU's call this to initialize themselves.
  557  */
  558 void
  559 init_secondary(void)
  560 {
  561         vm_offset_t addr;
  562         u_int   cpuid;
  563         int     gsel_tss;
  564         
  565         
  566         /* bootAP is set in start_ap() to our ID. */
  567         PCPU_SET(currentldt, _default_ldt);
  568         gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
  569 #if 0
  570         gdt[bootAP * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
  571 #endif
  572         PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */
  573         PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL));
  574         PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16);
  575 #if 0
  576         PCPU_SET(tss_gdt, &gdt[bootAP * NGDT + GPROC0_SEL].sd);
  577 
  578         PCPU_SET(common_tssd, *PCPU_GET(tss_gdt));
  579 #endif
  580         PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd);
  581 
  582         /*
  583          * Set to a known state:
  584          * Set by mpboot.s: CR0_PG, CR0_PE
  585          * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
  586          */
  587         /*
  588          * signal our startup to the BSP.
  589          */
  590         mp_naps++;
  591 
  592         /* Spin until the BSP releases the AP's. */
  593         while (!aps_ready)
  594                 ia32_pause();
  595 
  596         /* BSP may have changed PTD while we were waiting */
  597         invltlb();
  598         for (addr = 0; addr < NKPT * NBPDR - 1; addr += PAGE_SIZE)
  599                 invlpg(addr);
  600 
  601 #if 0
  602         /* set up SSE/NX */
  603         initializecpu();
  604 #endif
  605 
  606         /* set up FPU state on the AP */
  607         npxinit(false);
  608 #if 0
  609         /* A quick check from sanity claus */
  610         if (PCPU_GET(apic_id) != lapic_id()) {
  611                 printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
  612                 printf("SMP: actual apic_id = %d\n", lapic_id());
  613                 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
  614                 panic("cpuid mismatch! boom!!");
  615         }
  616 #endif
  617         
  618         /* Initialize curthread. */
  619         KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
  620         PCPU_SET(curthread, PCPU_GET(idlethread));
  621 
  622         mtx_lock_spin(&ap_boot_mtx);
  623 #if 0
  624         
  625         /* Init local apic for irq's */
  626         lapic_setup(1);
  627 #endif
  628         smp_cpus++;
  629 
  630         cpuid = PCPU_GET(cpuid);
  631         CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
  632         printf("SMP: AP CPU #%d Launched!\n", cpuid);
  633 
  634         /* Determine if we are a logical CPU. */
  635         if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0)
  636                 CPU_SET(cpuid, &logical_cpus_mask);
  637         
  638         /* Determine if we are a hyperthread. */
  639         if (hyperthreading_cpus > 1 &&
  640             PCPU_GET(apic_id) % hyperthreading_cpus != 0)
  641                 CPU_SET(cpuid, &hyperthreading_cpus_mask);
  642 #if 0
  643         if (bootverbose)
  644                 lapic_dump("AP");
  645 #endif
  646         if (smp_cpus == mp_ncpus) {
  647                 /* enable IPI's, tlb shootdown, freezes etc */
  648                 atomic_store_rel_int(&smp_started, 1);
  649         }
  650 
  651         mtx_unlock_spin(&ap_boot_mtx);
  652 
  653         /* wait until all the AP's are up */
  654         while (smp_started == 0)
  655                 ia32_pause();
  656 
  657         PCPU_SET(curthread, PCPU_GET(idlethread));
  658 
  659         /* Start per-CPU event timers. */
  660         cpu_initclocks_ap();
  661 
  662         /* enter the scheduler */
  663         sched_throw(NULL);
  664 
  665         panic("scheduler returned us to %s", __func__);
  666         /* NOTREACHED */
  667 }
  668 
  669 /*******************************************************************
  670  * local functions and data
  671  */
  672 
  673 /*
  674  * We tell the I/O APIC code about all the CPUs we want to receive
  675  * interrupts.  If we don't want certain CPUs to receive IRQs we
  676  * can simply not tell the I/O APIC code about them in this function.
  677  * We also do not tell it about the BSP since it tells itself about
  678  * the BSP internally to work with UP kernels and on UP machines.
  679  */
  680 static void
  681 set_interrupt_apic_ids(void)
  682 {
  683         u_int i, apic_id;
  684 
  685         for (i = 0; i < MAXCPU; i++) {
  686                 apic_id = cpu_apic_ids[i];
  687                 if (apic_id == -1)
  688                         continue;
  689                 if (cpu_info[apic_id].cpu_bsp)
  690                         continue;
  691                 if (cpu_info[apic_id].cpu_disabled)
  692                         continue;
  693 
  694                 /* Don't let hyperthreads service interrupts. */
  695                 if (hyperthreading_cpus > 1 &&
  696                     apic_id % hyperthreading_cpus != 0)
  697                         continue;
  698 
  699                 intr_add_cpu(i);
  700         }
  701 }
  702 
  703 /*
  704  * Assign logical CPU IDs to local APICs.
  705  */
  706 static void
  707 assign_cpu_ids(void)
  708 {
  709         u_int i;
  710 
  711         /* Check for explicitly disabled CPUs. */
  712         for (i = 0; i <= MAX_APIC_ID; i++) {
  713                 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp)
  714                         continue;
  715 
  716                 /* Don't use this CPU if it has been disabled by a tunable. */
  717                 if (resource_disabled("lapic", i)) {
  718                         cpu_info[i].cpu_disabled = 1;
  719                         continue;
  720                 }
  721         }
  722 
  723         /*
  724          * Assign CPU IDs to local APIC IDs and disable any CPUs
  725          * beyond MAXCPU.  CPU 0 has already been assigned to the BSP,
  726          * so we only have to assign IDs for APs.
  727          */
  728         mp_ncpus = 1;
  729         for (i = 0; i <= MAX_APIC_ID; i++) {
  730                 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp ||
  731                     cpu_info[i].cpu_disabled)
  732                         continue;
  733 
  734                 if (mp_ncpus < MAXCPU) {
  735                         cpu_apic_ids[mp_ncpus] = i;
  736                         apic_cpuids[i] = mp_ncpus;
  737                         mp_ncpus++;
  738                 } else
  739                         cpu_info[i].cpu_disabled = 1;
  740         }
  741         KASSERT(mp_maxid >= mp_ncpus - 1,
  742             ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid,
  743             mp_ncpus));         
  744 }
  745 
  746 /*
  747  * start each AP in our list
  748  */
  749 /* Lowest 1MB is already mapped: don't touch*/
  750 #define TMPMAP_START 1
  751 int
  752 start_all_aps(void)
  753 {
  754         int x,apic_id, cpu;
  755         struct pcpu *pc;
  756         
  757         mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
  758 
  759         /* set up temporary P==V mapping for AP boot */
  760         /* XXX this is a hack, we should boot the AP on its own stack/PTD */
  761 
  762         /* start each AP */
  763         for (cpu = 1; cpu < mp_ncpus; cpu++) {
  764                 apic_id = cpu_apic_ids[cpu];
  765 
  766 
  767                 bootAP = cpu;
  768                 bootAPgdt = gdt + (512*cpu);
  769 
  770                 /* Get per-cpu data */
  771                 pc = &__pcpu[bootAP];
  772                 pcpu_init(pc, bootAP, sizeof(struct pcpu));
  773                 dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
  774                     M_WAITOK | M_ZERO), bootAP);
  775                 pc->pc_apic_id = cpu_apic_ids[bootAP];
  776                 pc->pc_vcpu_id = cpu_apic_ids[bootAP];
  777                 pc->pc_prvspace = pc;
  778                 pc->pc_curthread = 0;
  779 
  780                 gdt_segs[GPRIV_SEL].ssd_base = (int) pc;
  781                 gdt_segs[GPROC0_SEL].ssd_base = (int) &pc->pc_common_tss;
  782                 
  783                 PT_SET_MA(bootAPgdt, VTOM(bootAPgdt) | PG_V | PG_RW);
  784                 bzero(bootAPgdt, PAGE_SIZE);
  785                 for (x = 0; x < NGDT; x++)
  786                         ssdtosd(&gdt_segs[x], &bootAPgdt[x].sd);
  787                 PT_SET_MA(bootAPgdt, vtomach(bootAPgdt) | PG_V);
  788 #ifdef notyet
  789                 
  790                 if (HYPERVISOR_vcpu_op(VCPUOP_get_physid, cpu, &cpu_id) == 0) { 
  791                         apicid = xen_vcpu_physid_to_x86_apicid(cpu_id.phys_id); 
  792                         acpiid = xen_vcpu_physid_to_x86_acpiid(cpu_id.phys_id); 
  793 #ifdef CONFIG_ACPI 
  794                         if (acpiid != 0xff) 
  795                                 x86_acpiid_to_apicid[acpiid] = apicid; 
  796 #endif 
  797                 } 
  798 #endif
  799                 
  800                 /* attempt to start the Application Processor */
  801                 if (!start_ap(cpu)) {
  802                         printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id);
  803                         /* better panic as the AP may be running loose */
  804                         printf("panic y/n? [y] ");
  805                         if (cngetc() != 'n')
  806                                 panic("bye-bye");
  807                 }
  808 
  809                 CPU_SET(cpu, &all_cpus);        /* record AP in CPU map */
  810         }
  811         
  812 
  813         pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
  814         
  815         /* number of APs actually started */
  816         return (mp_naps);
  817 }
  818 
  819 extern uint8_t *pcpu_boot_stack;
  820 extern trap_info_t trap_table[];
  821 
  822 static void
  823 smp_trap_init(trap_info_t *trap_ctxt)
  824 {
  825         const trap_info_t *t = trap_table;
  826 
  827         for (t = trap_table; t->address; t++) {
  828                 trap_ctxt[t->vector].flags = t->flags;
  829                 trap_ctxt[t->vector].cs = t->cs;
  830                 trap_ctxt[t->vector].address = t->address;
  831         }
  832 }
  833 
  834 extern struct rwlock pvh_global_lock;
  835 extern int nkpt;
  836 static void
  837 cpu_initialize_context(unsigned int cpu)
  838 {
  839         /* vcpu_guest_context_t is too large to allocate on the stack.
  840          * Hence we allocate statically and protect it with a lock */
  841         vm_page_t m[NPGPTD + 2];
  842         static vcpu_guest_context_t ctxt;
  843         vm_offset_t boot_stack;
  844         vm_offset_t newPTD;
  845         vm_paddr_t ma[NPGPTD];
  846         int i;
  847 
  848         /*
  849          * Page 0,[0-3] PTD
  850          * Page 1, [4]  boot stack
  851          * Page [5]     PDPT
  852          *
  853          */
  854         for (i = 0; i < NPGPTD + 2; i++) {
  855                 m[i] = vm_page_alloc(NULL, 0,
  856                     VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
  857                     VM_ALLOC_ZERO);
  858 
  859                 pmap_zero_page(m[i]);
  860 
  861         }
  862         boot_stack = kva_alloc(PAGE_SIZE);
  863         newPTD = kva_alloc(NPGPTD * PAGE_SIZE);
  864         ma[0] = VM_PAGE_TO_MACH(m[0])|PG_V;
  865 
  866 #ifdef PAE      
  867         pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD + 1]));
  868         for (i = 0; i < NPGPTD; i++) {
  869                 ((vm_paddr_t *)boot_stack)[i] =
  870                 ma[i] = VM_PAGE_TO_MACH(m[i])|PG_V;
  871         }
  872 #endif  
  873 
  874         /*
  875          * Copy cpu0 IdlePTD to new IdlePTD - copying only
  876          * kernel mappings
  877          */
  878         pmap_qenter(newPTD, m, 4);
  879         
  880         memcpy((uint8_t *)newPTD + KPTDI*sizeof(vm_paddr_t),
  881             (uint8_t *)PTOV(IdlePTD) + KPTDI*sizeof(vm_paddr_t),
  882             nkpt*sizeof(vm_paddr_t));
  883 
  884         pmap_qremove(newPTD, 4);
  885         kva_free(newPTD, 4 * PAGE_SIZE);
  886         /*
  887          * map actual idle stack to boot_stack
  888          */
  889         pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD]));
  890 
  891 
  892         xen_pgdpt_pin(VM_PAGE_TO_MACH(m[NPGPTD + 1]));
  893         rw_wlock(&pvh_global_lock);
  894         for (i = 0; i < 4; i++) {
  895                 int pdir = (PTDPTDI + i) / NPDEPG;
  896                 int curoffset = (PTDPTDI + i) % NPDEPG;
  897                 
  898                 xen_queue_pt_update((vm_paddr_t)
  899                     ((ma[pdir] & ~PG_V) + (curoffset*sizeof(vm_paddr_t))), 
  900                     ma[i]);
  901         }
  902         PT_UPDATES_FLUSH();
  903         rw_wunlock(&pvh_global_lock);
  904         
  905         memset(&ctxt, 0, sizeof(ctxt));
  906         ctxt.flags = VGCF_IN_KERNEL;
  907         ctxt.user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
  908         ctxt.user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
  909         ctxt.user_regs.fs = GSEL(GPRIV_SEL, SEL_KPL);
  910         ctxt.user_regs.gs = GSEL(GDATA_SEL, SEL_KPL);
  911         ctxt.user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
  912         ctxt.user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
  913         ctxt.user_regs.eip = (unsigned long)init_secondary;
  914         ctxt.user_regs.eflags = PSL_KERNEL | 0x1000; /* IOPL_RING1 */
  915 
  916         memset(&ctxt.fpu_ctxt, 0, sizeof(ctxt.fpu_ctxt));
  917 
  918         smp_trap_init(ctxt.trap_ctxt);
  919 
  920         ctxt.ldt_ents = 0;
  921         ctxt.gdt_frames[0] =
  922             (uint32_t)((uint64_t)vtomach(bootAPgdt) >> PAGE_SHIFT);
  923         ctxt.gdt_ents      = 512;
  924 
  925 #ifdef __i386__
  926         ctxt.user_regs.esp = boot_stack + PAGE_SIZE;
  927 
  928         ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
  929         ctxt.kernel_sp = boot_stack + PAGE_SIZE;
  930 
  931         ctxt.event_callback_cs     = GSEL(GCODE_SEL, SEL_KPL);
  932         ctxt.event_callback_eip    = (unsigned long)Xhypervisor_callback;
  933         ctxt.failsafe_callback_cs  = GSEL(GCODE_SEL, SEL_KPL);
  934         ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
  935 
  936         ctxt.ctrlreg[3] = VM_PAGE_TO_MACH(m[NPGPTD + 1]);
  937 #else /* __x86_64__ */
  938         ctxt.user_regs.esp = idle->thread.rsp0 - sizeof(struct pt_regs);
  939         ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
  940         ctxt.kernel_sp = idle->thread.rsp0;
  941 
  942         ctxt.event_callback_eip    = (unsigned long)hypervisor_callback;
  943         ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
  944         ctxt.syscall_callback_eip  = (unsigned long)system_call;
  945 
  946         ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(init_level4_pgt));
  947 
  948         ctxt.gs_base_kernel = (unsigned long)(cpu_pda(cpu));
  949 #endif
  950 
  951         printf("gdtpfn=%lx pdptpfn=%lx\n",
  952             ctxt.gdt_frames[0],
  953             ctxt.ctrlreg[3] >> PAGE_SHIFT);
  954 
  955         PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, &ctxt));
  956         DELAY(3000);
  957         PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL));
  958 }
  959 
  960 /*
  961  * This function starts the AP (application processor) identified
  962  * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
  963  * to accomplish this.  This is necessary because of the nuances
  964  * of the different hardware we might encounter.  It isn't pretty,
  965  * but it seems to work.
  966  */
  967 
  968 int cpus;
  969 static int
  970 start_ap(int apic_id)
  971 {
  972         int ms;
  973 
  974         /* used as a watchpoint to signal AP startup */
  975         cpus = mp_naps;
  976 
  977         cpu_initialize_context(apic_id);
  978         
  979         /* Wait up to 5 seconds for it to start. */
  980         for (ms = 0; ms < 5000; ms++) {
  981                 if (mp_naps > cpus)
  982                         return (1);     /* return SUCCESS */
  983                 DELAY(1000);
  984         }
  985         return (0);             /* return FAILURE */
  986 }
  987 
  988 static void
  989 ipi_pcpu(int cpu, u_int ipi)
  990 {
  991         KASSERT((ipi <= nitems(xen_ipis)), ("invalid IPI"));
  992         xen_intr_signal(DPCPU_ID_GET(cpu, ipi_handle[ipi]));
  993 }
  994 
  995 /*
  996  * send an IPI to a specific CPU.
  997  */
  998 static void
  999 ipi_send_cpu(int cpu, u_int ipi)
 1000 {
 1001         u_int bitmap, old_pending, new_pending;
 1002 
 1003         if (IPI_IS_BITMAPED(ipi)) { 
 1004                 bitmap = 1 << ipi;
 1005                 ipi = IPI_BITMAP_VECTOR;
 1006                 do {
 1007                         old_pending = cpu_ipi_pending[cpu];
 1008                         new_pending = old_pending | bitmap;
 1009                 } while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
 1010                     old_pending, new_pending)); 
 1011                 if (!old_pending)
 1012                         ipi_pcpu(cpu, RESCHEDULE_VECTOR);
 1013         } else {
 1014                 KASSERT(call_data != NULL, ("call_data not set"));
 1015                 ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
 1016         }
 1017 }
 1018 
 1019 /*
 1020  * Flush the TLB on all other CPU's
 1021  */
 1022 static void
 1023 smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
 1024 {
 1025         u_int ncpu;
 1026         struct _call_data data;
 1027 
 1028         ncpu = mp_ncpus - 1;    /* does not shootdown self */
 1029         if (ncpu < 1)
 1030                 return;         /* no other cpus */
 1031         if (!(read_eflags() & PSL_I))
 1032                 panic("%s: interrupts disabled", __func__);
 1033         mtx_lock_spin(&smp_ipi_mtx);
 1034         KASSERT(call_data == NULL, ("call_data isn't null?!"));
 1035         call_data = &data;
 1036         call_data->func_id = vector;
 1037         call_data->arg1 = addr1;
 1038         call_data->arg2 = addr2;
 1039         atomic_store_rel_int(&smp_tlb_wait, 0);
 1040         ipi_all_but_self(vector);
 1041         while (smp_tlb_wait < ncpu)
 1042                 ia32_pause();
 1043         call_data = NULL;
 1044         mtx_unlock_spin(&smp_ipi_mtx);
 1045 }
 1046 
 1047 static void
 1048 smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1,
 1049     vm_offset_t addr2)
 1050 {
 1051         int cpu, ncpu, othercpus;
 1052         struct _call_data data;
 1053 
 1054         othercpus = mp_ncpus - 1;
 1055         if (CPU_ISFULLSET(&mask)) {
 1056                 if (othercpus < 1)
 1057                         return;
 1058         } else {
 1059                 CPU_CLR(PCPU_GET(cpuid), &mask);
 1060                 if (CPU_EMPTY(&mask))
 1061                         return;
 1062         }
 1063         if (!(read_eflags() & PSL_I))
 1064                 panic("%s: interrupts disabled", __func__);
 1065         mtx_lock_spin(&smp_ipi_mtx);
 1066         KASSERT(call_data == NULL, ("call_data isn't null?!"));
 1067         call_data = &data;              
 1068         call_data->func_id = vector;
 1069         call_data->arg1 = addr1;
 1070         call_data->arg2 = addr2;
 1071         atomic_store_rel_int(&smp_tlb_wait, 0);
 1072         if (CPU_ISFULLSET(&mask)) {
 1073                 ncpu = othercpus;
 1074                 ipi_all_but_self(vector);
 1075         } else {
 1076                 ncpu = 0;
 1077                 while ((cpu = CPU_FFS(&mask)) != 0) {
 1078                         cpu--;
 1079                         CPU_CLR(cpu, &mask);
 1080                         CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu,
 1081                             vector);
 1082                         ipi_send_cpu(cpu, vector);
 1083                         ncpu++;
 1084                 }
 1085         }
 1086         while (smp_tlb_wait < ncpu)
 1087                 ia32_pause();
 1088         call_data = NULL;
 1089         mtx_unlock_spin(&smp_ipi_mtx);
 1090 }
 1091 
 1092 void
 1093 smp_cache_flush(void)
 1094 {
 1095 
 1096         if (smp_started)
 1097                 smp_tlb_shootdown(IPI_INVLCACHE, 0, 0);
 1098 }
 1099 
 1100 void
 1101 smp_invltlb(void)
 1102 {
 1103 
 1104         if (smp_started) {
 1105                 smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
 1106         }
 1107 }
 1108 
 1109 void
 1110 smp_invlpg(vm_offset_t addr)
 1111 {
 1112 
 1113         if (smp_started) {
 1114                 smp_tlb_shootdown(IPI_INVLPG, addr, 0);
 1115         }
 1116 }
 1117 
 1118 void
 1119 smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2)
 1120 {
 1121 
 1122         if (smp_started) {
 1123                 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
 1124         }
 1125 }
 1126 
 1127 void
 1128 smp_masked_invltlb(cpuset_t mask)
 1129 {
 1130 
 1131         if (smp_started) {
 1132                 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
 1133         }
 1134 }
 1135 
 1136 void
 1137 smp_masked_invlpg(cpuset_t mask, vm_offset_t addr)
 1138 {
 1139 
 1140         if (smp_started) {
 1141                 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
 1142         }
 1143 }
 1144 
 1145 void
 1146 smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2)
 1147 {
 1148 
 1149         if (smp_started) {
 1150                 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
 1151         }
 1152 }
 1153 
 1154 /*
 1155  * send an IPI to a set of cpus.
 1156  */
 1157 void
 1158 ipi_selected(cpuset_t cpus, u_int ipi)
 1159 {
 1160         int cpu;
 1161 
 1162         /*
 1163          * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
 1164          * of help in order to understand what is the source.
 1165          * Set the mask of receiving CPUs for this purpose.
 1166          */
 1167         if (ipi == IPI_STOP_HARD)
 1168                 CPU_OR_ATOMIC(&ipi_nmi_pending, &cpus);
 1169 
 1170         while ((cpu = CPU_FFS(&cpus)) != 0) {
 1171                 cpu--;
 1172                 CPU_CLR(cpu, &cpus);
 1173                 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
 1174                 ipi_send_cpu(cpu, ipi);
 1175         }
 1176 }
 1177 
 1178 /*
 1179  * send an IPI to a specific CPU.
 1180  */
 1181 void
 1182 ipi_cpu(int cpu, u_int ipi)
 1183 {
 1184 
 1185         /*
 1186          * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
 1187          * of help in order to understand what is the source.
 1188          * Set the mask of receiving CPUs for this purpose.
 1189          */
 1190         if (ipi == IPI_STOP_HARD)
 1191                 CPU_SET_ATOMIC(cpu, &ipi_nmi_pending);
 1192 
 1193         CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
 1194         ipi_send_cpu(cpu, ipi);
 1195 }
 1196 
 1197 /*
 1198  * send an IPI to all CPUs EXCEPT myself
 1199  */
 1200 void
 1201 ipi_all_but_self(u_int ipi)
 1202 {
 1203         cpuset_t other_cpus;
 1204 
 1205         /*
 1206          * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
 1207          * of help in order to understand what is the source.
 1208          * Set the mask of receiving CPUs for this purpose.
 1209          */
 1210         other_cpus = all_cpus;
 1211         CPU_CLR(PCPU_GET(cpuid), &other_cpus);
 1212         if (ipi == IPI_STOP_HARD)
 1213                 CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus);
 1214 
 1215         CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 1216         ipi_selected(other_cpus, ipi);
 1217 }
 1218 
 1219 int
 1220 ipi_nmi_handler()
 1221 {
 1222         u_int cpuid;
 1223 
 1224         /*
 1225          * As long as there is not a simple way to know about a NMI's
 1226          * source, if the bitmask for the current CPU is present in
 1227          * the global pending bitword an IPI_STOP_HARD has been issued
 1228          * and should be handled.
 1229          */
 1230         cpuid = PCPU_GET(cpuid);
 1231         if (!CPU_ISSET(cpuid, &ipi_nmi_pending))
 1232                 return (1);
 1233 
 1234         CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending);
 1235         cpustop_handler();
 1236         return (0);
 1237 }
 1238 
 1239 /*
 1240  * Handle an IPI_STOP by saving our current context and spinning until we
 1241  * are resumed.
 1242  */
 1243 void
 1244 cpustop_handler(void)
 1245 {
 1246         int cpu;
 1247 
 1248         cpu = PCPU_GET(cpuid);
 1249 
 1250         savectx(&stoppcbs[cpu]);
 1251 
 1252         /* Indicate that we are stopped */
 1253         CPU_SET_ATOMIC(cpu, &stopped_cpus);
 1254 
 1255         /* Wait for restart */
 1256         while (!CPU_ISSET(cpu, &started_cpus))
 1257             ia32_pause();
 1258 
 1259         CPU_CLR_ATOMIC(cpu, &started_cpus);
 1260         CPU_CLR_ATOMIC(cpu, &stopped_cpus);
 1261 
 1262         if (cpu == 0 && cpustop_restartfunc != NULL) {
 1263                 cpustop_restartfunc();
 1264                 cpustop_restartfunc = NULL;
 1265         }
 1266 }
 1267 
 1268 /*
 1269  * Handlers for TLB related IPIs
 1270  *
 1271  * On i386 Xen PV this are no-ops since this port doesn't support SMP.
 1272  */
 1273 void
 1274 invltlb_handler(void)
 1275 {
 1276 }
 1277 
 1278 void
 1279 invlpg_handler(void)
 1280 {
 1281 }
 1282 
 1283 void
 1284 invlrng_handler(void)
 1285 {
 1286 }
 1287 
 1288 void
 1289 invlcache_handler(void)
 1290 {
 1291 }
 1292 
 1293 /*
 1294  * This is called once the rest of the system is up and running and we're
 1295  * ready to let the AP's out of the pen.
 1296  */
 1297 static void
 1298 release_aps(void *dummy __unused)
 1299 {
 1300 
 1301         if (mp_ncpus == 1) 
 1302                 return;
 1303         atomic_store_rel_int(&aps_ready, 1);
 1304         while (smp_started == 0)
 1305                 ia32_pause();
 1306 }
 1307 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
 1308 SYSINIT(start_ipis, SI_SUB_SMP, SI_ORDER_ANY, xen_smp_intr_init_cpus, NULL);
 1309 SYSINIT(start_cpu, SI_SUB_INTR, SI_ORDER_ANY, xen_smp_intr_setup_cpus, NULL);

Cache object: 7f9433d4d2b9d922e92006ba747dcc8a


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.