The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_smp.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2001, John Baldwin <jhb@FreeBSD.org>.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 3. Neither the name of the author nor the names of any co-contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 /*
   31  * This module holds the global variables and machine independent functions
   32  * used for the kernel SMP support.
   33  */
   34 
   35 #include <sys/cdefs.h>
   36 __FBSDID("$FreeBSD: releng/8.3/sys/kern/subr_smp.c 227891 2011-11-23 16:02:36Z attilio $");
   37 
   38 #include <sys/param.h>
   39 #include <sys/systm.h>
   40 #include <sys/kernel.h>
   41 #include <sys/ktr.h>
   42 #include <sys/proc.h>
   43 #include <sys/bus.h>
   44 #include <sys/lock.h>
   45 #include <sys/mutex.h>
   46 #include <sys/pcpu.h>
   47 #include <sys/smp.h>
   48 #include <sys/sysctl.h>
   49 
   50 #include <machine/cpu.h>
   51 #include <machine/smp.h>
   52 
   53 #include "opt_sched.h"
   54 
   55 #ifdef SMP
   56 volatile cpumask_t stopped_cpus;
   57 volatile cpumask_t started_cpus;
   58 cpumask_t idle_cpus_mask;
   59 cpumask_t hlt_cpus_mask;
   60 cpumask_t logical_cpus_mask;
   61 
   62 void (*cpustop_restartfunc)(void);
   63 #endif
   64 /* This is used in modules that need to work in both SMP and UP. */
   65 cpumask_t all_cpus;
   66 
   67 int mp_ncpus;
   68 /* export this for libkvm consumers. */
   69 int mp_maxcpus = MAXCPU;
   70 
   71 volatile int smp_started;
   72 u_int mp_maxid;
   73 
   74 SYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD, NULL, "Kernel SMP");
   75 
   76 SYSCTL_INT(_kern_smp, OID_AUTO, maxid, CTLFLAG_RD, &mp_maxid, 0,
   77     "Max CPU ID.");
   78 
   79 SYSCTL_INT(_kern_smp, OID_AUTO, maxcpus, CTLFLAG_RD, &mp_maxcpus, 0,
   80     "Max number of CPUs that the system was compiled for.");
   81 
   82 int smp_active = 0;     /* are the APs allowed to run? */
   83 SYSCTL_INT(_kern_smp, OID_AUTO, active, CTLFLAG_RW, &smp_active, 0,
   84     "Number of Auxillary Processors (APs) that were successfully started");
   85 
   86 int smp_disabled = 0;   /* has smp been disabled? */
   87 SYSCTL_INT(_kern_smp, OID_AUTO, disabled, CTLFLAG_RDTUN, &smp_disabled, 0,
   88     "SMP has been disabled from the loader");
   89 TUNABLE_INT("kern.smp.disabled", &smp_disabled);
   90 
   91 int smp_cpus = 1;       /* how many cpu's running */
   92 SYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD, &smp_cpus, 0,
   93     "Number of CPUs online");
   94 
   95 int smp_topology = 0;   /* Which topology we're using. */
   96 SYSCTL_INT(_kern_smp, OID_AUTO, topology, CTLFLAG_RD, &smp_topology, 0,
   97     "Topology override setting; 0 is default provided by hardware.");
   98 TUNABLE_INT("kern.smp.topology", &smp_topology);
   99 
  100 #ifdef SMP
  101 /* Enable forwarding of a signal to a process running on a different CPU */
  102 static int forward_signal_enabled = 1;
  103 SYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
  104            &forward_signal_enabled, 0,
  105            "Forwarding of a signal to a process on a different CPU");
  106 
  107 /* Variables needed for SMP rendezvous. */
  108 static volatile int smp_rv_ncpus;
  109 static void (*volatile smp_rv_setup_func)(void *arg);
  110 static void (*volatile smp_rv_action_func)(void *arg);
  111 static void (*volatile smp_rv_teardown_func)(void *arg);
  112 static void *volatile smp_rv_func_arg;
  113 static volatile int smp_rv_waiters[4];
  114 
  115 /* 
  116  * Shared mutex to restrict busywaits between smp_rendezvous() and
  117  * smp(_targeted)_tlb_shootdown().  A deadlock occurs if both of these
  118  * functions trigger at once and cause multiple CPUs to busywait with
  119  * interrupts disabled. 
  120  */
  121 struct mtx smp_ipi_mtx;
  122 
  123 /*
  124  * Let the MD SMP code initialize mp_maxid very early if it can.
  125  */
  126 static void
  127 mp_setmaxid(void *dummy)
  128 {
  129         cpu_mp_setmaxid();
  130 }
  131 SYSINIT(cpu_mp_setmaxid, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_setmaxid, NULL);
  132 
  133 /*
  134  * Call the MD SMP initialization code.
  135  */
  136 static void
  137 mp_start(void *dummy)
  138 {
  139 
  140         mtx_init(&smp_ipi_mtx, "smp rendezvous", NULL, MTX_SPIN);
  141 
  142         /* Probe for MP hardware. */
  143         if (smp_disabled != 0 || cpu_mp_probe() == 0) {
  144                 mp_ncpus = 1;
  145                 all_cpus = PCPU_GET(cpumask);
  146                 return;
  147         }
  148 
  149         cpu_mp_start();
  150         printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n",
  151             mp_ncpus);
  152         cpu_mp_announce();
  153 }
  154 SYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_THIRD, mp_start, NULL);
  155 
  156 void
  157 forward_signal(struct thread *td)
  158 {
  159         int id;
  160 
  161         /*
  162          * signotify() has already set TDF_ASTPENDING and TDF_NEEDSIGCHECK on
  163          * this thread, so all we need to do is poke it if it is currently
  164          * executing so that it executes ast().
  165          */
  166         THREAD_LOCK_ASSERT(td, MA_OWNED);
  167         KASSERT(TD_IS_RUNNING(td),
  168             ("forward_signal: thread is not TDS_RUNNING"));
  169 
  170         CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc);
  171 
  172         if (!smp_started || cold || panicstr)
  173                 return;
  174         if (!forward_signal_enabled)
  175                 return;
  176 
  177         /* No need to IPI ourself. */
  178         if (td == curthread)
  179                 return;
  180 
  181         id = td->td_oncpu;
  182         if (id == NOCPU)
  183                 return;
  184         ipi_cpu(id, IPI_AST);
  185 }
  186 
  187 /*
  188  * When called the executing CPU will send an IPI to all other CPUs
  189  *  requesting that they halt execution.
  190  *
  191  * Usually (but not necessarily) called with 'other_cpus' as its arg.
  192  *
  193  *  - Signals all CPUs in map to stop.
  194  *  - Waits for each to stop.
  195  *
  196  * Returns:
  197  *  -1: error
  198  *   0: NA
  199  *   1: ok
  200  *
  201  */
  202 static int
  203 generic_stop_cpus(cpumask_t map, u_int type)
  204 {
  205         static volatile u_int stopping_cpu = NOCPU;
  206         int i;
  207 
  208         KASSERT(
  209 #if defined(__amd64__)
  210             type == IPI_STOP || type == IPI_STOP_HARD || type == IPI_SUSPEND,
  211 #else
  212             type == IPI_STOP || type == IPI_STOP_HARD,
  213 #endif
  214             ("%s: invalid stop type", __func__));
  215 
  216         if (!smp_started)
  217                 return (0);
  218 
  219         CTR2(KTR_SMP, "stop_cpus(%x) with %u type", map, type);
  220 
  221         if (stopping_cpu != PCPU_GET(cpuid))
  222                 while (atomic_cmpset_int(&stopping_cpu, NOCPU,
  223                     PCPU_GET(cpuid)) == 0)
  224                         while (stopping_cpu != NOCPU)
  225                                 cpu_spinwait(); /* spin */
  226 
  227         /* send the stop IPI to all CPUs in map */
  228         ipi_selected(map, type);
  229 
  230         i = 0;
  231         while ((stopped_cpus & map) != map) {
  232                 /* spin */
  233                 cpu_spinwait();
  234                 i++;
  235                 if (i == 100000000) {
  236                         printf("timeout stopping cpus\n");
  237                         break;
  238                 }
  239         }
  240 
  241         stopping_cpu = NOCPU;
  242         return (1);
  243 }
  244 
  245 int
  246 stop_cpus(cpumask_t map)
  247 {
  248 
  249         return (generic_stop_cpus(map, IPI_STOP));
  250 }
  251 
  252 int
  253 stop_cpus_hard(cpumask_t map)
  254 {
  255 
  256         return (generic_stop_cpus(map, IPI_STOP_HARD));
  257 }
  258 
  259 #if defined(__amd64__)
  260 int
  261 suspend_cpus(cpumask_t map)
  262 {
  263 
  264         return (generic_stop_cpus(map, IPI_SUSPEND));
  265 }
  266 #endif
  267 
  268 /*
  269  * Called by a CPU to restart stopped CPUs. 
  270  *
  271  * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
  272  *
  273  *  - Signals all CPUs in map to restart.
  274  *  - Waits for each to restart.
  275  *
  276  * Returns:
  277  *  -1: error
  278  *   0: NA
  279  *   1: ok
  280  */
  281 int
  282 restart_cpus(cpumask_t map)
  283 {
  284 
  285         if (!smp_started)
  286                 return 0;
  287 
  288         CTR1(KTR_SMP, "restart_cpus(%x)", map);
  289 
  290         /* signal other cpus to restart */
  291         atomic_store_rel_int(&started_cpus, map);
  292 
  293         /* wait for each to clear its bit */
  294         while ((stopped_cpus & map) != 0)
  295                 cpu_spinwait();
  296 
  297         return 1;
  298 }
  299 
  300 /*
  301  * All-CPU rendezvous.  CPUs are signalled, all execute the setup function 
  302  * (if specified), rendezvous, execute the action function (if specified),
  303  * rendezvous again, execute the teardown function (if specified), and then
  304  * resume.
  305  *
  306  * Note that the supplied external functions _must_ be reentrant and aware
  307  * that they are running in parallel and in an unknown lock context.
  308  */
  309 void
  310 smp_rendezvous_action(void)
  311 {
  312         struct thread *td;
  313         void *local_func_arg;
  314         void (*local_setup_func)(void*);
  315         void (*local_action_func)(void*);
  316         void (*local_teardown_func)(void*);
  317 #ifdef INVARIANTS
  318         int owepreempt;
  319 #endif
  320 
  321         /* Ensure we have up-to-date values. */
  322         atomic_add_acq_int(&smp_rv_waiters[0], 1);
  323         while (smp_rv_waiters[0] < smp_rv_ncpus)
  324                 cpu_spinwait();
  325 
  326         /* Fetch rendezvous parameters after acquire barrier. */
  327         local_func_arg = smp_rv_func_arg;
  328         local_setup_func = smp_rv_setup_func;
  329         local_action_func = smp_rv_action_func;
  330         local_teardown_func = smp_rv_teardown_func;
  331 
  332         /*
  333          * Use a nested critical section to prevent any preemptions
  334          * from occurring during a rendezvous action routine.
  335          * Specifically, if a rendezvous handler is invoked via an IPI
  336          * and the interrupted thread was in the critical_exit()
  337          * function after setting td_critnest to 0 but before
  338          * performing a deferred preemption, this routine can be
  339          * invoked with td_critnest set to 0 and td_owepreempt true.
  340          * In that case, a critical_exit() during the rendezvous
  341          * action would trigger a preemption which is not permitted in
  342          * a rendezvous action.  To fix this, wrap all of the
  343          * rendezvous action handlers in a critical section.  We
  344          * cannot use a regular critical section however as having
  345          * critical_exit() preempt from this routine would also be
  346          * problematic (the preemption must not occur before the IPI
  347          * has been acknowleged via an EOI).  Instead, we
  348          * intentionally ignore td_owepreempt when leaving the
  349          * critical setion.  This should be harmless because we do not
  350          * permit rendezvous action routines to schedule threads, and
  351          * thus td_owepreempt should never transition from 0 to 1
  352          * during this routine.
  353          */
  354         td = curthread;
  355         td->td_critnest++;
  356 #ifdef INVARIANTS
  357         owepreempt = td->td_owepreempt;
  358 #endif
  359         
  360         /*
  361          * If requested, run a setup function before the main action
  362          * function.  Ensure all CPUs have completed the setup
  363          * function before moving on to the action function.
  364          */
  365         if (local_setup_func != smp_no_rendevous_barrier) {
  366                 if (smp_rv_setup_func != NULL)
  367                         smp_rv_setup_func(smp_rv_func_arg);
  368                 atomic_add_int(&smp_rv_waiters[1], 1);
  369                 while (smp_rv_waiters[1] < smp_rv_ncpus)
  370                         cpu_spinwait();
  371         }
  372 
  373         if (local_action_func != NULL)
  374                 local_action_func(local_func_arg);
  375 
  376         if (local_teardown_func != smp_no_rendevous_barrier) {
  377                 /*
  378                  * Signal that the main action has been completed.  If a
  379                  * full exit rendezvous is requested, then all CPUs will
  380                  * wait here until all CPUs have finished the main action.
  381                  */
  382                 atomic_add_int(&smp_rv_waiters[2], 1);
  383                 while (smp_rv_waiters[2] < smp_rv_ncpus)
  384                         cpu_spinwait();
  385 
  386                 if (local_teardown_func != NULL)
  387                         local_teardown_func(local_func_arg);
  388         }
  389 
  390         /*
  391          * Signal that the rendezvous is fully completed by this CPU.
  392          * This means that no member of smp_rv_* pseudo-structure will be
  393          * accessed by this target CPU after this point; in particular,
  394          * memory pointed by smp_rv_func_arg.
  395          */
  396         atomic_add_int(&smp_rv_waiters[3], 1);
  397 
  398         td->td_critnest--;
  399         KASSERT(owepreempt == td->td_owepreempt,
  400             ("rendezvous action changed td_owepreempt"));
  401 }
  402 
  403 void
  404 smp_rendezvous_cpus(cpumask_t map,
  405         void (* setup_func)(void *), 
  406         void (* action_func)(void *),
  407         void (* teardown_func)(void *),
  408         void *arg)
  409 {
  410         int i, ncpus = 0;
  411 
  412         /* Look comments in the !SMP case. */
  413         if (!smp_started) {
  414                 spinlock_enter();
  415                 if (setup_func != NULL)
  416                         setup_func(arg);
  417                 if (action_func != NULL)
  418                         action_func(arg);
  419                 if (teardown_func != NULL)
  420                         teardown_func(arg);
  421                 spinlock_exit();
  422                 return;
  423         }
  424 
  425         CPU_FOREACH(i) {
  426                 if (((1 << i) & map) != 0)
  427                         ncpus++;
  428         }
  429         if (ncpus == 0)
  430                 panic("ncpus is 0 with map=0x%x", map);
  431 
  432         mtx_lock_spin(&smp_ipi_mtx);
  433 
  434         /* Pass rendezvous parameters via global variables. */
  435         smp_rv_ncpus = ncpus;
  436         smp_rv_setup_func = setup_func;
  437         smp_rv_action_func = action_func;
  438         smp_rv_teardown_func = teardown_func;
  439         smp_rv_func_arg = arg;
  440         smp_rv_waiters[1] = 0;
  441         smp_rv_waiters[2] = 0;
  442         smp_rv_waiters[3] = 0;
  443         atomic_store_rel_int(&smp_rv_waiters[0], 0);
  444 
  445         /*
  446          * Signal other processors, which will enter the IPI with
  447          * interrupts off.
  448          */
  449         ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS);
  450 
  451         /* Check if the current CPU is in the map */
  452         if ((map & (1 << curcpu)) != 0)
  453                 smp_rendezvous_action();
  454 
  455         /*
  456          * Ensure that the master CPU waits for all the other
  457          * CPUs to finish the rendezvous, so that smp_rv_*
  458          * pseudo-structure and the arg are guaranteed to not
  459          * be in use.
  460          */
  461         while (atomic_load_acq_int(&smp_rv_waiters[3]) < ncpus)
  462                 cpu_spinwait();
  463 
  464         mtx_unlock_spin(&smp_ipi_mtx);
  465 }
  466 
  467 void
  468 smp_rendezvous(void (* setup_func)(void *), 
  469                void (* action_func)(void *),
  470                void (* teardown_func)(void *),
  471                void *arg)
  472 {
  473         smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func, arg);
  474 }
  475 
  476 static struct cpu_group group[MAXCPU];
  477 
  478 struct cpu_group *
  479 smp_topo(void)
  480 {
  481         struct cpu_group *top;
  482 
  483         /*
  484          * Check for a fake topology request for debugging purposes.
  485          */
  486         switch (smp_topology) {
  487         case 1:
  488                 /* Dual core with no sharing.  */
  489                 top = smp_topo_1level(CG_SHARE_NONE, 2, 0);
  490                 break;
  491         case 2:
  492                 /* No topology, all cpus are equal. */
  493                 top = smp_topo_none();
  494                 break;
  495         case 3:
  496                 /* Dual core with shared L2.  */
  497                 top = smp_topo_1level(CG_SHARE_L2, 2, 0);
  498                 break;
  499         case 4:
  500                 /* quad core, shared l3 among each package, private l2.  */
  501                 top = smp_topo_1level(CG_SHARE_L3, 4, 0);
  502                 break;
  503         case 5:
  504                 /* quad core,  2 dualcore parts on each package share l2.  */
  505                 top = smp_topo_2level(CG_SHARE_NONE, 2, CG_SHARE_L2, 2, 0);
  506                 break;
  507         case 6:
  508                 /* Single-core 2xHTT */
  509                 top = smp_topo_1level(CG_SHARE_L1, 2, CG_FLAG_HTT);
  510                 break;
  511         case 7:
  512                 /* quad core with a shared l3, 8 threads sharing L2.  */
  513                 top = smp_topo_2level(CG_SHARE_L3, 4, CG_SHARE_L2, 8,
  514                     CG_FLAG_SMT);
  515                 break;
  516         default:
  517                 /* Default, ask the system what it wants. */
  518                 top = cpu_topo();
  519                 break;
  520         }
  521         /*
  522          * Verify the returned topology.
  523          */
  524         if (top->cg_count != mp_ncpus)
  525                 panic("Built bad topology at %p.  CPU count %d != %d",
  526                     top, top->cg_count, mp_ncpus);
  527         if (top->cg_mask != all_cpus)
  528                 panic("Built bad topology at %p.  CPU mask 0x%X != 0x%X",
  529                     top, top->cg_mask, all_cpus);
  530         return (top);
  531 }
  532 
  533 struct cpu_group *
  534 smp_topo_none(void)
  535 {
  536         struct cpu_group *top;
  537 
  538         top = &group[0];
  539         top->cg_parent = NULL;
  540         top->cg_child = NULL;
  541         top->cg_mask = ~0U >> (32 - mp_ncpus);
  542         top->cg_count = mp_ncpus;
  543         top->cg_children = 0;
  544         top->cg_level = CG_SHARE_NONE;
  545         top->cg_flags = 0;
  546         
  547         return (top);
  548 }
  549 
  550 static int
  551 smp_topo_addleaf(struct cpu_group *parent, struct cpu_group *child, int share,
  552     int count, int flags, int start)
  553 {
  554         cpumask_t mask;
  555         int i;
  556 
  557         for (mask = 0, i = 0; i < count; i++, start++)
  558                 mask |= (1 << start);
  559         child->cg_parent = parent;
  560         child->cg_child = NULL;
  561         child->cg_children = 0;
  562         child->cg_level = share;
  563         child->cg_count = count;
  564         child->cg_flags = flags;
  565         child->cg_mask = mask;
  566         parent->cg_children++;
  567         for (; parent != NULL; parent = parent->cg_parent) {
  568                 if ((parent->cg_mask & child->cg_mask) != 0)
  569                         panic("Duplicate children in %p.  mask 0x%X child 0x%X",
  570                             parent, parent->cg_mask, child->cg_mask);
  571                 parent->cg_mask |= child->cg_mask;
  572                 parent->cg_count += child->cg_count;
  573         }
  574 
  575         return (start);
  576 }
  577 
  578 struct cpu_group *
  579 smp_topo_1level(int share, int count, int flags)
  580 {
  581         struct cpu_group *child;
  582         struct cpu_group *top;
  583         int packages;
  584         int cpu;
  585         int i;
  586 
  587         cpu = 0;
  588         top = &group[0];
  589         packages = mp_ncpus / count;
  590         top->cg_child = child = &group[1];
  591         top->cg_level = CG_SHARE_NONE;
  592         for (i = 0; i < packages; i++, child++)
  593                 cpu = smp_topo_addleaf(top, child, share, count, flags, cpu);
  594         return (top);
  595 }
  596 
  597 struct cpu_group *
  598 smp_topo_2level(int l2share, int l2count, int l1share, int l1count,
  599     int l1flags)
  600 {
  601         struct cpu_group *top;
  602         struct cpu_group *l1g;
  603         struct cpu_group *l2g;
  604         int cpu;
  605         int i;
  606         int j;
  607 
  608         cpu = 0;
  609         top = &group[0];
  610         l2g = &group[1];
  611         top->cg_child = l2g;
  612         top->cg_level = CG_SHARE_NONE;
  613         top->cg_children = mp_ncpus / (l2count * l1count);
  614         l1g = l2g + top->cg_children;
  615         for (i = 0; i < top->cg_children; i++, l2g++) {
  616                 l2g->cg_parent = top;
  617                 l2g->cg_child = l1g;
  618                 l2g->cg_level = l2share;
  619                 for (j = 0; j < l2count; j++, l1g++)
  620                         cpu = smp_topo_addleaf(l2g, l1g, l1share, l1count,
  621                             l1flags, cpu);
  622         }
  623         return (top);
  624 }
  625 
  626 
  627 struct cpu_group *
  628 smp_topo_find(struct cpu_group *top, int cpu)
  629 {
  630         struct cpu_group *cg;
  631         cpumask_t mask;
  632         int children;
  633         int i;
  634 
  635         mask = (1 << cpu);
  636         cg = top;
  637         for (;;) {
  638                 if ((cg->cg_mask & mask) == 0)
  639                         return (NULL);
  640                 if (cg->cg_children == 0)
  641                         return (cg);
  642                 children = cg->cg_children;
  643                 for (i = 0, cg = cg->cg_child; i < children; cg++, i++)
  644                         if ((cg->cg_mask & mask) != 0)
  645                                 break;
  646         }
  647         return (NULL);
  648 }
  649 #else /* !SMP */
  650 
  651 void
  652 smp_rendezvous_cpus(cpumask_t map,
  653         void (*setup_func)(void *), 
  654         void (*action_func)(void *),
  655         void (*teardown_func)(void *),
  656         void *arg)
  657 {
  658         /*
  659          * In the !SMP case we just need to ensure the same initial conditions
  660          * as the SMP case.
  661          */
  662         spinlock_enter();
  663         if (setup_func != NULL)
  664                 setup_func(arg);
  665         if (action_func != NULL)
  666                 action_func(arg);
  667         if (teardown_func != NULL)
  668                 teardown_func(arg);
  669         spinlock_exit();
  670 }
  671 
  672 void
  673 smp_rendezvous(void (*setup_func)(void *), 
  674                void (*action_func)(void *),
  675                void (*teardown_func)(void *),
  676                void *arg)
  677 {
  678 
  679         /* Look comments in the smp_rendezvous_cpus() case. */
  680         spinlock_enter();
  681         if (setup_func != NULL)
  682                 setup_func(arg);
  683         if (action_func != NULL)
  684                 action_func(arg);
  685         if (teardown_func != NULL)
  686                 teardown_func(arg);
  687         spinlock_exit();
  688 }
  689 
  690 /*
  691  * Provide dummy SMP support for UP kernels.  Modules that need to use SMP
  692  * APIs will still work using this dummy support.
  693  */
  694 static void
  695 mp_setvariables_for_up(void *dummy)
  696 {
  697         mp_ncpus = 1;
  698         mp_maxid = PCPU_GET(cpuid);
  699         all_cpus = PCPU_GET(cpumask);
  700         KASSERT(PCPU_GET(cpuid) == 0, ("UP must have a CPU ID of zero"));
  701 }
  702 SYSINIT(cpu_mp_setvariables, SI_SUB_TUNABLES, SI_ORDER_FIRST,
  703     mp_setvariables_for_up, NULL);
  704 #endif /* SMP */
  705 
  706 void
  707 smp_no_rendevous_barrier(void *dummy)
  708 {
  709 #ifdef SMP
  710         KASSERT((!smp_started),("smp_no_rendevous called and smp is started"));
  711 #endif
  712 }

Cache object: 1d6c2700c0893d5e9cb1da67c1b19297


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.