The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_smp.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2001
    3  *      John Baldwin <jhb@FreeBSD.org>.  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 4. Neither the name of the author nor the names of any co-contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY JOHN BALDWIN AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL JOHN BALDWIN OR THE VOICES IN HIS HEAD
   21  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   22  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   25  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
   27  * THE POSSIBILITY OF SUCH DAMAGE.
   28  */
   29 
   30 /*
   31  * This module holds the global variables and machine independent functions
   32  * used for the kernel SMP support.
   33  */
   34 
   35 #include <sys/cdefs.h>
   36 __FBSDID("$FreeBSD: src/sys/kern/subr_smp.c,v 1.188.2.5 2005/05/01 05:34:47 dwhite Exp $");
   37 
   38 #include <sys/param.h>
   39 #include <sys/systm.h>
   40 #include <sys/kernel.h>
   41 #include <sys/ktr.h>
   42 #include <sys/proc.h>
   43 #include <sys/bus.h>
   44 #include <sys/lock.h>
   45 #include <sys/mutex.h>
   46 #include <sys/pcpu.h>
   47 #include <sys/smp.h>
   48 #include <sys/sysctl.h>
   49 
   50 #include <machine/smp.h>
   51 
   52 #include "opt_sched.h"
   53 
   54 #ifdef SMP
   55 volatile cpumask_t stopped_cpus;
   56 volatile cpumask_t started_cpus;
   57 cpumask_t idle_cpus_mask;
   58 cpumask_t hlt_cpus_mask;
   59 cpumask_t logical_cpus_mask;
   60 
   61 void (*cpustop_restartfunc)(void);
   62 #endif
   63 /* This is used in modules that need to work in both SMP and UP. */
   64 cpumask_t all_cpus;
   65 
   66 int mp_ncpus;
   67 /* export this for libkvm consumers. */
   68 int mp_maxcpus = MAXCPU;
   69 
   70 struct cpu_top *smp_topology;
   71 volatile int smp_started;
   72 u_int mp_maxid;
   73 
   74 SYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD, NULL, "Kernel SMP");
   75 
   76 SYSCTL_INT(_kern_smp, OID_AUTO, maxcpus, CTLFLAG_RD, &mp_maxcpus, 0,
   77     "Max number of CPUs that the system was compiled for.");
   78 
   79 int smp_active = 0;     /* are the APs allowed to run? */
   80 SYSCTL_INT(_kern_smp, OID_AUTO, active, CTLFLAG_RW, &smp_active, 0,
   81     "Number of Auxillary Processors (APs) that were successfully started");
   82 
   83 int smp_disabled = 0;   /* has smp been disabled? */
   84 SYSCTL_INT(_kern_smp, OID_AUTO, disabled, CTLFLAG_RDTUN, &smp_disabled, 0,
   85     "SMP has been disabled from the loader");
   86 TUNABLE_INT("kern.smp.disabled", &smp_disabled);
   87 
   88 int smp_cpus = 1;       /* how many cpu's running */
   89 SYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD, &smp_cpus, 0,
   90     "Number of CPUs online");
   91 
   92 #ifdef SMP
   93 /* Enable forwarding of a signal to a process running on a different CPU */
   94 static int forward_signal_enabled = 1;
   95 SYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
   96            &forward_signal_enabled, 0,
   97            "Forwarding of a signal to a process on a different CPU");
   98 
   99 /* Enable forwarding of roundrobin to all other cpus */
  100 static int forward_roundrobin_enabled = 1;
  101 SYSCTL_INT(_kern_smp, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
  102            &forward_roundrobin_enabled, 0,
  103            "Forwarding of roundrobin to all other CPUs");
  104 
  105 /* Variables needed for SMP rendezvous. */
  106 static void (*smp_rv_setup_func)(void *arg);
  107 static void (*smp_rv_action_func)(void *arg);
  108 static void (*smp_rv_teardown_func)(void *arg);
  109 static void *smp_rv_func_arg;
  110 static volatile int smp_rv_waiters[2];
  111 
  112 /* 
  113  * Shared mutex to restrict busywaits between smp_rendezvous() and
  114  * smp(_targeted)_tlb_shootdown().  A deadlock occurs if both of these
  115  * functions trigger at once and cause multiple CPUs to busywait with
  116  * interrupts disabled. 
  117  */
  118 struct mtx smp_ipi_mtx;
  119 
  120 /*
  121  * Let the MD SMP code initialize mp_maxid very early if it can.
  122  */
  123 static void
  124 mp_setmaxid(void *dummy)
  125 {
  126         cpu_mp_setmaxid();
  127 }
  128 SYSINIT(cpu_mp_setmaxid, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_setmaxid, NULL)
  129 
  130 /*
  131  * Call the MD SMP initialization code.
  132  */
  133 static void
  134 mp_start(void *dummy)
  135 {
  136 
  137         /* Probe for MP hardware. */
  138         if (smp_disabled != 0 || cpu_mp_probe() == 0) {
  139                 mp_ncpus = 1;
  140                 all_cpus = PCPU_GET(cpumask);
  141                 return;
  142         }
  143 
  144         mtx_init(&smp_ipi_mtx, "smp rendezvous", NULL, MTX_SPIN);
  145         cpu_mp_start();
  146         printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n",
  147             mp_ncpus);
  148         cpu_mp_announce();
  149 }
  150 SYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_SECOND, mp_start, NULL)
  151 
  152 void
  153 forward_signal(struct thread *td)
  154 {
  155         int id;
  156 
  157         /*
  158          * signotify() has already set TDF_ASTPENDING and TDF_NEEDSIGCHECK on
  159          * this thread, so all we need to do is poke it if it is currently
  160          * executing so that it executes ast().
  161          */
  162         mtx_assert(&sched_lock, MA_OWNED);
  163         KASSERT(TD_IS_RUNNING(td),
  164             ("forward_signal: thread is not TDS_RUNNING"));
  165 
  166         CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc);
  167 
  168         if (!smp_started || cold || panicstr)
  169                 return;
  170         if (!forward_signal_enabled)
  171                 return;
  172 
  173         /* No need to IPI ourself. */
  174         if (td == curthread)
  175                 return;
  176 
  177         id = td->td_oncpu;
  178         if (id == NOCPU)
  179                 return;
  180         ipi_selected(1 << id, IPI_AST);
  181 }
  182 
  183 void
  184 forward_roundrobin(void)
  185 {
  186         struct pcpu *pc;
  187         struct thread *td;
  188         cpumask_t id, map, me;
  189 
  190         mtx_assert(&sched_lock, MA_OWNED);
  191 
  192         CTR0(KTR_SMP, "forward_roundrobin()");
  193 
  194         if (!smp_started || cold || panicstr)
  195                 return;
  196         if (!forward_roundrobin_enabled)
  197                 return;
  198         map = 0;
  199         me = PCPU_GET(cpumask);
  200         SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
  201                 td = pc->pc_curthread;
  202                 id = pc->pc_cpumask;
  203                 if (id != me && (id & stopped_cpus) == 0 &&
  204                     td != pc->pc_idlethread) {
  205                         td->td_flags |= TDF_NEEDRESCHED;
  206                         map |= id;
  207                 }
  208         }
  209         ipi_selected(map, IPI_AST);
  210 }
  211 
  212 /*
  213  * When called the executing CPU will send an IPI to all other CPUs
  214  *  requesting that they halt execution.
  215  *
  216  * Usually (but not necessarily) called with 'other_cpus' as its arg.
  217  *
  218  *  - Signals all CPUs in map to stop.
  219  *  - Waits for each to stop.
  220  *
  221  * Returns:
  222  *  -1: error
  223  *   0: NA
  224  *   1: ok
  225  *
  226  * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
  227  *            from executing at same time.
  228  */
  229 int
  230 stop_cpus(cpumask_t map)
  231 {
  232         int i;
  233 
  234         if (!smp_started)
  235                 return 0;
  236 
  237         CTR1(KTR_SMP, "stop_cpus(%x)", map);
  238 
  239         /* send the stop IPI to all CPUs in map */
  240         ipi_selected(map, IPI_STOP);
  241 
  242         i = 0;
  243         while ((atomic_load_acq_int(&stopped_cpus) & map) != map) {
  244                 /* spin */
  245                 i++;
  246 #ifdef DIAGNOSTIC
  247                 if (i == 100000) {
  248                         printf("timeout stopping cpus\n");
  249                         break;
  250                 }
  251 #endif
  252         }
  253 
  254         return 1;
  255 }
  256 
  257 #ifdef KDB_STOP_NMI
  258 int
  259 stop_cpus_nmi(cpumask_t map)
  260 {
  261         int i;
  262 
  263         if (!smp_started)
  264                 return 0;
  265 
  266         CTR1(KTR_SMP, "stop_cpus(%x)", map);
  267 
  268         /* send the stop IPI to all CPUs in map */
  269         ipi_nmi_selected(map);
  270 
  271         i = 0;
  272         while ((atomic_load_acq_int(&stopped_cpus) & map) != map) {
  273                 /* spin */
  274                 i++;
  275 #ifdef DIAGNOSTIC
  276                 if (i == 100000) {
  277                         printf("timeout stopping cpus\n");
  278                         break;
  279                 }
  280 #endif
  281         }
  282 
  283         return 1;
  284 }
  285 #endif /* KDB_STOP_NMI */
  286 
  287 /*
  288  * Called by a CPU to restart stopped CPUs. 
  289  *
  290  * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
  291  *
  292  *  - Signals all CPUs in map to restart.
  293  *  - Waits for each to restart.
  294  *
  295  * Returns:
  296  *  -1: error
  297  *   0: NA
  298  *   1: ok
  299  */
  300 int
  301 restart_cpus(cpumask_t map)
  302 {
  303 
  304         if (!smp_started)
  305                 return 0;
  306 
  307         CTR1(KTR_SMP, "restart_cpus(%x)", map);
  308 
  309         /* signal other cpus to restart */
  310         atomic_store_rel_int(&started_cpus, map);
  311 
  312         /* wait for each to clear its bit */
  313         while ((atomic_load_acq_int(&stopped_cpus) & map) != 0)
  314                 ;       /* nothing */
  315 
  316         return 1;
  317 }
  318 
  319 /*
  320  * All-CPU rendezvous.  CPUs are signalled, all execute the setup function 
  321  * (if specified), rendezvous, execute the action function (if specified),
  322  * rendezvous again, execute the teardown function (if specified), and then
  323  * resume.
  324  *
  325  * Note that the supplied external functions _must_ be reentrant and aware
  326  * that they are running in parallel and in an unknown lock context.
  327  */
  328 void
  329 smp_rendezvous_action(void)
  330 {
  331 
  332         /* setup function */
  333         if (smp_rv_setup_func != NULL)
  334                 smp_rv_setup_func(smp_rv_func_arg);
  335         /* spin on entry rendezvous */
  336         atomic_add_int(&smp_rv_waiters[0], 1);
  337         while (atomic_load_acq_int(&smp_rv_waiters[0]) < mp_ncpus)
  338                 ;       /* nothing */
  339         /* action function */
  340         if (smp_rv_action_func != NULL)
  341                 smp_rv_action_func(smp_rv_func_arg);
  342         /* spin on exit rendezvous */
  343         atomic_add_int(&smp_rv_waiters[1], 1);
  344         while (atomic_load_acq_int(&smp_rv_waiters[1]) < mp_ncpus)
  345                 ;       /* nothing */
  346         /* teardown function */
  347         if (smp_rv_teardown_func != NULL)
  348                 smp_rv_teardown_func(smp_rv_func_arg);
  349 }
  350 
  351 void
  352 smp_rendezvous(void (* setup_func)(void *), 
  353                void (* action_func)(void *),
  354                void (* teardown_func)(void *),
  355                void *arg)
  356 {
  357 
  358         if (!smp_started) {
  359                 if (setup_func != NULL)
  360                         setup_func(arg);
  361                 if (action_func != NULL)
  362                         action_func(arg);
  363                 if (teardown_func != NULL)
  364                         teardown_func(arg);
  365                 return;
  366         }
  367                 
  368         /* obtain rendezvous lock */
  369         mtx_lock_spin(&smp_ipi_mtx);
  370 
  371         /* set static function pointers */
  372         smp_rv_setup_func = setup_func;
  373         smp_rv_action_func = action_func;
  374         smp_rv_teardown_func = teardown_func;
  375         smp_rv_func_arg = arg;
  376         smp_rv_waiters[0] = 0;
  377         smp_rv_waiters[1] = 0;
  378 
  379         /* signal other processors, which will enter the IPI with interrupts off */
  380         ipi_all_but_self(IPI_RENDEZVOUS);
  381 
  382         /* call executor function */
  383         smp_rendezvous_action();
  384 
  385         /* release lock */
  386         mtx_unlock_spin(&smp_ipi_mtx);
  387 }
  388 #else /* !SMP */
  389 
  390 /*
  391  * Provide dummy SMP support for UP kernels.  Modules that need to use SMP
  392  * APIs will still work using this dummy support.
  393  */
  394 static void
  395 mp_setvariables_for_up(void *dummy)
  396 {
  397         mp_ncpus = 1;
  398         mp_maxid = PCPU_GET(cpuid);
  399         all_cpus = PCPU_GET(cpumask);
  400         KASSERT(PCPU_GET(cpuid) == 0, ("UP must have a CPU ID of zero"));
  401 }
  402 SYSINIT(cpu_mp_setvariables, SI_SUB_TUNABLES, SI_ORDER_FIRST,
  403     mp_setvariables_for_up, NULL)
  404 
  405 void
  406 smp_rendezvous(void (* setup_func)(void *), 
  407                void (* action_func)(void *),
  408                void (* teardown_func)(void *),
  409                void *arg)
  410 {
  411 
  412         if (setup_func != NULL)
  413                 setup_func(arg);
  414         if (action_func != NULL)
  415                 action_func(arg);
  416         if (teardown_func != NULL)
  417                 teardown_func(arg);
  418 }
  419 #endif /* SMP */

Cache object: 87d623a1dc3a61bd9fd3928a2149aa6e


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.