subr_cpu.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*      $NetBSD: subr_cpu.c,v 1.18 2022/01/24 09:42:14 andvar Exp $     */
    2 
    3 /*-
    4  * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019, 2020
    5  *     The NetBSD Foundation, Inc.
    6  * All rights reserved.
    7  *
    8  * This code is derived from software contributed to The NetBSD Foundation
    9  * by Andrew Doran.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   30  * POSSIBILITY OF SUCH DAMAGE.
   31  */
   32 
   33 /*-
   34  * Copyright (c)2007 YAMAMOTO Takashi,
   35  * All rights reserved.
   36  *
   37  * Redistribution and use in source and binary forms, with or without
   38  * modification, are permitted provided that the following conditions
   39  * are met:
   40  * 1. Redistributions of source code must retain the above copyright
   41  *    notice, this list of conditions and the following disclaimer.
   42  * 2. Redistributions in binary form must reproduce the above copyright
   43  *    notice, this list of conditions and the following disclaimer in the
   44  *    documentation and/or other materials provided with the distribution.
   45  *
   46  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   47  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   48  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   49  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   50  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   51  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   52  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   53  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   54  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   55  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   56  * SUCH DAMAGE.
   57  */
   58 
   59 /*
   60  * CPU related routines shared with rump.
   61  */
   62 
   63 #include <sys/cdefs.h>
   64 __KERNEL_RCSID(0, "$NetBSD: subr_cpu.c,v 1.18 2022/01/24 09:42:14 andvar Exp $");
   65 
   66 #include <sys/param.h>
   67 #include <sys/atomic.h>
   68 #include <sys/systm.h>
   69 #include <sys/sched.h>
   70 #include <sys/conf.h>
   71 #include <sys/cpu.h>
   72 #include <sys/proc.h>
   73 #include <sys/kernel.h>
   74 #include <sys/kmem.h>
   75 
   76 static void     cpu_topology_fake1(struct cpu_info *);
   77 
   78 kmutex_t        cpu_lock                __cacheline_aligned;
   79 int             ncpu                    __read_mostly;
   80 int             ncpuonline              __read_mostly;
   81 bool            mp_online               __read_mostly;
   82 static bool     cpu_topology_present    __read_mostly;
   83 static bool     cpu_topology_haveslow   __read_mostly;
   84 int64_t         cpu_counts[CPU_COUNT_MAX];
   85 
   86 /* An array of CPUs.  There are ncpu entries. */
   87 struct cpu_info **cpu_infos             __read_mostly;
   88 
   89 /* Note: set on mi_cpu_attach() and idle_loop(). */
   90 kcpuset_t *     kcpuset_attached        __read_mostly   = NULL;
   91 kcpuset_t *     kcpuset_running         __read_mostly   = NULL;
   92 
   93 static char cpu_model[128];
   94 
   95 /*
   96  * mi_cpu_init: early initialisation of MI CPU related structures.
   97  *
   98  * Note: may not block and memory allocator is not yet available.
   99  */
  100 void
  101 mi_cpu_init(void)
  102 {
  103         struct cpu_info *ci;
  104 
  105         mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE);
  106 
  107         kcpuset_create(&kcpuset_attached, true);
  108         kcpuset_create(&kcpuset_running, true);
  109         kcpuset_set(kcpuset_running, 0);
  110 
  111         ci = curcpu();
  112         cpu_topology_fake1(ci);
  113 }
  114 
  115 int
  116 cpu_setmodel(const char *fmt, ...)
  117 {
  118         int len;
  119         va_list ap;
  120 
  121         va_start(ap, fmt);
  122         len = vsnprintf(cpu_model, sizeof(cpu_model), fmt, ap);
  123         va_end(ap);
  124         return len;
  125 }
  126 
  127 const char *
  128 cpu_getmodel(void)
  129 {
  130         return cpu_model;
  131 }
  132 
  133 bool
  134 cpu_softintr_p(void)
  135 {
  136 
  137         return (curlwp->l_pflag & LP_INTR) != 0;
  138 }
  139 
  140 /*
  141  * Collect CPU topology information as each CPU is attached.  This can be
  142  * called early during boot, so we need to be careful what we do.
  143  */
  144 void
  145 cpu_topology_set(struct cpu_info *ci, u_int package_id, u_int core_id,
  146     u_int smt_id, u_int numa_id)
  147 {
  148         enum cpu_rel rel;
  149 
  150         cpu_topology_present = true;
  151         ci->ci_package_id = package_id;
  152         ci->ci_core_id = core_id;
  153         ci->ci_smt_id = smt_id;
  154         ci->ci_numa_id = numa_id;
  155         for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
  156                 ci->ci_sibling[rel] = ci;
  157                 ci->ci_nsibling[rel] = 1;
  158         }
  159 }
  160 
  161 /*
  162  * Collect CPU relative speed
  163  */
  164 void
  165 cpu_topology_setspeed(struct cpu_info *ci, bool slow)
  166 {
  167 
  168         cpu_topology_haveslow |= slow;
  169         ci->ci_is_slow = slow;
  170 }
  171 
  172 /*
  173  * Link a CPU into the given circular list.
  174  */
  175 static void
  176 cpu_topology_link(struct cpu_info *ci, struct cpu_info *ci2, enum cpu_rel rel)
  177 {
  178         struct cpu_info *ci3;
  179 
  180         /* Walk to the end of the existing circular list and append. */
  181         for (ci3 = ci2;; ci3 = ci3->ci_sibling[rel]) {
  182                 ci3->ci_nsibling[rel]++;
  183                 if (ci3->ci_sibling[rel] == ci2) {
  184                         break;
  185                 }
  186         }
  187         ci->ci_sibling[rel] = ci2;
  188         ci3->ci_sibling[rel] = ci;
  189         ci->ci_nsibling[rel] = ci3->ci_nsibling[rel];
  190 }
  191 
  192 /*
  193  * Print out the topology lists.
  194  */
  195 static void
  196 cpu_topology_dump(void)
  197 {
  198 #ifdef DEBUG
  199         CPU_INFO_ITERATOR cii;
  200         struct cpu_info *ci, *ci2;
  201         const char *names[] = { "core", "pkg", "1st" };
  202         enum cpu_rel rel;
  203         int i;
  204 
  205         CTASSERT(__arraycount(names) >= __arraycount(ci->ci_sibling));
  206         if (ncpu == 1) {
  207                 return;
  208         }
  209 
  210         for (CPU_INFO_FOREACH(cii, ci)) {
  211                 if (cpu_topology_haveslow)
  212                         printf("%s ", ci->ci_is_slow ? "slow" : "fast");
  213                 for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
  214                         printf("%s has %d %s siblings:", cpu_name(ci),
  215                             ci->ci_nsibling[rel], names[rel]);
  216                         ci2 = ci->ci_sibling[rel];
  217                         i = 0;
  218                         do {
  219                                 printf(" %s", cpu_name(ci2));
  220                                 ci2 = ci2->ci_sibling[rel];
  221                         } while (++i < 64 && ci2 != ci->ci_sibling[rel]);
  222                         if (i == 64) {
  223                                 printf(" GAVE UP");
  224                         }
  225                         printf("\n");
  226                 }
  227                 printf("%s first in package: %s\n", cpu_name(ci),
  228                     cpu_name(ci->ci_package1st));
  229         }
  230 #endif  /* DEBUG */
  231 }
  232 
  233 /*
  234  * Fake up topology info if we have none, or if what we got was bogus.
  235  * Used early in boot, and by cpu_topology_fake().
  236  */
  237 static void
  238 cpu_topology_fake1(struct cpu_info *ci)
  239 {
  240         enum cpu_rel rel;
  241 
  242         for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
  243                 ci->ci_sibling[rel] = ci;
  244                 ci->ci_nsibling[rel] = 1;
  245         }
  246         if (!cpu_topology_present) {
  247                 ci->ci_package_id = cpu_index(ci);
  248         }
  249         ci->ci_schedstate.spc_flags |=
  250             (SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS);
  251         ci->ci_package1st = ci;
  252         if (!cpu_topology_haveslow) {
  253                 ci->ci_is_slow = false;
  254         }
  255 }
  256 
  257 /*
  258  * Fake up topology info if we have none, or if what we got was bogus.
  259  * Don't override ci_package_id, etc, if cpu_topology_present is set.
  260  * MD code also uses these.
  261  */
  262 static void
  263 cpu_topology_fake(void)
  264 {
  265         CPU_INFO_ITERATOR cii;
  266         struct cpu_info *ci;
  267 
  268         for (CPU_INFO_FOREACH(cii, ci)) {
  269                 cpu_topology_fake1(ci);
  270                 /* Undo (early boot) flag set so everything links OK. */
  271                 ci->ci_schedstate.spc_flags &=
  272                     ~(SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS);
  273         }
  274 }
  275 
  276 /*
  277  * Fix up basic CPU topology info.  Right now that means attach each CPU to
  278  * circular lists of its siblings in the same core, and in the same package.
  279  */
  280 void
  281 cpu_topology_init(void)
  282 {
  283         CPU_INFO_ITERATOR cii, cii2;
  284         struct cpu_info *ci, *ci2, *ci3;
  285         u_int minsmt, mincore;
  286 
  287         if (!cpu_topology_present) {
  288                 cpu_topology_fake();
  289                 goto linkit;
  290         }
  291 
  292         /* Find siblings in same core and package. */
  293         for (CPU_INFO_FOREACH(cii, ci)) {
  294                 ci->ci_schedstate.spc_flags &=
  295                     ~(SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS);
  296                 for (CPU_INFO_FOREACH(cii2, ci2)) {
  297                         /* Avoid bad things happening. */
  298                         if (ci2->ci_package_id == ci->ci_package_id &&
  299                             ci2->ci_core_id == ci->ci_core_id &&
  300                             ci2->ci_smt_id == ci->ci_smt_id &&
  301                             ci2 != ci) {
  302 #ifdef DEBUG
  303                                 printf("cpu%u %p pkg %u core %u smt %u same as "
  304                                        "cpu%u %p pkg %u core %u smt %u\n",
  305                                        cpu_index(ci), ci, ci->ci_package_id,
  306                                        ci->ci_core_id, ci->ci_smt_id,
  307                                        cpu_index(ci2), ci2, ci2->ci_package_id,
  308                                        ci2->ci_core_id, ci2->ci_smt_id);
  309 #endif
  310                                 printf("cpu_topology_init: info bogus, "
  311                                     "faking it\n");
  312                                 cpu_topology_fake();
  313                                 goto linkit;
  314                         }
  315                         if (ci2 == ci ||
  316                             ci2->ci_package_id != ci->ci_package_id) {
  317                                 continue;
  318                         }
  319                         /* Find CPUs in the same core. */
  320                         if (ci->ci_nsibling[CPUREL_CORE] == 1 &&
  321                             ci->ci_core_id == ci2->ci_core_id) {
  322                                 cpu_topology_link(ci, ci2, CPUREL_CORE);
  323                         }
  324                         /* Find CPUs in the same package. */
  325                         if (ci->ci_nsibling[CPUREL_PACKAGE] == 1) {
  326                                 cpu_topology_link(ci, ci2, CPUREL_PACKAGE);
  327                         }
  328                         if (ci->ci_nsibling[CPUREL_CORE] > 1 &&
  329                             ci->ci_nsibling[CPUREL_PACKAGE] > 1) {
  330                                 break;
  331                         }
  332                 }
  333         }
  334 
  335  linkit:
  336         /* Identify lowest numbered SMT in each core. */
  337         for (CPU_INFO_FOREACH(cii, ci)) {
  338                 ci2 = ci3 = ci;
  339                 minsmt = ci->ci_smt_id;
  340                 do {
  341                         if (ci2->ci_smt_id < minsmt) {
  342                                 ci3 = ci2;
  343                                 minsmt = ci2->ci_smt_id;
  344                         }
  345                         ci2 = ci2->ci_sibling[CPUREL_CORE];
  346                 } while (ci2 != ci);
  347                 ci3->ci_schedstate.spc_flags |= SPCF_CORE1ST;
  348         }
  349 
  350         /* Identify lowest numbered SMT in each package. */
  351         ci3 = NULL;
  352         for (CPU_INFO_FOREACH(cii, ci)) {
  353                 if ((ci->ci_schedstate.spc_flags & SPCF_CORE1ST) == 0) {
  354                         continue;
  355                 }
  356                 ci2 = ci3 = ci;
  357                 mincore = ci->ci_core_id;
  358                 do {
  359                         if ((ci2->ci_schedstate.spc_flags &
  360                             SPCF_CORE1ST) != 0 &&
  361                             ci2->ci_core_id < mincore) {
  362                                 ci3 = ci2;
  363                                 mincore = ci2->ci_core_id;
  364                         }
  365                         ci2 = ci2->ci_sibling[CPUREL_PACKAGE];
  366                 } while (ci2 != ci);
  367 
  368                 if ((ci3->ci_schedstate.spc_flags & SPCF_PACKAGE1ST) != 0) {
  369                         /* Already identified - nothing more to do. */
  370                         continue;
  371                 }
  372                 ci3->ci_schedstate.spc_flags |= SPCF_PACKAGE1ST;
  373 
  374                 /* Walk through all CPUs in package and point to first. */
  375                 ci2 = ci3;
  376                 do {
  377                         ci2->ci_package1st = ci3;
  378                         ci2->ci_sibling[CPUREL_PACKAGE1ST] = ci3;
  379                         ci2 = ci2->ci_sibling[CPUREL_PACKAGE];
  380                 } while (ci2 != ci3);
  381 
  382                 /* Now look for somebody else to link to. */
  383                 for (CPU_INFO_FOREACH(cii2, ci2)) {
  384                         if ((ci2->ci_schedstate.spc_flags & SPCF_PACKAGE1ST)
  385                             != 0 && ci2 != ci3) {
  386                                 cpu_topology_link(ci3, ci2, CPUREL_PACKAGE1ST);
  387                                 break;
  388                         }
  389                 }
  390         }
  391 
  392         /* Walk through all packages, starting with value of ci3 from above. */
  393         KASSERT(ci3 != NULL);
  394         ci = ci3;
  395         do {
  396                 /* Walk through CPUs in the package and copy in PACKAGE1ST. */
  397                 ci2 = ci;
  398                 do {
  399                         ci2->ci_sibling[CPUREL_PACKAGE1ST] =
  400                             ci->ci_sibling[CPUREL_PACKAGE1ST];
  401                         ci2->ci_nsibling[CPUREL_PACKAGE1ST] =
  402                             ci->ci_nsibling[CPUREL_PACKAGE1ST];
  403                         ci2 = ci2->ci_sibling[CPUREL_PACKAGE];
  404                 } while (ci2 != ci);
  405                 ci = ci->ci_sibling[CPUREL_PACKAGE1ST];
  406         } while (ci != ci3);
  407 
  408         if (cpu_topology_haveslow) {
  409                 /*
  410                  * For asymmetric systems where some CPUs are slower than
  411                  * others, mark first class CPUs for the scheduler.  This
  412                  * conflicts with SMT right now so whinge if observed.
  413                  */
  414                 if (curcpu()->ci_nsibling[CPUREL_CORE] > 1) {
  415                         printf("cpu_topology_init: asymmetric & SMT??\n");
  416                 }
  417                 for (CPU_INFO_FOREACH(cii, ci)) {
  418                         if (!ci->ci_is_slow) {
  419                                 ci->ci_schedstate.spc_flags |= SPCF_1STCLASS;
  420                         }
  421                 }
  422         } else {
  423                 /*
  424                  * For any other configuration mark the 1st CPU in each
  425                  * core as a first class CPU.
  426                  */
  427                 for (CPU_INFO_FOREACH(cii, ci)) {
  428                         if ((ci->ci_schedstate.spc_flags & SPCF_CORE1ST) != 0) {
  429                                 ci->ci_schedstate.spc_flags |= SPCF_1STCLASS;
  430                         }
  431                 }
  432         }
  433 
  434         cpu_topology_dump();
  435 }
  436 
  437 /*
  438  * Adjust one count, for a counter that's NOT updated from interrupt
  439  * context.  Hardly worth making an inline due to preemption stuff.
  440  */
  441 void
  442 cpu_count(enum cpu_count idx, int64_t delta)
  443 {
  444         lwp_t *l = curlwp;
  445         KPREEMPT_DISABLE(l);
  446         l->l_cpu->ci_counts[idx] += delta;
  447         KPREEMPT_ENABLE(l);
  448 }
  449 
  450 /*
  451  * Fetch fresh sum total for all counts.  Expensive - don't call often.
  452  *
  453  * If poll is true, the caller is okay with less recent values (but
  454  * no more than 1/hz seconds old).  Where this is called very often that
  455  * should be the case.
  456  *
  457  * This should be reasonably quick so that any value collected get isn't
  458  * totally out of whack, and it can also be called from interrupt context,
  459  * so go to splvm() while summing the counters.  It's tempting to use a spin
  460  * mutex here but this routine is called from DDB.
  461  */
  462 void
  463 cpu_count_sync(bool poll)
  464 {
  465         CPU_INFO_ITERATOR cii;
  466         struct cpu_info *ci;
  467         int64_t sum[CPU_COUNT_MAX], *ptr;
  468         static int lasttick;
  469         int curtick, s;
  470         enum cpu_count i;
  471 
  472         KASSERT(sizeof(ci->ci_counts) == sizeof(cpu_counts));
  473 
  474         if (__predict_false(!mp_online)) {
  475                 memcpy(cpu_counts, curcpu()->ci_counts, sizeof(cpu_counts));
  476                 return;
  477         }
  478 
  479         s = splvm();
  480         curtick = getticks();
  481         if (poll && atomic_load_acquire(&lasttick) == curtick) {
  482                 splx(s);
  483                 return;
  484         }
  485         memset(sum, 0, sizeof(sum));
  486         curcpu()->ci_counts[CPU_COUNT_SYNC]++;
  487         for (CPU_INFO_FOREACH(cii, ci)) {
  488                 ptr = ci->ci_counts;
  489                 for (i = 0; i < CPU_COUNT_MAX; i += 8) {
  490                         sum[i+0] += ptr[i+0];
  491                         sum[i+1] += ptr[i+1];
  492                         sum[i+2] += ptr[i+2];
  493                         sum[i+3] += ptr[i+3];
  494                         sum[i+4] += ptr[i+4];
  495                         sum[i+5] += ptr[i+5];
  496                         sum[i+6] += ptr[i+6];
  497                         sum[i+7] += ptr[i+7];
  498                 }
  499                 KASSERT(i == CPU_COUNT_MAX);
  500         }
  501         memcpy(cpu_counts, sum, sizeof(cpu_counts));
  502         atomic_store_release(&lasttick, curtick);
  503         splx(s);
  504 }
Cache object: 676f6baca2e9a9c34c6381397043d8e9
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/subr_cpu.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_cpu.c
