kern_clocksource.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org>
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer,
   12  *    without modification, immediately at the beginning of the file.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   27  */
   28 
   29 #include <sys/cdefs.h>
   30 __FBSDID("$FreeBSD$");
   31 
   32 /*
   33  * Common routines to manage event timers hardware.
   34  */
   35 
   36 #include "opt_device_polling.h"
   37 
   38 #include <sys/param.h>
   39 #include <sys/systm.h>
   40 #include <sys/bus.h>
   41 #include <sys/limits.h>
   42 #include <sys/lock.h>
   43 #include <sys/kdb.h>
   44 #include <sys/ktr.h>
   45 #include <sys/mutex.h>
   46 #include <sys/proc.h>
   47 #include <sys/kernel.h>
   48 #include <sys/sched.h>
   49 #include <sys/smp.h>
   50 #include <sys/sysctl.h>
   51 #include <sys/timeet.h>
   52 #include <sys/timetc.h>
   53 
   54 #include <machine/atomic.h>
   55 #include <machine/clock.h>
   56 #include <machine/cpu.h>
   57 #include <machine/smp.h>
   58 
   59 int                     cpu_disable_c2_sleep = 0; /* Timer dies in C2. */
   60 int                     cpu_disable_c3_sleep = 0; /* Timer dies in C3. */
   61 
   62 static void             setuptimer(void);
   63 static void             loadtimer(sbintime_t now, int first);
   64 static int              doconfigtimer(void);
   65 static void             configtimer(int start);
   66 static int              round_freq(struct eventtimer *et, int freq);
   67 
   68 struct pcpu_state;
   69 static sbintime_t       getnextcpuevent(struct pcpu_state *state, int idle);
   70 static sbintime_t       getnextevent(struct pcpu_state *state);
   71 static int              handleevents(sbintime_t now, int fake);
   72 
   73 static struct mtx       et_hw_mtx;
   74 
   75 #define ET_HW_LOCK(state)                                               \
   76         {                                                               \
   77                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
   78                         mtx_lock_spin(&(state)->et_hw_mtx);             \
   79                 else                                                    \
   80                         mtx_lock_spin(&et_hw_mtx);                      \
   81         }
   82 
   83 #define ET_HW_UNLOCK(state)                                             \
   84         {                                                               \
   85                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
   86                         mtx_unlock_spin(&(state)->et_hw_mtx);           \
   87                 else                                                    \
   88                         mtx_unlock_spin(&et_hw_mtx);                    \
   89         }
   90 
   91 static struct eventtimer *timer = NULL;
   92 static sbintime_t       timerperiod;    /* Timer period for periodic mode. */
   93 static sbintime_t       statperiod;     /* statclock() events period. */
   94 static sbintime_t       profperiod;     /* profclock() events period. */
   95 static sbintime_t       nexttick;       /* Next global timer tick time. */
   96 static u_int            busy = 1;       /* Reconfiguration is in progress. */
   97 static int              profiling;      /* Profiling events enabled. */
   98 
   99 static char             timername[32];  /* Wanted timer. */
  100 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
  101 
  102 static int              singlemul;      /* Multiplier for periodic mode. */
  103 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RWTUN, &singlemul,
  104     0, "Multiplier for periodic mode");
  105 
  106 static u_int            idletick;       /* Run periodic events when idle. */
  107 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RWTUN, &idletick,
  108     0, "Run periodic events when idle");
  109 
  110 static int              periodic;       /* Periodic or one-shot mode. */
  111 static int              want_periodic;  /* What mode to prefer. */
  112 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
  113 
  114 struct pcpu_state {
  115         struct mtx      et_hw_mtx;      /* Per-CPU timer mutex. */
  116         u_int           action;         /* Reconfiguration requests. */
  117         u_int           handle;         /* Immediate handle resuests. */
  118         sbintime_t      now;            /* Last tick time. */
  119         sbintime_t      nextevent;      /* Next scheduled event on this CPU. */
  120         sbintime_t      nexttick;       /* Next timer tick time. */
  121         sbintime_t      nexthard;       /* Next hardclock() event. */
  122         sbintime_t      nextstat;       /* Next statclock() event. */
  123         sbintime_t      nextprof;       /* Next profclock() event. */
  124         sbintime_t      nextcall;       /* Next callout event. */
  125         sbintime_t      nextcallopt;    /* Next optional callout event. */
  126         int             ipi;            /* This CPU needs IPI. */
  127         int             idle;           /* This CPU is in idle mode. */
  128 };
  129 
  130 DPCPU_DEFINE_STATIC(struct pcpu_state, timerstate);
  131 DPCPU_DEFINE(sbintime_t, hardclocktime);
  132 
  133 /*
  134  * Timer broadcast IPI handler.
  135  */
  136 int
  137 hardclockintr(void)
  138 {
  139         sbintime_t now;
  140         struct pcpu_state *state;
  141         int done;
  142 
  143         if (doconfigtimer() || busy)
  144                 return (FILTER_HANDLED);
  145         state = DPCPU_PTR(timerstate);
  146         now = state->now;
  147         CTR2(KTR_SPARE2, "ipi:    now  %d.%08x",
  148             (int)(now >> 32), (u_int)(now & 0xffffffff));
  149         done = handleevents(now, 0);
  150         return (done ? FILTER_HANDLED : FILTER_STRAY);
  151 }
  152 
  153 /*
  154  * Handle all events for specified time on this CPU
  155  */
  156 static int
  157 handleevents(sbintime_t now, int fake)
  158 {
  159         sbintime_t t, *hct;
  160         struct trapframe *frame;
  161         struct pcpu_state *state;
  162         int usermode;
  163         int done, runs;
  164 
  165         CTR2(KTR_SPARE2, "handle:  now  %d.%08x",
  166             (int)(now >> 32), (u_int)(now & 0xffffffff));
  167         done = 0;
  168         if (fake) {
  169                 frame = NULL;
  170                 usermode = 0;
  171         } else {
  172                 frame = curthread->td_intr_frame;
  173                 usermode = TRAPF_USERMODE(frame);
  174         }
  175 
  176         state = DPCPU_PTR(timerstate);
  177 
  178         runs = 0;
  179         while (now >= state->nexthard) {
  180                 state->nexthard += tick_sbt;
  181                 runs++;
  182         }
  183         if (runs) {
  184                 hct = DPCPU_PTR(hardclocktime);
  185                 *hct = state->nexthard - tick_sbt;
  186                 if (fake < 2) {
  187                         hardclock(runs, usermode);
  188                         done = 1;
  189                 }
  190         }
  191         runs = 0;
  192         while (now >= state->nextstat) {
  193                 state->nextstat += statperiod;
  194                 runs++;
  195         }
  196         if (runs && fake < 2) {
  197                 statclock(runs, usermode);
  198                 done = 1;
  199         }
  200         if (profiling) {
  201                 runs = 0;
  202                 while (now >= state->nextprof) {
  203                         state->nextprof += profperiod;
  204                         runs++;
  205                 }
  206                 if (runs && !fake) {
  207                         profclock(runs, usermode, TRAPF_PC(frame));
  208                         done = 1;
  209                 }
  210         } else
  211                 state->nextprof = state->nextstat;
  212         if (now >= state->nextcallopt || now >= state->nextcall) {
  213                 state->nextcall = state->nextcallopt = SBT_MAX;
  214                 callout_process(now);
  215         }
  216 
  217         ET_HW_LOCK(state);
  218         t = getnextcpuevent(state, 0);
  219         if (!busy) {
  220                 state->idle = 0;
  221                 state->nextevent = t;
  222                 loadtimer(now, (fake == 2) &&
  223                     (timer->et_flags & ET_FLAGS_PERCPU));
  224         }
  225         ET_HW_UNLOCK(state);
  226         return (done);
  227 }
  228 
  229 /*
  230  * Schedule binuptime of the next event on current CPU.
  231  */
  232 static sbintime_t
  233 getnextcpuevent(struct pcpu_state *state, int idle)
  234 {
  235         sbintime_t event;
  236         u_int hardfreq;
  237 
  238         /* Handle hardclock() events, skipping some if CPU is idle. */
  239         event = state->nexthard;
  240         if (idle) {
  241                 if (tc_min_ticktock_freq > 1
  242 #ifdef SMP
  243                     && curcpu == CPU_FIRST()
  244 #endif
  245                     )
  246                         hardfreq = hz / tc_min_ticktock_freq;
  247                 else
  248                         hardfreq = hz;
  249                 if (hardfreq > 1)
  250                         event += tick_sbt * (hardfreq - 1);
  251         }
  252         /* Handle callout events. */
  253         if (event > state->nextcall)
  254                 event = state->nextcall;
  255         if (!idle) { /* If CPU is active - handle other types of events. */
  256                 if (event > state->nextstat)
  257                         event = state->nextstat;
  258                 if (profiling && event > state->nextprof)
  259                         event = state->nextprof;
  260         }
  261         return (event);
  262 }
  263 
  264 /*
  265  * Schedule binuptime of the next event on all CPUs.
  266  */
  267 static sbintime_t
  268 getnextevent(struct pcpu_state *state)
  269 {
  270         sbintime_t event;
  271 #ifdef SMP
  272         int     cpu;
  273 #endif
  274 #ifdef KTR
  275         int     c;
  276 
  277         c = -1;
  278 #endif
  279         event = state->nextevent;
  280 #ifdef SMP
  281         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
  282                 CPU_FOREACH(cpu) {
  283                         state = DPCPU_ID_PTR(cpu, timerstate);
  284                         if (event > state->nextevent) {
  285                                 event = state->nextevent;
  286 #ifdef KTR
  287                                 c = cpu;
  288 #endif
  289                         }
  290                 }
  291         }
  292 #endif
  293         CTR3(KTR_SPARE2, "next:    next %d.%08x by %d",
  294             (int)(event >> 32), (u_int)(event & 0xffffffff), c);
  295         return (event);
  296 }
  297 
  298 /* Hardware timer callback function. */
  299 static void
  300 timercb(struct eventtimer *et, void *arg)
  301 {
  302         sbintime_t now;
  303         sbintime_t *next;
  304         struct pcpu_state *state;
  305 #ifdef SMP
  306         int cpu, bcast;
  307 #endif
  308 
  309         /* Do not touch anything if somebody reconfiguring timers. */
  310         if (busy)
  311                 return;
  312         /* Update present and next tick times. */
  313         state = DPCPU_PTR(timerstate);
  314         if (et->et_flags & ET_FLAGS_PERCPU) {
  315                 next = &state->nexttick;
  316         } else
  317                 next = &nexttick;
  318         now = sbinuptime();
  319         if (periodic)
  320                 *next = now + timerperiod;
  321         else
  322                 *next = -1;     /* Next tick is not scheduled yet. */
  323         state->now = now;
  324         CTR2(KTR_SPARE2, "intr:    now  %d.%08x",
  325             (int)(now >> 32), (u_int)(now & 0xffffffff));
  326 
  327 #ifdef SMP
  328 #ifdef EARLY_AP_STARTUP
  329         MPASS(mp_ncpus == 1 || smp_started);
  330 #endif
  331         /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
  332         bcast = 0;
  333 #ifdef EARLY_AP_STARTUP
  334         if ((et->et_flags & ET_FLAGS_PERCPU) == 0) {
  335 #else
  336         if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
  337 #endif
  338                 CPU_FOREACH(cpu) {
  339                         state = DPCPU_ID_PTR(cpu, timerstate);
  340                         ET_HW_LOCK(state);
  341                         state->now = now;
  342                         if (now >= state->nextevent) {
  343                                 state->nextevent += SBT_1S;
  344                                 if (curcpu != cpu) {
  345                                         state->ipi = 1;
  346                                         bcast = 1;
  347                                 }
  348                         }
  349                         ET_HW_UNLOCK(state);
  350                 }
  351         }
  352 #endif
  353 
  354         /* Handle events for this time on this CPU. */
  355         handleevents(now, 0);
  356 
  357 #ifdef SMP
  358         /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
  359         if (bcast) {
  360                 CPU_FOREACH(cpu) {
  361                         if (curcpu == cpu)
  362                                 continue;
  363                         state = DPCPU_ID_PTR(cpu, timerstate);
  364                         if (state->ipi) {
  365                                 state->ipi = 0;
  366                                 ipi_cpu(cpu, IPI_HARDCLOCK);
  367                         }
  368                 }
  369         }
  370 #endif
  371 }
  372 
  373 /*
  374  * Load new value into hardware timer.
  375  */
  376 static void
  377 loadtimer(sbintime_t now, int start)
  378 {
  379         struct pcpu_state *state;
  380         sbintime_t new;
  381         sbintime_t *next;
  382         uint64_t tmp;
  383         int eq;
  384 
  385         state = DPCPU_PTR(timerstate);
  386         if (timer->et_flags & ET_FLAGS_PERCPU)
  387                 next = &state->nexttick;
  388         else
  389                 next = &nexttick;
  390         if (periodic) {
  391                 if (start) {
  392                         /*
  393                          * Try to start all periodic timers aligned
  394                          * to period to make events synchronous.
  395                          */
  396                         tmp = now % timerperiod;
  397                         new = timerperiod - tmp;
  398                         if (new < tmp)          /* Left less then passed. */
  399                                 new += timerperiod;
  400                         CTR4(KTR_SPARE2, "load p:   now %d.%08x first in %d.%08x",
  401                             (int)(now >> 32), (u_int)(now & 0xffffffff),
  402                             (int)(new >> 32), (u_int)(new & 0xffffffff));
  403                         *next = new + now;
  404                         et_start(timer, new, timerperiod);
  405                 }
  406         } else {
  407                 new = getnextevent(state);
  408                 eq = (new == *next);
  409                 CTR3(KTR_SPARE2, "load:    next %d.%08x eq %d",
  410                     (int)(new >> 32), (u_int)(new & 0xffffffff), eq);
  411                 if (!eq) {
  412                         *next = new;
  413                         et_start(timer, new - now, 0);
  414                 }
  415         }
  416 }
  417 
  418 /*
  419  * Prepare event timer parameters after configuration changes.
  420  */
  421 static void
  422 setuptimer(void)
  423 {
  424         int freq;
  425 
  426         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
  427                 periodic = 0;
  428         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
  429                 periodic = 1;
  430         singlemul = MIN(MAX(singlemul, 1), 20);
  431         freq = hz * singlemul;
  432         while (freq < (profiling ? profhz : stathz))
  433                 freq += hz;
  434         freq = round_freq(timer, freq);
  435         timerperiod = SBT_1S / freq;
  436 }
  437 
  438 /*
  439  * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
  440  */
  441 static int
  442 doconfigtimer(void)
  443 {
  444         sbintime_t now;
  445         struct pcpu_state *state;
  446 
  447         state = DPCPU_PTR(timerstate);
  448         switch (atomic_load_acq_int(&state->action)) {
  449         case 1:
  450                 now = sbinuptime();
  451                 ET_HW_LOCK(state);
  452                 loadtimer(now, 1);
  453                 ET_HW_UNLOCK(state);
  454                 state->handle = 0;
  455                 atomic_store_rel_int(&state->action, 0);
  456                 return (1);
  457         case 2:
  458                 ET_HW_LOCK(state);
  459                 et_stop(timer);
  460                 ET_HW_UNLOCK(state);
  461                 state->handle = 0;
  462                 atomic_store_rel_int(&state->action, 0);
  463                 return (1);
  464         }
  465         if (atomic_readandclear_int(&state->handle) && !busy) {
  466                 now = sbinuptime();
  467                 handleevents(now, 0);
  468                 return (1);
  469         }
  470         return (0);
  471 }
  472 
  473 /*
  474  * Reconfigure specified timer.
  475  * For per-CPU timers use IPI to make other CPUs to reconfigure.
  476  */
  477 static void
  478 configtimer(int start)
  479 {
  480         sbintime_t now, next;
  481         struct pcpu_state *state;
  482         int cpu;
  483 
  484         if (start) {
  485                 setuptimer();
  486                 now = sbinuptime();
  487         } else
  488                 now = 0;
  489         critical_enter();
  490         ET_HW_LOCK(DPCPU_PTR(timerstate));
  491         if (start) {
  492                 /* Initialize time machine parameters. */
  493                 next = now + timerperiod;
  494                 if (periodic)
  495                         nexttick = next;
  496                 else
  497                         nexttick = -1;
  498 #ifdef EARLY_AP_STARTUP
  499                 MPASS(mp_ncpus == 1 || smp_started);
  500 #endif
  501                 CPU_FOREACH(cpu) {
  502                         state = DPCPU_ID_PTR(cpu, timerstate);
  503                         state->now = now;
  504 #ifndef EARLY_AP_STARTUP
  505                         if (!smp_started && cpu != CPU_FIRST())
  506                                 state->nextevent = SBT_MAX;
  507                         else
  508 #endif
  509                                 state->nextevent = next;
  510                         if (periodic)
  511                                 state->nexttick = next;
  512                         else
  513                                 state->nexttick = -1;
  514                         state->nexthard = next;
  515                         state->nextstat = next;
  516                         state->nextprof = next;
  517                         state->nextcall = next;
  518                         state->nextcallopt = next;
  519                         hardclock_sync(cpu);
  520                 }
  521                 busy = 0;
  522                 /* Start global timer or per-CPU timer of this CPU. */
  523                 loadtimer(now, 1);
  524         } else {
  525                 busy = 1;
  526                 /* Stop global timer or per-CPU timer of this CPU. */
  527                 et_stop(timer);
  528         }
  529         ET_HW_UNLOCK(DPCPU_PTR(timerstate));
  530 #ifdef SMP
  531 #ifdef EARLY_AP_STARTUP
  532         /* If timer is global we are done. */
  533         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
  534 #else
  535         /* If timer is global or there is no other CPUs yet - we are done. */
  536         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
  537 #endif
  538                 critical_exit();
  539                 return;
  540         }
  541         /* Set reconfigure flags for other CPUs. */
  542         CPU_FOREACH(cpu) {
  543                 state = DPCPU_ID_PTR(cpu, timerstate);
  544                 atomic_store_rel_int(&state->action,
  545                     (cpu == curcpu) ? 0 : ( start ? 1 : 2));
  546         }
  547         /* Broadcast reconfigure IPI. */
  548         ipi_all_but_self(IPI_HARDCLOCK);
  549         /* Wait for reconfiguration completed. */
  550 restart:
  551         cpu_spinwait();
  552         CPU_FOREACH(cpu) {
  553                 if (cpu == curcpu)
  554                         continue;
  555                 state = DPCPU_ID_PTR(cpu, timerstate);
  556                 if (atomic_load_acq_int(&state->action))
  557                         goto restart;
  558         }
  559 #endif
  560         critical_exit();
  561 }
  562 
  563 /*
  564  * Calculate nearest frequency supported by hardware timer.
  565  */
  566 static int
  567 round_freq(struct eventtimer *et, int freq)
  568 {
  569         uint64_t div;
  570 
  571         if (et->et_frequency != 0) {
  572                 div = lmax((et->et_frequency + freq / 2) / freq, 1);
  573                 if (et->et_flags & ET_FLAGS_POW2DIV)
  574                         div = 1 << (flsl(div + div / 2) - 1);
  575                 freq = (et->et_frequency + div / 2) / div;
  576         }
  577         if (et->et_min_period > SBT_1S)
  578                 panic("Event timer \"%s\" doesn't support sub-second periods!",
  579                     et->et_name);
  580         else if (et->et_min_period != 0)
  581                 freq = min(freq, SBT2FREQ(et->et_min_period));
  582         if (et->et_max_period < SBT_1S && et->et_max_period != 0)
  583                 freq = max(freq, SBT2FREQ(et->et_max_period));
  584         return (freq);
  585 }
  586 
  587 /*
  588  * Configure and start event timers (BSP part).
  589  */
  590 void
  591 cpu_initclocks_bsp(void)
  592 {
  593         struct pcpu_state *state;
  594         int base, div, cpu;
  595 
  596         mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
  597         CPU_FOREACH(cpu) {
  598                 state = DPCPU_ID_PTR(cpu, timerstate);
  599                 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
  600                 state->nextcall = SBT_MAX;
  601                 state->nextcallopt = SBT_MAX;
  602         }
  603         periodic = want_periodic;
  604         /* Grab requested timer or the best of present. */
  605         if (timername[0])
  606                 timer = et_find(timername, 0, 0);
  607         if (timer == NULL && periodic) {
  608                 timer = et_find(NULL,
  609                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
  610         }
  611         if (timer == NULL) {
  612                 timer = et_find(NULL,
  613                     ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
  614         }
  615         if (timer == NULL && !periodic) {
  616                 timer = et_find(NULL,
  617                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
  618         }
  619         if (timer == NULL)
  620                 panic("No usable event timer found!");
  621         et_init(timer, timercb, NULL, NULL);
  622 
  623         /* Adapt to timer capabilities. */
  624         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
  625                 periodic = 0;
  626         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
  627                 periodic = 1;
  628         if (timer->et_flags & ET_FLAGS_C3STOP)
  629                 cpu_disable_c3_sleep++;
  630 
  631         /*
  632          * We honor the requested 'hz' value.
  633          * We want to run stathz in the neighborhood of 128hz.
  634          * We would like profhz to run as often as possible.
  635          */
  636         if (singlemul <= 0 || singlemul > 20) {
  637                 if (hz >= 1500 || (hz % 128) == 0)
  638                         singlemul = 1;
  639                 else if (hz >= 750)
  640                         singlemul = 2;
  641                 else
  642                         singlemul = 4;
  643         }
  644         if (periodic) {
  645                 base = round_freq(timer, hz * singlemul);
  646                 singlemul = max((base + hz / 2) / hz, 1);
  647                 hz = (base + singlemul / 2) / singlemul;
  648                 if (base <= 128)
  649                         stathz = base;
  650                 else {
  651                         div = base / 128;
  652                         if (div >= singlemul && (div % singlemul) == 0)
  653                                 div++;
  654                         stathz = base / div;
  655                 }
  656                 profhz = stathz;
  657                 while ((profhz + stathz) <= 128 * 64)
  658                         profhz += stathz;
  659                 profhz = round_freq(timer, profhz);
  660         } else {
  661                 hz = round_freq(timer, hz);
  662                 stathz = round_freq(timer, 127);
  663                 profhz = round_freq(timer, stathz * 64);
  664         }
  665         tick = 1000000 / hz;
  666         tick_sbt = SBT_1S / hz;
  667         tick_bt = sbttobt(tick_sbt);
  668         statperiod = SBT_1S / stathz;
  669         profperiod = SBT_1S / profhz;
  670         ET_LOCK();
  671         configtimer(1);
  672         ET_UNLOCK();
  673 }
  674 
  675 /*
  676  * Start per-CPU event timers on APs.
  677  */
  678 void
  679 cpu_initclocks_ap(void)
  680 {
  681         struct pcpu_state *state;
  682         struct thread *td;
  683 
  684         state = DPCPU_PTR(timerstate);
  685         ET_HW_LOCK(state);
  686         state->now = sbinuptime();
  687         hardclock_sync(curcpu);
  688         spinlock_enter();
  689         ET_HW_UNLOCK(state);
  690         td = curthread;
  691         td->td_intr_nesting_level++;
  692         handleevents(state->now, 2);
  693         td->td_intr_nesting_level--;
  694         spinlock_exit();
  695 }
  696 
  697 void
  698 suspendclock(void)
  699 {
  700         ET_LOCK();
  701         configtimer(0);
  702         ET_UNLOCK();
  703 }
  704 
  705 void
  706 resumeclock(void)
  707 {
  708         ET_LOCK();
  709         configtimer(1);
  710         ET_UNLOCK();
  711 }
  712 
  713 /*
  714  * Switch to profiling clock rates.
  715  */
  716 void
  717 cpu_startprofclock(void)
  718 {
  719 
  720         ET_LOCK();
  721         if (profiling == 0) {
  722                 if (periodic) {
  723                         configtimer(0);
  724                         profiling = 1;
  725                         configtimer(1);
  726                 } else
  727                         profiling = 1;
  728         } else
  729                 profiling++;
  730         ET_UNLOCK();
  731 }
  732 
  733 /*
  734  * Switch to regular clock rates.
  735  */
  736 void
  737 cpu_stopprofclock(void)
  738 {
  739 
  740         ET_LOCK();
  741         if (profiling == 1) {
  742                 if (periodic) {
  743                         configtimer(0);
  744                         profiling = 0;
  745                         configtimer(1);
  746                 } else
  747                 profiling = 0;
  748         } else
  749                 profiling--;
  750         ET_UNLOCK();
  751 }
  752 
  753 /*
  754  * Switch to idle mode (all ticks handled).
  755  */
  756 sbintime_t
  757 cpu_idleclock(void)
  758 {
  759         sbintime_t now, t;
  760         struct pcpu_state *state;
  761 
  762         if (idletick || busy ||
  763             (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
  764 #ifdef DEVICE_POLLING
  765             || curcpu == CPU_FIRST()
  766 #endif
  767             )
  768                 return (-1);
  769         state = DPCPU_PTR(timerstate);
  770         ET_HW_LOCK(state);
  771         if (periodic)
  772                 now = state->now;
  773         else
  774                 now = sbinuptime();
  775         CTR2(KTR_SPARE2, "idle:    now  %d.%08x",
  776             (int)(now >> 32), (u_int)(now & 0xffffffff));
  777         t = getnextcpuevent(state, 1);
  778         state->idle = 1;
  779         state->nextevent = t;
  780         if (!periodic)
  781                 loadtimer(now, 0);
  782         ET_HW_UNLOCK(state);
  783         return (MAX(t - now, 0));
  784 }
  785 
  786 /*
  787  * Switch to active mode (skip empty ticks).
  788  */
  789 void
  790 cpu_activeclock(void)
  791 {
  792         sbintime_t now;
  793         struct pcpu_state *state;
  794         struct thread *td;
  795 
  796         state = DPCPU_PTR(timerstate);
  797         if (atomic_load_int(&state->idle) == 0 || busy)
  798                 return;
  799         spinlock_enter();
  800         if (periodic)
  801                 now = state->now;
  802         else
  803                 now = sbinuptime();
  804         CTR2(KTR_SPARE2, "active:  now  %d.%08x",
  805             (int)(now >> 32), (u_int)(now & 0xffffffff));
  806         td = curthread;
  807         td->td_intr_nesting_level++;
  808         handleevents(now, 1);
  809         td->td_intr_nesting_level--;
  810         spinlock_exit();
  811 }
  812 
  813 /*
  814  * Change the frequency of the given timer.  This changes et->et_frequency and
  815  * if et is the active timer it reconfigures the timer on all CPUs.  This is
  816  * intended to be a private interface for the use of et_change_frequency() only.
  817  */
  818 void
  819 cpu_et_frequency(struct eventtimer *et, uint64_t newfreq)
  820 {
  821 
  822         ET_LOCK();
  823         if (et == timer) {
  824                 configtimer(0);
  825                 et->et_frequency = newfreq;
  826                 configtimer(1);
  827         } else
  828                 et->et_frequency = newfreq;
  829         ET_UNLOCK();
  830 }
  831 
  832 void
  833 cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt)
  834 {
  835         struct pcpu_state *state;
  836 
  837         /* Do not touch anything if somebody reconfiguring timers. */
  838         if (busy)
  839                 return;
  840 
  841         CTR5(KTR_SPARE2, "new co:  on %d at %d.%08x - %d.%08x",
  842             cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff),
  843             (int)(bt >> 32), (u_int)(bt & 0xffffffff));
  844 
  845         KASSERT(!CPU_ABSENT(cpu), ("Absent CPU %d", cpu));
  846         state = DPCPU_ID_PTR(cpu, timerstate);
  847         ET_HW_LOCK(state);
  848 
  849         /*
  850          * If there is callout time already set earlier -- do nothing.
  851          * This check may appear redundant because we check already in
  852          * callout_process() but this double check guarantees we're safe
  853          * with respect to race conditions between interrupts execution
  854          * and scheduling.
  855          */
  856         state->nextcallopt = bt_opt;
  857         if (bt >= state->nextcall)
  858                 goto done;
  859         state->nextcall = bt;
  860         /* If there is some other event set earlier -- do nothing. */
  861         if (bt >= state->nextevent)
  862                 goto done;
  863         state->nextevent = bt;
  864         /* If timer is periodic -- there is nothing to reprogram. */
  865         if (periodic)
  866                 goto done;
  867         /* If timer is global or of the current CPU -- reprogram it. */
  868         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) {
  869                 loadtimer(sbinuptime(), 0);
  870 done:
  871                 ET_HW_UNLOCK(state);
  872                 return;
  873         }
  874         /* Otherwise make other CPU to reprogram it. */
  875         state->handle = 1;
  876         ET_HW_UNLOCK(state);
  877 #ifdef SMP
  878         ipi_cpu(cpu, IPI_HARDCLOCK);
  879 #endif
  880 }
  881 
  882 /*
  883  * Report or change the active event timers hardware.
  884  */
  885 static int
  886 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
  887 {
  888         char buf[32];
  889         struct eventtimer *et;
  890         int error;
  891 
  892         ET_LOCK();
  893         et = timer;
  894         snprintf(buf, sizeof(buf), "%s", et->et_name);
  895         ET_UNLOCK();
  896         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
  897         ET_LOCK();
  898         et = timer;
  899         if (error != 0 || req->newptr == NULL ||
  900             strcasecmp(buf, et->et_name) == 0) {
  901                 ET_UNLOCK();
  902                 return (error);
  903         }
  904         et = et_find(buf, 0, 0);
  905         if (et == NULL) {
  906                 ET_UNLOCK();
  907                 return (ENOENT);
  908         }
  909         configtimer(0);
  910         et_free(timer);
  911         if (et->et_flags & ET_FLAGS_C3STOP)
  912                 cpu_disable_c3_sleep++;
  913         if (timer->et_flags & ET_FLAGS_C3STOP)
  914                 cpu_disable_c3_sleep--;
  915         periodic = want_periodic;
  916         timer = et;
  917         et_init(timer, timercb, NULL, NULL);
  918         configtimer(1);
  919         ET_UNLOCK();
  920         return (error);
  921 }
  922 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
  923     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
  924     0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
  925 
  926 /*
  927  * Report or change the active event timer periodicity.
  928  */
  929 static int
  930 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
  931 {
  932         int error, val;
  933 
  934         val = periodic;
  935         error = sysctl_handle_int(oidp, &val, 0, req);
  936         if (error != 0 || req->newptr == NULL)
  937                 return (error);
  938         ET_LOCK();
  939         configtimer(0);
  940         periodic = want_periodic = val;
  941         configtimer(1);
  942         ET_UNLOCK();
  943         return (error);
  944 }
  945 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
  946     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
  947     0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
  948 
  949 #include "opt_ddb.h"
  950 
  951 #ifdef DDB
  952 #include <ddb/ddb.h>
  953 
  954 DB_SHOW_COMMAND(clocksource, db_show_clocksource)
  955 {
  956         struct pcpu_state *st;
  957         int c;
  958 
  959         CPU_FOREACH(c) {
  960                 st = DPCPU_ID_PTR(c, timerstate);
  961                 db_printf(
  962                     "CPU %2d: action %d handle %d  ipi %d idle %d\n"
  963                     "        now %#jx nevent %#jx (%jd)\n"
  964                     "        ntick %#jx (%jd) nhard %#jx (%jd)\n"
  965                     "        nstat %#jx (%jd) nprof %#jx (%jd)\n"
  966                     "        ncall %#jx (%jd) ncallopt %#jx (%jd)\n",
  967                     c, st->action, st->handle, st->ipi, st->idle,
  968                     (uintmax_t)st->now,
  969                     (uintmax_t)st->nextevent,
  970                     (uintmax_t)(st->nextevent - st->now) / tick_sbt,
  971                     (uintmax_t)st->nexttick,
  972                     (uintmax_t)(st->nexttick - st->now) / tick_sbt,
  973                     (uintmax_t)st->nexthard,
  974                     (uintmax_t)(st->nexthard - st->now) / tick_sbt,
  975                     (uintmax_t)st->nextstat,
  976                     (uintmax_t)(st->nextstat - st->now) / tick_sbt,
  977                     (uintmax_t)st->nextprof,
  978                     (uintmax_t)(st->nextprof - st->now) / tick_sbt,
  979                     (uintmax_t)st->nextcall,
  980                     (uintmax_t)(st->nextcall - st->now) / tick_sbt,
  981                     (uintmax_t)st->nextcallopt,
  982                     (uintmax_t)(st->nextcallopt - st->now) / tick_sbt);
  983         }
  984 }
  985 
  986 #endif
Cache object: 0a8178f57770e3924230abc31f209fb7
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/kern_clocksource.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_clocksource.c
