kern_clocksource.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer,
   10  *    without modification, immediately at the beginning of the file.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD: releng/11.0/sys/kern/kern_clocksource.c 299746 2016-05-14 18:22:52Z jhb $");
   29 
   30 /*
   31  * Common routines to manage event timers hardware.
   32  */
   33 
   34 #include "opt_device_polling.h"
   35 
   36 #include <sys/param.h>
   37 #include <sys/systm.h>
   38 #include <sys/bus.h>
   39 #include <sys/limits.h>
   40 #include <sys/lock.h>
   41 #include <sys/kdb.h>
   42 #include <sys/ktr.h>
   43 #include <sys/mutex.h>
   44 #include <sys/proc.h>
   45 #include <sys/kernel.h>
   46 #include <sys/sched.h>
   47 #include <sys/smp.h>
   48 #include <sys/sysctl.h>
   49 #include <sys/timeet.h>
   50 #include <sys/timetc.h>
   51 
   52 #include <machine/atomic.h>
   53 #include <machine/clock.h>
   54 #include <machine/cpu.h>
   55 #include <machine/smp.h>
   56 
   57 int                     cpu_deepest_sleep = 0;  /* Deepest Cx state available. */
   58 int                     cpu_disable_c2_sleep = 0; /* Timer dies in C2. */
   59 int                     cpu_disable_c3_sleep = 0; /* Timer dies in C3. */
   60 
   61 static void             setuptimer(void);
   62 static void             loadtimer(sbintime_t now, int first);
   63 static int              doconfigtimer(void);
   64 static void             configtimer(int start);
   65 static int              round_freq(struct eventtimer *et, int freq);
   66 
   67 static sbintime_t       getnextcpuevent(int idle);
   68 static sbintime_t       getnextevent(void);
   69 static int              handleevents(sbintime_t now, int fake);
   70 
   71 static struct mtx       et_hw_mtx;
   72 
   73 #define ET_HW_LOCK(state)                                               \
   74         {                                                               \
   75                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
   76                         mtx_lock_spin(&(state)->et_hw_mtx);             \
   77                 else                                                    \
   78                         mtx_lock_spin(&et_hw_mtx);                      \
   79         }
   80 
   81 #define ET_HW_UNLOCK(state)                                             \
   82         {                                                               \
   83                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
   84                         mtx_unlock_spin(&(state)->et_hw_mtx);           \
   85                 else                                                    \
   86                         mtx_unlock_spin(&et_hw_mtx);                    \
   87         }
   88 
   89 static struct eventtimer *timer = NULL;
   90 static sbintime_t       timerperiod;    /* Timer period for periodic mode. */
   91 static sbintime_t       statperiod;     /* statclock() events period. */
   92 static sbintime_t       profperiod;     /* profclock() events period. */
   93 static sbintime_t       nexttick;       /* Next global timer tick time. */
   94 static u_int            busy = 1;       /* Reconfiguration is in progress. */
   95 static int              profiling;      /* Profiling events enabled. */
   96 
   97 static char             timername[32];  /* Wanted timer. */
   98 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
   99 
  100 static int              singlemul;      /* Multiplier for periodic mode. */
  101 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RWTUN, &singlemul,
  102     0, "Multiplier for periodic mode");
  103 
  104 static u_int            idletick;       /* Run periodic events when idle. */
  105 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RWTUN, &idletick,
  106     0, "Run periodic events when idle");
  107 
  108 static int              periodic;       /* Periodic or one-shot mode. */
  109 static int              want_periodic;  /* What mode to prefer. */
  110 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
  111 
  112 struct pcpu_state {
  113         struct mtx      et_hw_mtx;      /* Per-CPU timer mutex. */
  114         u_int           action;         /* Reconfiguration requests. */
  115         u_int           handle;         /* Immediate handle resuests. */
  116         sbintime_t      now;            /* Last tick time. */
  117         sbintime_t      nextevent;      /* Next scheduled event on this CPU. */
  118         sbintime_t      nexttick;       /* Next timer tick time. */
  119         sbintime_t      nexthard;       /* Next hardclock() event. */
  120         sbintime_t      nextstat;       /* Next statclock() event. */
  121         sbintime_t      nextprof;       /* Next profclock() event. */
  122         sbintime_t      nextcall;       /* Next callout event. */
  123         sbintime_t      nextcallopt;    /* Next optional callout event. */
  124         int             ipi;            /* This CPU needs IPI. */
  125         int             idle;           /* This CPU is in idle mode. */
  126 };
  127 
  128 static DPCPU_DEFINE(struct pcpu_state, timerstate);
  129 DPCPU_DEFINE(sbintime_t, hardclocktime);
  130 
  131 /*
  132  * Timer broadcast IPI handler.
  133  */
  134 int
  135 hardclockintr(void)
  136 {
  137         sbintime_t now;
  138         struct pcpu_state *state;
  139         int done;
  140 
  141         if (doconfigtimer() || busy)
  142                 return (FILTER_HANDLED);
  143         state = DPCPU_PTR(timerstate);
  144         now = state->now;
  145         CTR3(KTR_SPARE2, "ipi  at %d:    now  %d.%08x",
  146             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  147         done = handleevents(now, 0);
  148         return (done ? FILTER_HANDLED : FILTER_STRAY);
  149 }
  150 
  151 /*
  152  * Handle all events for specified time on this CPU
  153  */
  154 static int
  155 handleevents(sbintime_t now, int fake)
  156 {
  157         sbintime_t t, *hct;
  158         struct trapframe *frame;
  159         struct pcpu_state *state;
  160         int usermode;
  161         int done, runs;
  162 
  163         CTR3(KTR_SPARE2, "handle at %d:  now  %d.%08x",
  164             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  165         done = 0;
  166         if (fake) {
  167                 frame = NULL;
  168                 usermode = 0;
  169         } else {
  170                 frame = curthread->td_intr_frame;
  171                 usermode = TRAPF_USERMODE(frame);
  172         }
  173 
  174         state = DPCPU_PTR(timerstate);
  175 
  176         runs = 0;
  177         while (now >= state->nexthard) {
  178                 state->nexthard += tick_sbt;
  179                 runs++;
  180         }
  181         if (runs) {
  182                 hct = DPCPU_PTR(hardclocktime);
  183                 *hct = state->nexthard - tick_sbt;
  184                 if (fake < 2) {
  185                         hardclock_cnt(runs, usermode);
  186                         done = 1;
  187                 }
  188         }
  189         runs = 0;
  190         while (now >= state->nextstat) {
  191                 state->nextstat += statperiod;
  192                 runs++;
  193         }
  194         if (runs && fake < 2) {
  195                 statclock_cnt(runs, usermode);
  196                 done = 1;
  197         }
  198         if (profiling) {
  199                 runs = 0;
  200                 while (now >= state->nextprof) {
  201                         state->nextprof += profperiod;
  202                         runs++;
  203                 }
  204                 if (runs && !fake) {
  205                         profclock_cnt(runs, usermode, TRAPF_PC(frame));
  206                         done = 1;
  207                 }
  208         } else
  209                 state->nextprof = state->nextstat;
  210         if (now >= state->nextcallopt) {
  211                 state->nextcall = state->nextcallopt = SBT_MAX;
  212                 callout_process(now);
  213         }
  214 
  215         t = getnextcpuevent(0);
  216         ET_HW_LOCK(state);
  217         if (!busy) {
  218                 state->idle = 0;
  219                 state->nextevent = t;
  220                 loadtimer(now, (fake == 2) &&
  221                     (timer->et_flags & ET_FLAGS_PERCPU));
  222         }
  223         ET_HW_UNLOCK(state);
  224         return (done);
  225 }
  226 
  227 /*
  228  * Schedule binuptime of the next event on current CPU.
  229  */
  230 static sbintime_t
  231 getnextcpuevent(int idle)
  232 {
  233         sbintime_t event;
  234         struct pcpu_state *state;
  235         u_int hardfreq;
  236 
  237         state = DPCPU_PTR(timerstate);
  238         /* Handle hardclock() events, skipping some if CPU is idle. */
  239         event = state->nexthard;
  240         if (idle) {
  241                 hardfreq = (u_int)hz / 2;
  242                 if (tc_min_ticktock_freq > 2
  243 #ifdef SMP
  244                     && curcpu == CPU_FIRST()
  245 #endif
  246                     )
  247                         hardfreq = hz / tc_min_ticktock_freq;
  248                 if (hardfreq > 1)
  249                         event += tick_sbt * (hardfreq - 1);
  250         }
  251         /* Handle callout events. */
  252         if (event > state->nextcall)
  253                 event = state->nextcall;
  254         if (!idle) { /* If CPU is active - handle other types of events. */
  255                 if (event > state->nextstat)
  256                         event = state->nextstat;
  257                 if (profiling && event > state->nextprof)
  258                         event = state->nextprof;
  259         }
  260         return (event);
  261 }
  262 
  263 /*
  264  * Schedule binuptime of the next event on all CPUs.
  265  */
  266 static sbintime_t
  267 getnextevent(void)
  268 {
  269         struct pcpu_state *state;
  270         sbintime_t event;
  271 #ifdef SMP
  272         int     cpu;
  273 #endif
  274         int     c;
  275 
  276         state = DPCPU_PTR(timerstate);
  277         event = state->nextevent;
  278         c = -1;
  279 #ifdef SMP
  280         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
  281                 CPU_FOREACH(cpu) {
  282                         state = DPCPU_ID_PTR(cpu, timerstate);
  283                         if (event > state->nextevent) {
  284                                 event = state->nextevent;
  285                                 c = cpu;
  286                         }
  287                 }
  288         }
  289 #endif
  290         CTR4(KTR_SPARE2, "next at %d:    next %d.%08x by %d",
  291             curcpu, (int)(event >> 32), (u_int)(event & 0xffffffff), c);
  292         return (event);
  293 }
  294 
  295 /* Hardware timer callback function. */
  296 static void
  297 timercb(struct eventtimer *et, void *arg)
  298 {
  299         sbintime_t now;
  300         sbintime_t *next;
  301         struct pcpu_state *state;
  302 #ifdef SMP
  303         int cpu, bcast;
  304 #endif
  305 
  306         /* Do not touch anything if somebody reconfiguring timers. */
  307         if (busy)
  308                 return;
  309         /* Update present and next tick times. */
  310         state = DPCPU_PTR(timerstate);
  311         if (et->et_flags & ET_FLAGS_PERCPU) {
  312                 next = &state->nexttick;
  313         } else
  314                 next = &nexttick;
  315         now = sbinuptime();
  316         if (periodic)
  317                 *next = now + timerperiod;
  318         else
  319                 *next = -1;     /* Next tick is not scheduled yet. */
  320         state->now = now;
  321         CTR3(KTR_SPARE2, "intr at %d:    now  %d.%08x",
  322             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  323 
  324 #ifdef SMP
  325 #ifdef EARLY_AP_STARTUP
  326         MPASS(mp_ncpus == 1 || smp_started);
  327 #endif
  328         /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
  329         bcast = 0;
  330 #ifdef EARLY_AP_STARTUP
  331         if ((et->et_flags & ET_FLAGS_PERCPU) == 0) {
  332 #else
  333         if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
  334 #endif
  335                 CPU_FOREACH(cpu) {
  336                         state = DPCPU_ID_PTR(cpu, timerstate);
  337                         ET_HW_LOCK(state);
  338                         state->now = now;
  339                         if (now >= state->nextevent) {
  340                                 state->nextevent += SBT_1S;
  341                                 if (curcpu != cpu) {
  342                                         state->ipi = 1;
  343                                         bcast = 1;
  344                                 }
  345                         }
  346                         ET_HW_UNLOCK(state);
  347                 }
  348         }
  349 #endif
  350 
  351         /* Handle events for this time on this CPU. */
  352         handleevents(now, 0);
  353 
  354 #ifdef SMP
  355         /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
  356         if (bcast) {
  357                 CPU_FOREACH(cpu) {
  358                         if (curcpu == cpu)
  359                                 continue;
  360                         state = DPCPU_ID_PTR(cpu, timerstate);
  361                         if (state->ipi) {
  362                                 state->ipi = 0;
  363                                 ipi_cpu(cpu, IPI_HARDCLOCK);
  364                         }
  365                 }
  366         }
  367 #endif
  368 }
  369 
  370 /*
  371  * Load new value into hardware timer.
  372  */
  373 static void
  374 loadtimer(sbintime_t now, int start)
  375 {
  376         struct pcpu_state *state;
  377         sbintime_t new;
  378         sbintime_t *next;
  379         uint64_t tmp;
  380         int eq;
  381 
  382         if (timer->et_flags & ET_FLAGS_PERCPU) {
  383                 state = DPCPU_PTR(timerstate);
  384                 next = &state->nexttick;
  385         } else
  386                 next = &nexttick;
  387         if (periodic) {
  388                 if (start) {
  389                         /*
  390                          * Try to start all periodic timers aligned
  391                          * to period to make events synchronous.
  392                          */
  393                         tmp = now % timerperiod;
  394                         new = timerperiod - tmp;
  395                         if (new < tmp)          /* Left less then passed. */
  396                                 new += timerperiod;
  397                         CTR5(KTR_SPARE2, "load p at %d:   now %d.%08x first in %d.%08x",
  398                             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff),
  399                             (int)(new >> 32), (u_int)(new & 0xffffffff));
  400                         *next = new + now;
  401                         et_start(timer, new, timerperiod);
  402                 }
  403         } else {
  404                 new = getnextevent();
  405                 eq = (new == *next);
  406                 CTR4(KTR_SPARE2, "load at %d:    next %d.%08x eq %d",
  407                     curcpu, (int)(new >> 32), (u_int)(new & 0xffffffff), eq);
  408                 if (!eq) {
  409                         *next = new;
  410                         et_start(timer, new - now, 0);
  411                 }
  412         }
  413 }
  414 
  415 /*
  416  * Prepare event timer parameters after configuration changes.
  417  */
  418 static void
  419 setuptimer(void)
  420 {
  421         int freq;
  422 
  423         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
  424                 periodic = 0;
  425         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
  426                 periodic = 1;
  427         singlemul = MIN(MAX(singlemul, 1), 20);
  428         freq = hz * singlemul;
  429         while (freq < (profiling ? profhz : stathz))
  430                 freq += hz;
  431         freq = round_freq(timer, freq);
  432         timerperiod = SBT_1S / freq;
  433 }
  434 
  435 /*
  436  * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
  437  */
  438 static int
  439 doconfigtimer(void)
  440 {
  441         sbintime_t now;
  442         struct pcpu_state *state;
  443 
  444         state = DPCPU_PTR(timerstate);
  445         switch (atomic_load_acq_int(&state->action)) {
  446         case 1:
  447                 now = sbinuptime();
  448                 ET_HW_LOCK(state);
  449                 loadtimer(now, 1);
  450                 ET_HW_UNLOCK(state);
  451                 state->handle = 0;
  452                 atomic_store_rel_int(&state->action, 0);
  453                 return (1);
  454         case 2:
  455                 ET_HW_LOCK(state);
  456                 et_stop(timer);
  457                 ET_HW_UNLOCK(state);
  458                 state->handle = 0;
  459                 atomic_store_rel_int(&state->action, 0);
  460                 return (1);
  461         }
  462         if (atomic_readandclear_int(&state->handle) && !busy) {
  463                 now = sbinuptime();
  464                 handleevents(now, 0);
  465                 return (1);
  466         }
  467         return (0);
  468 }
  469 
  470 /*
  471  * Reconfigure specified timer.
  472  * For per-CPU timers use IPI to make other CPUs to reconfigure.
  473  */
  474 static void
  475 configtimer(int start)
  476 {
  477         sbintime_t now, next;
  478         struct pcpu_state *state;
  479         int cpu;
  480 
  481         if (start) {
  482                 setuptimer();
  483                 now = sbinuptime();
  484         } else
  485                 now = 0;
  486         critical_enter();
  487         ET_HW_LOCK(DPCPU_PTR(timerstate));
  488         if (start) {
  489                 /* Initialize time machine parameters. */
  490                 next = now + timerperiod;
  491                 if (periodic)
  492                         nexttick = next;
  493                 else
  494                         nexttick = -1;
  495 #ifdef EARLY_AP_STARTUP
  496                 MPASS(mp_ncpus == 1 || smp_started);
  497 #endif
  498                 CPU_FOREACH(cpu) {
  499                         state = DPCPU_ID_PTR(cpu, timerstate);
  500                         state->now = now;
  501 #ifndef EARLY_AP_STARTUP
  502                         if (!smp_started && cpu != CPU_FIRST())
  503                                 state->nextevent = SBT_MAX;
  504                         else
  505 #endif
  506                                 state->nextevent = next;
  507                         if (periodic)
  508                                 state->nexttick = next;
  509                         else
  510                                 state->nexttick = -1;
  511                         state->nexthard = next;
  512                         state->nextstat = next;
  513                         state->nextprof = next;
  514                         state->nextcall = next;
  515                         state->nextcallopt = next;
  516                         hardclock_sync(cpu);
  517                 }
  518                 busy = 0;
  519                 /* Start global timer or per-CPU timer of this CPU. */
  520                 loadtimer(now, 1);
  521         } else {
  522                 busy = 1;
  523                 /* Stop global timer or per-CPU timer of this CPU. */
  524                 et_stop(timer);
  525         }
  526         ET_HW_UNLOCK(DPCPU_PTR(timerstate));
  527 #ifdef SMP
  528 #ifdef EARLY_AP_STARTUP
  529         /* If timer is global we are done. */
  530         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
  531 #else
  532         /* If timer is global or there is no other CPUs yet - we are done. */
  533         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
  534 #endif
  535                 critical_exit();
  536                 return;
  537         }
  538         /* Set reconfigure flags for other CPUs. */
  539         CPU_FOREACH(cpu) {
  540                 state = DPCPU_ID_PTR(cpu, timerstate);
  541                 atomic_store_rel_int(&state->action,
  542                     (cpu == curcpu) ? 0 : ( start ? 1 : 2));
  543         }
  544         /* Broadcast reconfigure IPI. */
  545         ipi_all_but_self(IPI_HARDCLOCK);
  546         /* Wait for reconfiguration completed. */
  547 restart:
  548         cpu_spinwait();
  549         CPU_FOREACH(cpu) {
  550                 if (cpu == curcpu)
  551                         continue;
  552                 state = DPCPU_ID_PTR(cpu, timerstate);
  553                 if (atomic_load_acq_int(&state->action))
  554                         goto restart;
  555         }
  556 #endif
  557         critical_exit();
  558 }
  559 
  560 /*
  561  * Calculate nearest frequency supported by hardware timer.
  562  */
  563 static int
  564 round_freq(struct eventtimer *et, int freq)
  565 {
  566         uint64_t div;
  567 
  568         if (et->et_frequency != 0) {
  569                 div = lmax((et->et_frequency + freq / 2) / freq, 1);
  570                 if (et->et_flags & ET_FLAGS_POW2DIV)
  571                         div = 1 << (flsl(div + div / 2) - 1);
  572                 freq = (et->et_frequency + div / 2) / div;
  573         }
  574         if (et->et_min_period > SBT_1S)
  575                 panic("Event timer \"%s\" doesn't support sub-second periods!",
  576                     et->et_name);
  577         else if (et->et_min_period != 0)
  578                 freq = min(freq, SBT2FREQ(et->et_min_period));
  579         if (et->et_max_period < SBT_1S && et->et_max_period != 0)
  580                 freq = max(freq, SBT2FREQ(et->et_max_period));
  581         return (freq);
  582 }
  583 
  584 /*
  585  * Configure and start event timers (BSP part).
  586  */
  587 void
  588 cpu_initclocks_bsp(void)
  589 {
  590         struct pcpu_state *state;
  591         int base, div, cpu;
  592 
  593         mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
  594         CPU_FOREACH(cpu) {
  595                 state = DPCPU_ID_PTR(cpu, timerstate);
  596                 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
  597                 state->nextcall = SBT_MAX;
  598                 state->nextcallopt = SBT_MAX;
  599         }
  600         periodic = want_periodic;
  601         /* Grab requested timer or the best of present. */
  602         if (timername[0])
  603                 timer = et_find(timername, 0, 0);
  604         if (timer == NULL && periodic) {
  605                 timer = et_find(NULL,
  606                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
  607         }
  608         if (timer == NULL) {
  609                 timer = et_find(NULL,
  610                     ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
  611         }
  612         if (timer == NULL && !periodic) {
  613                 timer = et_find(NULL,
  614                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
  615         }
  616         if (timer == NULL)
  617                 panic("No usable event timer found!");
  618         et_init(timer, timercb, NULL, NULL);
  619 
  620         /* Adapt to timer capabilities. */
  621         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
  622                 periodic = 0;
  623         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
  624                 periodic = 1;
  625         if (timer->et_flags & ET_FLAGS_C3STOP)
  626                 cpu_disable_c3_sleep++;
  627 
  628         /*
  629          * We honor the requested 'hz' value.
  630          * We want to run stathz in the neighborhood of 128hz.
  631          * We would like profhz to run as often as possible.
  632          */
  633         if (singlemul <= 0 || singlemul > 20) {
  634                 if (hz >= 1500 || (hz % 128) == 0)
  635                         singlemul = 1;
  636                 else if (hz >= 750)
  637                         singlemul = 2;
  638                 else
  639                         singlemul = 4;
  640         }
  641         if (periodic) {
  642                 base = round_freq(timer, hz * singlemul);
  643                 singlemul = max((base + hz / 2) / hz, 1);
  644                 hz = (base + singlemul / 2) / singlemul;
  645                 if (base <= 128)
  646                         stathz = base;
  647                 else {
  648                         div = base / 128;
  649                         if (div >= singlemul && (div % singlemul) == 0)
  650                                 div++;
  651                         stathz = base / div;
  652                 }
  653                 profhz = stathz;
  654                 while ((profhz + stathz) <= 128 * 64)
  655                         profhz += stathz;
  656                 profhz = round_freq(timer, profhz);
  657         } else {
  658                 hz = round_freq(timer, hz);
  659                 stathz = round_freq(timer, 127);
  660                 profhz = round_freq(timer, stathz * 64);
  661         }
  662         tick = 1000000 / hz;
  663         tick_sbt = SBT_1S / hz;
  664         tick_bt = sbttobt(tick_sbt);
  665         statperiod = SBT_1S / stathz;
  666         profperiod = SBT_1S / profhz;
  667         ET_LOCK();
  668         configtimer(1);
  669         ET_UNLOCK();
  670 }
  671 
  672 /*
  673  * Start per-CPU event timers on APs.
  674  */
  675 void
  676 cpu_initclocks_ap(void)
  677 {
  678         sbintime_t now;
  679         struct pcpu_state *state;
  680         struct thread *td;
  681 
  682         state = DPCPU_PTR(timerstate);
  683         now = sbinuptime();
  684         ET_HW_LOCK(state);
  685         state->now = now;
  686         hardclock_sync(curcpu);
  687         spinlock_enter();
  688         ET_HW_UNLOCK(state);
  689         td = curthread;
  690         td->td_intr_nesting_level++;
  691         handleevents(state->now, 2);
  692         td->td_intr_nesting_level--;
  693         spinlock_exit();
  694 }
  695 
  696 /*
  697  * Switch to profiling clock rates.
  698  */
  699 void
  700 cpu_startprofclock(void)
  701 {
  702 
  703         ET_LOCK();
  704         if (profiling == 0) {
  705                 if (periodic) {
  706                         configtimer(0);
  707                         profiling = 1;
  708                         configtimer(1);
  709                 } else
  710                         profiling = 1;
  711         } else
  712                 profiling++;
  713         ET_UNLOCK();
  714 }
  715 
  716 /*
  717  * Switch to regular clock rates.
  718  */
  719 void
  720 cpu_stopprofclock(void)
  721 {
  722 
  723         ET_LOCK();
  724         if (profiling == 1) {
  725                 if (periodic) {
  726                         configtimer(0);
  727                         profiling = 0;
  728                         configtimer(1);
  729                 } else
  730                 profiling = 0;
  731         } else
  732                 profiling--;
  733         ET_UNLOCK();
  734 }
  735 
  736 /*
  737  * Switch to idle mode (all ticks handled).
  738  */
  739 sbintime_t
  740 cpu_idleclock(void)
  741 {
  742         sbintime_t now, t;
  743         struct pcpu_state *state;
  744 
  745         if (idletick || busy ||
  746             (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
  747 #ifdef DEVICE_POLLING
  748             || curcpu == CPU_FIRST()
  749 #endif
  750             )
  751                 return (-1);
  752         state = DPCPU_PTR(timerstate);
  753         if (periodic)
  754                 now = state->now;
  755         else
  756                 now = sbinuptime();
  757         CTR3(KTR_SPARE2, "idle at %d:    now  %d.%08x",
  758             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  759         t = getnextcpuevent(1);
  760         ET_HW_LOCK(state);
  761         state->idle = 1;
  762         state->nextevent = t;
  763         if (!periodic)
  764                 loadtimer(now, 0);
  765         ET_HW_UNLOCK(state);
  766         return (MAX(t - now, 0));
  767 }
  768 
  769 /*
  770  * Switch to active mode (skip empty ticks).
  771  */
  772 void
  773 cpu_activeclock(void)
  774 {
  775         sbintime_t now;
  776         struct pcpu_state *state;
  777         struct thread *td;
  778 
  779         state = DPCPU_PTR(timerstate);
  780         if (state->idle == 0 || busy)
  781                 return;
  782         if (periodic)
  783                 now = state->now;
  784         else
  785                 now = sbinuptime();
  786         CTR3(KTR_SPARE2, "active at %d:  now  %d.%08x",
  787             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  788         spinlock_enter();
  789         td = curthread;
  790         td->td_intr_nesting_level++;
  791         handleevents(now, 1);
  792         td->td_intr_nesting_level--;
  793         spinlock_exit();
  794 }
  795 
  796 /*
  797  * Change the frequency of the given timer.  This changes et->et_frequency and
  798  * if et is the active timer it reconfigures the timer on all CPUs.  This is
  799  * intended to be a private interface for the use of et_change_frequency() only.
  800  */
  801 void
  802 cpu_et_frequency(struct eventtimer *et, uint64_t newfreq)
  803 {
  804 
  805         ET_LOCK();
  806         if (et == timer) {
  807                 configtimer(0);
  808                 et->et_frequency = newfreq;
  809                 configtimer(1);
  810         } else
  811                 et->et_frequency = newfreq;
  812         ET_UNLOCK();
  813 }
  814 
  815 void
  816 cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt)
  817 {
  818         struct pcpu_state *state;
  819 
  820         /* Do not touch anything if somebody reconfiguring timers. */
  821         if (busy)
  822                 return;
  823         CTR6(KTR_SPARE2, "new co at %d:    on %d at %d.%08x - %d.%08x",
  824             curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff),
  825             (int)(bt >> 32), (u_int)(bt & 0xffffffff));
  826         state = DPCPU_ID_PTR(cpu, timerstate);
  827         ET_HW_LOCK(state);
  828 
  829         /*
  830          * If there is callout time already set earlier -- do nothing.
  831          * This check may appear redundant because we check already in
  832          * callout_process() but this double check guarantees we're safe
  833          * with respect to race conditions between interrupts execution
  834          * and scheduling.
  835          */
  836         state->nextcallopt = bt_opt;
  837         if (bt >= state->nextcall)
  838                 goto done;
  839         state->nextcall = bt;
  840         /* If there is some other event set earlier -- do nothing. */
  841         if (bt >= state->nextevent)
  842                 goto done;
  843         state->nextevent = bt;
  844         /* If timer is periodic -- there is nothing to reprogram. */
  845         if (periodic)
  846                 goto done;
  847         /* If timer is global or of the current CPU -- reprogram it. */
  848         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) {
  849                 loadtimer(sbinuptime(), 0);
  850 done:
  851                 ET_HW_UNLOCK(state);
  852                 return;
  853         }
  854         /* Otherwise make other CPU to reprogram it. */
  855         state->handle = 1;
  856         ET_HW_UNLOCK(state);
  857 #ifdef SMP
  858         ipi_cpu(cpu, IPI_HARDCLOCK);
  859 #endif
  860 }
  861 
  862 /*
  863  * Report or change the active event timers hardware.
  864  */
  865 static int
  866 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
  867 {
  868         char buf[32];
  869         struct eventtimer *et;
  870         int error;
  871 
  872         ET_LOCK();
  873         et = timer;
  874         snprintf(buf, sizeof(buf), "%s", et->et_name);
  875         ET_UNLOCK();
  876         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
  877         ET_LOCK();
  878         et = timer;
  879         if (error != 0 || req->newptr == NULL ||
  880             strcasecmp(buf, et->et_name) == 0) {
  881                 ET_UNLOCK();
  882                 return (error);
  883         }
  884         et = et_find(buf, 0, 0);
  885         if (et == NULL) {
  886                 ET_UNLOCK();
  887                 return (ENOENT);
  888         }
  889         configtimer(0);
  890         et_free(timer);
  891         if (et->et_flags & ET_FLAGS_C3STOP)
  892                 cpu_disable_c3_sleep++;
  893         if (timer->et_flags & ET_FLAGS_C3STOP)
  894                 cpu_disable_c3_sleep--;
  895         periodic = want_periodic;
  896         timer = et;
  897         et_init(timer, timercb, NULL, NULL);
  898         configtimer(1);
  899         ET_UNLOCK();
  900         return (error);
  901 }
  902 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
  903     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
  904     0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
  905 
  906 /*
  907  * Report or change the active event timer periodicity.
  908  */
  909 static int
  910 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
  911 {
  912         int error, val;
  913 
  914         val = periodic;
  915         error = sysctl_handle_int(oidp, &val, 0, req);
  916         if (error != 0 || req->newptr == NULL)
  917                 return (error);
  918         ET_LOCK();
  919         configtimer(0);
  920         periodic = want_periodic = val;
  921         configtimer(1);
  922         ET_UNLOCK();
  923         return (error);
  924 }
  925 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
  926     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
  927     0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
  928 
  929 #include "opt_ddb.h"
  930 
  931 #ifdef DDB
  932 #include <ddb/ddb.h>
  933 
  934 DB_SHOW_COMMAND(clocksource, db_show_clocksource)
  935 {
  936         struct pcpu_state *st;
  937         int c;
  938 
  939         CPU_FOREACH(c) {
  940                 st = DPCPU_ID_PTR(c, timerstate);
  941                 db_printf(
  942                     "CPU %2d: action %d handle %d  ipi %d idle %d\n"
  943                     "        now %#jx nevent %#jx (%jd)\n"
  944                     "        ntick %#jx (%jd) nhard %#jx (%jd)\n"
  945                     "        nstat %#jx (%jd) nprof %#jx (%jd)\n"
  946                     "        ncall %#jx (%jd) ncallopt %#jx (%jd)\n",
  947                     c, st->action, st->handle, st->ipi, st->idle,
  948                     (uintmax_t)st->now,
  949                     (uintmax_t)st->nextevent,
  950                     (uintmax_t)(st->nextevent - st->now) / tick_sbt,
  951                     (uintmax_t)st->nexttick,
  952                     (uintmax_t)(st->nexttick - st->now) / tick_sbt,
  953                     (uintmax_t)st->nexthard,
  954                     (uintmax_t)(st->nexthard - st->now) / tick_sbt,
  955                     (uintmax_t)st->nextstat,
  956                     (uintmax_t)(st->nextstat - st->now) / tick_sbt,
  957                     (uintmax_t)st->nextprof,
  958                     (uintmax_t)(st->nextprof - st->now) / tick_sbt,
  959                     (uintmax_t)st->nextcall,
  960                     (uintmax_t)(st->nextcall - st->now) / tick_sbt,
  961                     (uintmax_t)st->nextcallopt,
  962                     (uintmax_t)(st->nextcallopt - st->now) / tick_sbt);
  963         }
  964 }
  965 
  966 #endif
Cache object: 42cff516782568a2cc624ec589f1a6f1
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/kern_clocksource.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_clocksource.c
