kern_clocksource.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer,
   10  *    without modification, immediately at the beginning of the file.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD: releng/10.3/sys/kern/kern_clocksource.c 282748 2015-05-11 07:54:39Z avg $");
   29 
   30 /*
   31  * Common routines to manage event timers hardware.
   32  */
   33 
   34 #include "opt_device_polling.h"
   35 #include "opt_kdtrace.h"
   36 
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/bus.h>
   40 #include <sys/limits.h>
   41 #include <sys/lock.h>
   42 #include <sys/kdb.h>
   43 #include <sys/ktr.h>
   44 #include <sys/mutex.h>
   45 #include <sys/proc.h>
   46 #include <sys/kernel.h>
   47 #include <sys/sched.h>
   48 #include <sys/smp.h>
   49 #include <sys/sysctl.h>
   50 #include <sys/timeet.h>
   51 #include <sys/timetc.h>
   52 
   53 #include <machine/atomic.h>
   54 #include <machine/clock.h>
   55 #include <machine/cpu.h>
   56 #include <machine/smp.h>
   57 
   58 int                     cpu_deepest_sleep = 0;  /* Deepest Cx state available. */
   59 int                     cpu_disable_c2_sleep = 0; /* Timer dies in C2. */
   60 int                     cpu_disable_c3_sleep = 0; /* Timer dies in C3. */
   61 
   62 static void             setuptimer(void);
   63 static void             loadtimer(sbintime_t now, int first);
   64 static int              doconfigtimer(void);
   65 static void             configtimer(int start);
   66 static int              round_freq(struct eventtimer *et, int freq);
   67 
   68 static sbintime_t       getnextcpuevent(int idle);
   69 static sbintime_t       getnextevent(void);
   70 static int              handleevents(sbintime_t now, int fake);
   71 
   72 static struct mtx       et_hw_mtx;
   73 
   74 #define ET_HW_LOCK(state)                                               \
   75         {                                                               \
   76                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
   77                         mtx_lock_spin(&(state)->et_hw_mtx);             \
   78                 else                                                    \
   79                         mtx_lock_spin(&et_hw_mtx);                      \
   80         }
   81 
   82 #define ET_HW_UNLOCK(state)                                             \
   83         {                                                               \
   84                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
   85                         mtx_unlock_spin(&(state)->et_hw_mtx);           \
   86                 else                                                    \
   87                         mtx_unlock_spin(&et_hw_mtx);                    \
   88         }
   89 
   90 static struct eventtimer *timer = NULL;
   91 static sbintime_t       timerperiod;    /* Timer period for periodic mode. */
   92 static sbintime_t       statperiod;     /* statclock() events period. */
   93 static sbintime_t       profperiod;     /* profclock() events period. */
   94 static sbintime_t       nexttick;       /* Next global timer tick time. */
   95 static u_int            busy = 1;       /* Reconfiguration is in progress. */
   96 static int              profiling = 0;  /* Profiling events enabled. */
   97 
   98 static char             timername[32];  /* Wanted timer. */
   99 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
  100 
  101 static int              singlemul = 0;  /* Multiplier for periodic mode. */
  102 TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
  103 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
  104     0, "Multiplier for periodic mode");
  105 
  106 static u_int            idletick = 0;   /* Run periodic events when idle. */
  107 TUNABLE_INT("kern.eventtimer.idletick", &idletick);
  108 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RW, &idletick,
  109     0, "Run periodic events when idle");
  110 
  111 static int              periodic = 0;   /* Periodic or one-shot mode. */
  112 static int              want_periodic = 0; /* What mode to prefer. */
  113 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
  114 
  115 struct pcpu_state {
  116         struct mtx      et_hw_mtx;      /* Per-CPU timer mutex. */
  117         u_int           action;         /* Reconfiguration requests. */
  118         u_int           handle;         /* Immediate handle resuests. */
  119         sbintime_t      now;            /* Last tick time. */
  120         sbintime_t      nextevent;      /* Next scheduled event on this CPU. */
  121         sbintime_t      nexttick;       /* Next timer tick time. */
  122         sbintime_t      nexthard;       /* Next hardlock() event. */
  123         sbintime_t      nextstat;       /* Next statclock() event. */
  124         sbintime_t      nextprof;       /* Next profclock() event. */
  125         sbintime_t      nextcall;       /* Next callout event. */
  126         sbintime_t      nextcallopt;    /* Next optional callout event. */
  127         int             ipi;            /* This CPU needs IPI. */
  128         int             idle;           /* This CPU is in idle mode. */
  129 };
  130 
  131 static DPCPU_DEFINE(struct pcpu_state, timerstate);
  132 DPCPU_DEFINE(sbintime_t, hardclocktime);
  133 
  134 /*
  135  * Timer broadcast IPI handler.
  136  */
  137 int
  138 hardclockintr(void)
  139 {
  140         sbintime_t now;
  141         struct pcpu_state *state;
  142         int done;
  143 
  144         if (doconfigtimer() || busy)
  145                 return (FILTER_HANDLED);
  146         state = DPCPU_PTR(timerstate);
  147         now = state->now;
  148         CTR3(KTR_SPARE2, "ipi  at %d:    now  %d.%08x",
  149             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  150         done = handleevents(now, 0);
  151         return (done ? FILTER_HANDLED : FILTER_STRAY);
  152 }
  153 
  154 /*
  155  * Handle all events for specified time on this CPU
  156  */
  157 static int
  158 handleevents(sbintime_t now, int fake)
  159 {
  160         sbintime_t t, *hct;
  161         struct trapframe *frame;
  162         struct pcpu_state *state;
  163         int usermode;
  164         int done, runs;
  165 
  166         CTR3(KTR_SPARE2, "handle at %d:  now  %d.%08x",
  167             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  168         done = 0;
  169         if (fake) {
  170                 frame = NULL;
  171                 usermode = 0;
  172         } else {
  173                 frame = curthread->td_intr_frame;
  174                 usermode = TRAPF_USERMODE(frame);
  175         }
  176 
  177         state = DPCPU_PTR(timerstate);
  178 
  179         runs = 0;
  180         while (now >= state->nexthard) {
  181                 state->nexthard += tick_sbt;
  182                 runs++;
  183         }
  184         if (runs) {
  185                 hct = DPCPU_PTR(hardclocktime);
  186                 *hct = state->nexthard - tick_sbt;
  187                 if (fake < 2) {
  188                         hardclock_cnt(runs, usermode);
  189                         done = 1;
  190                 }
  191         }
  192         runs = 0;
  193         while (now >= state->nextstat) {
  194                 state->nextstat += statperiod;
  195                 runs++;
  196         }
  197         if (runs && fake < 2) {
  198                 statclock_cnt(runs, usermode);
  199                 done = 1;
  200         }
  201         if (profiling) {
  202                 runs = 0;
  203                 while (now >= state->nextprof) {
  204                         state->nextprof += profperiod;
  205                         runs++;
  206                 }
  207                 if (runs && !fake) {
  208                         profclock_cnt(runs, usermode, TRAPF_PC(frame));
  209                         done = 1;
  210                 }
  211         } else
  212                 state->nextprof = state->nextstat;
  213         if (now >= state->nextcallopt) {
  214                 state->nextcall = state->nextcallopt = INT64_MAX;
  215                 callout_process(now);
  216         }
  217 
  218         t = getnextcpuevent(0);
  219         ET_HW_LOCK(state);
  220         if (!busy) {
  221                 state->idle = 0;
  222                 state->nextevent = t;
  223                 loadtimer(now, (fake == 2) &&
  224                     (timer->et_flags & ET_FLAGS_PERCPU));
  225         }
  226         ET_HW_UNLOCK(state);
  227         return (done);
  228 }
  229 
  230 /*
  231  * Schedule binuptime of the next event on current CPU.
  232  */
  233 static sbintime_t
  234 getnextcpuevent(int idle)
  235 {
  236         sbintime_t event;
  237         struct pcpu_state *state;
  238         u_int hardfreq;
  239 
  240         state = DPCPU_PTR(timerstate);
  241         /* Handle hardclock() events, skipping some if CPU is idle. */
  242         event = state->nexthard;
  243         if (idle) {
  244                 hardfreq = (u_int)hz / 2;
  245                 if (tc_min_ticktock_freq > 2
  246 #ifdef SMP
  247                     && curcpu == CPU_FIRST()
  248 #endif
  249                     )
  250                         hardfreq = hz / tc_min_ticktock_freq;
  251                 if (hardfreq > 1)
  252                         event += tick_sbt * (hardfreq - 1);
  253         }
  254         /* Handle callout events. */
  255         if (event > state->nextcall)
  256                 event = state->nextcall;
  257         if (!idle) { /* If CPU is active - handle other types of events. */
  258                 if (event > state->nextstat)
  259                         event = state->nextstat;
  260                 if (profiling && event > state->nextprof)
  261                         event = state->nextprof;
  262         }
  263         return (event);
  264 }
  265 
  266 /*
  267  * Schedule binuptime of the next event on all CPUs.
  268  */
  269 static sbintime_t
  270 getnextevent(void)
  271 {
  272         struct pcpu_state *state;
  273         sbintime_t event;
  274 #ifdef SMP
  275         int     cpu;
  276 #endif
  277         int     c;
  278 
  279         state = DPCPU_PTR(timerstate);
  280         event = state->nextevent;
  281         c = -1;
  282 #ifdef SMP
  283         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
  284                 CPU_FOREACH(cpu) {
  285                         state = DPCPU_ID_PTR(cpu, timerstate);
  286                         if (event > state->nextevent) {
  287                                 event = state->nextevent;
  288                                 c = cpu;
  289                         }
  290                 }
  291         }
  292 #endif
  293         CTR4(KTR_SPARE2, "next at %d:    next %d.%08x by %d",
  294             curcpu, (int)(event >> 32), (u_int)(event & 0xffffffff), c);
  295         return (event);
  296 }
  297 
  298 /* Hardware timer callback function. */
  299 static void
  300 timercb(struct eventtimer *et, void *arg)
  301 {
  302         sbintime_t now;
  303         sbintime_t *next;
  304         struct pcpu_state *state;
  305 #ifdef SMP
  306         int cpu, bcast;
  307 #endif
  308 
  309         /* Do not touch anything if somebody reconfiguring timers. */
  310         if (busy)
  311                 return;
  312         /* Update present and next tick times. */
  313         state = DPCPU_PTR(timerstate);
  314         if (et->et_flags & ET_FLAGS_PERCPU) {
  315                 next = &state->nexttick;
  316         } else
  317                 next = &nexttick;
  318         now = sbinuptime();
  319         if (periodic)
  320                 *next = now + timerperiod;
  321         else
  322                 *next = -1;     /* Next tick is not scheduled yet. */
  323         state->now = now;
  324         CTR3(KTR_SPARE2, "intr at %d:    now  %d.%08x",
  325             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  326 
  327 #ifdef SMP
  328         /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
  329         bcast = 0;
  330         if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
  331                 CPU_FOREACH(cpu) {
  332                         state = DPCPU_ID_PTR(cpu, timerstate);
  333                         ET_HW_LOCK(state);
  334                         state->now = now;
  335                         if (now >= state->nextevent) {
  336                                 state->nextevent += SBT_1S;
  337                                 if (curcpu != cpu) {
  338                                         state->ipi = 1;
  339                                         bcast = 1;
  340                                 }
  341                         }
  342                         ET_HW_UNLOCK(state);
  343                 }
  344         }
  345 #endif
  346 
  347         /* Handle events for this time on this CPU. */
  348         handleevents(now, 0);
  349 
  350 #ifdef SMP
  351         /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
  352         if (bcast) {
  353                 CPU_FOREACH(cpu) {
  354                         if (curcpu == cpu)
  355                                 continue;
  356                         state = DPCPU_ID_PTR(cpu, timerstate);
  357                         if (state->ipi) {
  358                                 state->ipi = 0;
  359                                 ipi_cpu(cpu, IPI_HARDCLOCK);
  360                         }
  361                 }
  362         }
  363 #endif
  364 }
  365 
  366 /*
  367  * Load new value into hardware timer.
  368  */
  369 static void
  370 loadtimer(sbintime_t now, int start)
  371 {
  372         struct pcpu_state *state;
  373         sbintime_t new;
  374         sbintime_t *next;
  375         uint64_t tmp;
  376         int eq;
  377 
  378         if (timer->et_flags & ET_FLAGS_PERCPU) {
  379                 state = DPCPU_PTR(timerstate);
  380                 next = &state->nexttick;
  381         } else
  382                 next = &nexttick;
  383         if (periodic) {
  384                 if (start) {
  385                         /*
  386                          * Try to start all periodic timers aligned
  387                          * to period to make events synchronous.
  388                          */
  389                         tmp = now % timerperiod;
  390                         new = timerperiod - tmp;
  391                         if (new < tmp)          /* Left less then passed. */
  392                                 new += timerperiod;
  393                         CTR5(KTR_SPARE2, "load p at %d:   now %d.%08x first in %d.%08x",
  394                             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff),
  395                             (int)(new >> 32), (u_int)(new & 0xffffffff));
  396                         *next = new + now;
  397                         et_start(timer, new, timerperiod);
  398                 }
  399         } else {
  400                 new = getnextevent();
  401                 eq = (new == *next);
  402                 CTR4(KTR_SPARE2, "load at %d:    next %d.%08x eq %d",
  403                     curcpu, (int)(new >> 32), (u_int)(new & 0xffffffff), eq);
  404                 if (!eq) {
  405                         *next = new;
  406                         et_start(timer, new - now, 0);
  407                 }
  408         }
  409 }
  410 
  411 /*
  412  * Prepare event timer parameters after configuration changes.
  413  */
  414 static void
  415 setuptimer(void)
  416 {
  417         int freq;
  418 
  419         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
  420                 periodic = 0;
  421         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
  422                 periodic = 1;
  423         singlemul = MIN(MAX(singlemul, 1), 20);
  424         freq = hz * singlemul;
  425         while (freq < (profiling ? profhz : stathz))
  426                 freq += hz;
  427         freq = round_freq(timer, freq);
  428         timerperiod = SBT_1S / freq;
  429 }
  430 
  431 /*
  432  * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
  433  */
  434 static int
  435 doconfigtimer(void)
  436 {
  437         sbintime_t now;
  438         struct pcpu_state *state;
  439 
  440         state = DPCPU_PTR(timerstate);
  441         switch (atomic_load_acq_int(&state->action)) {
  442         case 1:
  443                 now = sbinuptime();
  444                 ET_HW_LOCK(state);
  445                 loadtimer(now, 1);
  446                 ET_HW_UNLOCK(state);
  447                 state->handle = 0;
  448                 atomic_store_rel_int(&state->action, 0);
  449                 return (1);
  450         case 2:
  451                 ET_HW_LOCK(state);
  452                 et_stop(timer);
  453                 ET_HW_UNLOCK(state);
  454                 state->handle = 0;
  455                 atomic_store_rel_int(&state->action, 0);
  456                 return (1);
  457         }
  458         if (atomic_readandclear_int(&state->handle) && !busy) {
  459                 now = sbinuptime();
  460                 handleevents(now, 0);
  461                 return (1);
  462         }
  463         return (0);
  464 }
  465 
  466 /*
  467  * Reconfigure specified timer.
  468  * For per-CPU timers use IPI to make other CPUs to reconfigure.
  469  */
  470 static void
  471 configtimer(int start)
  472 {
  473         sbintime_t now, next;
  474         struct pcpu_state *state;
  475         int cpu;
  476 
  477         if (start) {
  478                 setuptimer();
  479                 now = sbinuptime();
  480         } else
  481                 now = 0;
  482         critical_enter();
  483         ET_HW_LOCK(DPCPU_PTR(timerstate));
  484         if (start) {
  485                 /* Initialize time machine parameters. */
  486                 next = now + timerperiod;
  487                 if (periodic)
  488                         nexttick = next;
  489                 else
  490                         nexttick = -1;
  491                 CPU_FOREACH(cpu) {
  492                         state = DPCPU_ID_PTR(cpu, timerstate);
  493                         state->now = now;
  494                         if (!smp_started && cpu != CPU_FIRST())
  495                                 state->nextevent = INT64_MAX;
  496                         else
  497                                 state->nextevent = next;
  498                         if (periodic)
  499                                 state->nexttick = next;
  500                         else
  501                                 state->nexttick = -1;
  502                         state->nexthard = next;
  503                         state->nextstat = next;
  504                         state->nextprof = next;
  505                         state->nextcall = next;
  506                         state->nextcallopt = next;
  507                         hardclock_sync(cpu);
  508                 }
  509                 busy = 0;
  510                 /* Start global timer or per-CPU timer of this CPU. */
  511                 loadtimer(now, 1);
  512         } else {
  513                 busy = 1;
  514                 /* Stop global timer or per-CPU timer of this CPU. */
  515                 et_stop(timer);
  516         }
  517         ET_HW_UNLOCK(DPCPU_PTR(timerstate));
  518 #ifdef SMP
  519         /* If timer is global or there is no other CPUs yet - we are done. */
  520         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
  521                 critical_exit();
  522                 return;
  523         }
  524         /* Set reconfigure flags for other CPUs. */
  525         CPU_FOREACH(cpu) {
  526                 state = DPCPU_ID_PTR(cpu, timerstate);
  527                 atomic_store_rel_int(&state->action,
  528                     (cpu == curcpu) ? 0 : ( start ? 1 : 2));
  529         }
  530         /* Broadcast reconfigure IPI. */
  531         ipi_all_but_self(IPI_HARDCLOCK);
  532         /* Wait for reconfiguration completed. */
  533 restart:
  534         cpu_spinwait();
  535         CPU_FOREACH(cpu) {
  536                 if (cpu == curcpu)
  537                         continue;
  538                 state = DPCPU_ID_PTR(cpu, timerstate);
  539                 if (atomic_load_acq_int(&state->action))
  540                         goto restart;
  541         }
  542 #endif
  543         critical_exit();
  544 }
  545 
  546 /*
  547  * Calculate nearest frequency supported by hardware timer.
  548  */
  549 static int
  550 round_freq(struct eventtimer *et, int freq)
  551 {
  552         uint64_t div;
  553 
  554         if (et->et_frequency != 0) {
  555                 div = lmax((et->et_frequency + freq / 2) / freq, 1);
  556                 if (et->et_flags & ET_FLAGS_POW2DIV)
  557                         div = 1 << (flsl(div + div / 2) - 1);
  558                 freq = (et->et_frequency + div / 2) / div;
  559         }
  560         if (et->et_min_period > SBT_1S)
  561                 panic("Event timer \"%s\" doesn't support sub-second periods!",
  562                     et->et_name);
  563         else if (et->et_min_period != 0)
  564                 freq = min(freq, SBT2FREQ(et->et_min_period));
  565         if (et->et_max_period < SBT_1S && et->et_max_period != 0)
  566                 freq = max(freq, SBT2FREQ(et->et_max_period));
  567         return (freq);
  568 }
  569 
  570 /*
  571  * Configure and start event timers (BSP part).
  572  */
  573 void
  574 cpu_initclocks_bsp(void)
  575 {
  576         struct pcpu_state *state;
  577         int base, div, cpu;
  578 
  579         mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
  580         CPU_FOREACH(cpu) {
  581                 state = DPCPU_ID_PTR(cpu, timerstate);
  582                 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
  583                 state->nextcall = INT64_MAX;
  584                 state->nextcallopt = INT64_MAX;
  585         }
  586         periodic = want_periodic;
  587         /* Grab requested timer or the best of present. */
  588         if (timername[0])
  589                 timer = et_find(timername, 0, 0);
  590         if (timer == NULL && periodic) {
  591                 timer = et_find(NULL,
  592                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
  593         }
  594         if (timer == NULL) {
  595                 timer = et_find(NULL,
  596                     ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
  597         }
  598         if (timer == NULL && !periodic) {
  599                 timer = et_find(NULL,
  600                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
  601         }
  602         if (timer == NULL)
  603                 panic("No usable event timer found!");
  604         et_init(timer, timercb, NULL, NULL);
  605 
  606         /* Adapt to timer capabilities. */
  607         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
  608                 periodic = 0;
  609         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
  610                 periodic = 1;
  611         if (timer->et_flags & ET_FLAGS_C3STOP)
  612                 cpu_disable_c3_sleep++;
  613 
  614         /*
  615          * We honor the requested 'hz' value.
  616          * We want to run stathz in the neighborhood of 128hz.
  617          * We would like profhz to run as often as possible.
  618          */
  619         if (singlemul <= 0 || singlemul > 20) {
  620                 if (hz >= 1500 || (hz % 128) == 0)
  621                         singlemul = 1;
  622                 else if (hz >= 750)
  623                         singlemul = 2;
  624                 else
  625                         singlemul = 4;
  626         }
  627         if (periodic) {
  628                 base = round_freq(timer, hz * singlemul);
  629                 singlemul = max((base + hz / 2) / hz, 1);
  630                 hz = (base + singlemul / 2) / singlemul;
  631                 if (base <= 128)
  632                         stathz = base;
  633                 else {
  634                         div = base / 128;
  635                         if (div >= singlemul && (div % singlemul) == 0)
  636                                 div++;
  637                         stathz = base / div;
  638                 }
  639                 profhz = stathz;
  640                 while ((profhz + stathz) <= 128 * 64)
  641                         profhz += stathz;
  642                 profhz = round_freq(timer, profhz);
  643         } else {
  644                 hz = round_freq(timer, hz);
  645                 stathz = round_freq(timer, 127);
  646                 profhz = round_freq(timer, stathz * 64);
  647         }
  648         tick = 1000000 / hz;
  649         tick_sbt = SBT_1S / hz;
  650         tick_bt = sbttobt(tick_sbt);
  651         statperiod = SBT_1S / stathz;
  652         profperiod = SBT_1S / profhz;
  653         ET_LOCK();
  654         configtimer(1);
  655         ET_UNLOCK();
  656 }
  657 
  658 /*
  659  * Start per-CPU event timers on APs.
  660  */
  661 void
  662 cpu_initclocks_ap(void)
  663 {
  664         sbintime_t now;
  665         struct pcpu_state *state;
  666         struct thread *td;
  667 
  668         state = DPCPU_PTR(timerstate);
  669         now = sbinuptime();
  670         ET_HW_LOCK(state);
  671         state->now = now;
  672         hardclock_sync(curcpu);
  673         spinlock_enter();
  674         ET_HW_UNLOCK(state);
  675         td = curthread;
  676         td->td_intr_nesting_level++;
  677         handleevents(state->now, 2);
  678         td->td_intr_nesting_level--;
  679         spinlock_exit();
  680 }
  681 
  682 /*
  683  * Switch to profiling clock rates.
  684  */
  685 void
  686 cpu_startprofclock(void)
  687 {
  688 
  689         ET_LOCK();
  690         if (profiling == 0) {
  691                 if (periodic) {
  692                         configtimer(0);
  693                         profiling = 1;
  694                         configtimer(1);
  695                 } else
  696                         profiling = 1;
  697         } else
  698                 profiling++;
  699         ET_UNLOCK();
  700 }
  701 
  702 /*
  703  * Switch to regular clock rates.
  704  */
  705 void
  706 cpu_stopprofclock(void)
  707 {
  708 
  709         ET_LOCK();
  710         if (profiling == 1) {
  711                 if (periodic) {
  712                         configtimer(0);
  713                         profiling = 0;
  714                         configtimer(1);
  715                 } else
  716                 profiling = 0;
  717         } else
  718                 profiling--;
  719         ET_UNLOCK();
  720 }
  721 
  722 /*
  723  * Switch to idle mode (all ticks handled).
  724  */
  725 sbintime_t
  726 cpu_idleclock(void)
  727 {
  728         sbintime_t now, t;
  729         struct pcpu_state *state;
  730 
  731         if (idletick || busy ||
  732             (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
  733 #ifdef DEVICE_POLLING
  734             || curcpu == CPU_FIRST()
  735 #endif
  736             )
  737                 return (-1);
  738         state = DPCPU_PTR(timerstate);
  739         if (periodic)
  740                 now = state->now;
  741         else
  742                 now = sbinuptime();
  743         CTR3(KTR_SPARE2, "idle at %d:    now  %d.%08x",
  744             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  745         t = getnextcpuevent(1);
  746         ET_HW_LOCK(state);
  747         state->idle = 1;
  748         state->nextevent = t;
  749         if (!periodic)
  750                 loadtimer(now, 0);
  751         ET_HW_UNLOCK(state);
  752         return (MAX(t - now, 0));
  753 }
  754 
  755 /*
  756  * Switch to active mode (skip empty ticks).
  757  */
  758 void
  759 cpu_activeclock(void)
  760 {
  761         sbintime_t now;
  762         struct pcpu_state *state;
  763         struct thread *td;
  764 
  765         state = DPCPU_PTR(timerstate);
  766         if (state->idle == 0 || busy)
  767                 return;
  768         if (periodic)
  769                 now = state->now;
  770         else
  771                 now = sbinuptime();
  772         CTR3(KTR_SPARE2, "active at %d:  now  %d.%08x",
  773             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
  774         spinlock_enter();
  775         td = curthread;
  776         td->td_intr_nesting_level++;
  777         handleevents(now, 1);
  778         td->td_intr_nesting_level--;
  779         spinlock_exit();
  780 }
  781 
  782 /*
  783  * Change the frequency of the given timer.  This changes et->et_frequency and
  784  * if et is the active timer it reconfigures the timer on all CPUs.  This is
  785  * intended to be a private interface for the use of et_change_frequency() only.
  786  */
  787 void
  788 cpu_et_frequency(struct eventtimer *et, uint64_t newfreq)
  789 {
  790 
  791         ET_LOCK();
  792         if (et == timer) {
  793                 configtimer(0);
  794                 et->et_frequency = newfreq;
  795                 configtimer(1);
  796         } else
  797                 et->et_frequency = newfreq;
  798         ET_UNLOCK();
  799 }
  800 
  801 void
  802 cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt)
  803 {
  804         struct pcpu_state *state;
  805 
  806         /* Do not touch anything if somebody reconfiguring timers. */
  807         if (busy)
  808                 return;
  809         CTR6(KTR_SPARE2, "new co at %d:    on %d at %d.%08x - %d.%08x",
  810             curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff),
  811             (int)(bt >> 32), (u_int)(bt & 0xffffffff));
  812         state = DPCPU_ID_PTR(cpu, timerstate);
  813         ET_HW_LOCK(state);
  814 
  815         /*
  816          * If there is callout time already set earlier -- do nothing.
  817          * This check may appear redundant because we check already in
  818          * callout_process() but this double check guarantees we're safe
  819          * with respect to race conditions between interrupts execution
  820          * and scheduling.
  821          */
  822         state->nextcallopt = bt_opt;
  823         if (bt >= state->nextcall)
  824                 goto done;
  825         state->nextcall = bt;
  826         /* If there is some other event set earlier -- do nothing. */
  827         if (bt >= state->nextevent)
  828                 goto done;
  829         state->nextevent = bt;
  830         /* If timer is periodic -- there is nothing to reprogram. */
  831         if (periodic)
  832                 goto done;
  833         /* If timer is global or of the current CPU -- reprogram it. */
  834         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) {
  835                 loadtimer(sbinuptime(), 0);
  836 done:
  837                 ET_HW_UNLOCK(state);
  838                 return;
  839         }
  840         /* Otherwise make other CPU to reprogram it. */
  841         state->handle = 1;
  842         ET_HW_UNLOCK(state);
  843 #ifdef SMP
  844         ipi_cpu(cpu, IPI_HARDCLOCK);
  845 #endif
  846 }
  847 
  848 /*
  849  * Report or change the active event timers hardware.
  850  */
  851 static int
  852 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
  853 {
  854         char buf[32];
  855         struct eventtimer *et;
  856         int error;
  857 
  858         ET_LOCK();
  859         et = timer;
  860         snprintf(buf, sizeof(buf), "%s", et->et_name);
  861         ET_UNLOCK();
  862         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
  863         ET_LOCK();
  864         et = timer;
  865         if (error != 0 || req->newptr == NULL ||
  866             strcasecmp(buf, et->et_name) == 0) {
  867                 ET_UNLOCK();
  868                 return (error);
  869         }
  870         et = et_find(buf, 0, 0);
  871         if (et == NULL) {
  872                 ET_UNLOCK();
  873                 return (ENOENT);
  874         }
  875         configtimer(0);
  876         et_free(timer);
  877         if (et->et_flags & ET_FLAGS_C3STOP)
  878                 cpu_disable_c3_sleep++;
  879         if (timer->et_flags & ET_FLAGS_C3STOP)
  880                 cpu_disable_c3_sleep--;
  881         periodic = want_periodic;
  882         timer = et;
  883         et_init(timer, timercb, NULL, NULL);
  884         configtimer(1);
  885         ET_UNLOCK();
  886         return (error);
  887 }
  888 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
  889     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
  890     0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
  891 
  892 /*
  893  * Report or change the active event timer periodicity.
  894  */
  895 static int
  896 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
  897 {
  898         int error, val;
  899 
  900         val = periodic;
  901         error = sysctl_handle_int(oidp, &val, 0, req);
  902         if (error != 0 || req->newptr == NULL)
  903                 return (error);
  904         ET_LOCK();
  905         configtimer(0);
  906         periodic = want_periodic = val;
  907         configtimer(1);
  908         ET_UNLOCK();
  909         return (error);
  910 }
  911 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
  912     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
  913     0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
  914 
  915 #include "opt_ddb.h"
  916 
  917 #ifdef DDB
  918 #include <ddb/ddb.h>
  919 
  920 DB_SHOW_COMMAND(clocksource, db_show_clocksource)
  921 {
  922         struct pcpu_state *st;
  923         int c;
  924 
  925         CPU_FOREACH(c) {
  926                 st = DPCPU_ID_PTR(c, timerstate);
  927                 db_printf(
  928                     "CPU %2d: action %d handle %d  ipi %d idle %d\n"
  929                     "        now %#jx nevent %#jx (%jd)\n"
  930                     "        ntick %#jx (%jd) nhard %#jx (%jd)\n"
  931                     "        nstat %#jx (%jd) nprof %#jx (%jd)\n"
  932                     "        ncall %#jx (%jd) ncallopt %#jx (%jd)\n",
  933                     c, st->action, st->handle, st->ipi, st->idle,
  934                     (uintmax_t)st->now,
  935                     (uintmax_t)st->nextevent,
  936                     (uintmax_t)(st->nextevent - st->now) / tick_sbt,
  937                     (uintmax_t)st->nexttick,
  938                     (uintmax_t)(st->nexttick - st->now) / tick_sbt,
  939                     (uintmax_t)st->nexthard,
  940                     (uintmax_t)(st->nexthard - st->now) / tick_sbt,
  941                     (uintmax_t)st->nextstat,
  942                     (uintmax_t)(st->nextstat - st->now) / tick_sbt,
  943                     (uintmax_t)st->nextprof,
  944                     (uintmax_t)(st->nextprof - st->now) / tick_sbt,
  945                     (uintmax_t)st->nextcall,
  946                     (uintmax_t)(st->nextcall - st->now) / tick_sbt,
  947                     (uintmax_t)st->nextcallopt,
  948                     (uintmax_t)(st->nextcallopt - st->now) / tick_sbt);
  949         }
  950 }
  951 
  952 #endif
Cache object: 054fcd512d77d44d2e129bdb7d2a3839
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/kern_clocksource.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_clocksource.c
