The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_tc.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: Beerware
    3  *
    4  * ----------------------------------------------------------------------------
    5  * "THE BEER-WARE LICENSE" (Revision 42):
    6  * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
    7  * can do whatever you want with this stuff. If we meet some day, and you think
    8  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
    9  * ----------------------------------------------------------------------------
   10  *
   11  * Copyright (c) 2011, 2015, 2016 The FreeBSD Foundation
   12  *
   13  * Portions of this software were developed by Julien Ridoux at the University
   14  * of Melbourne under sponsorship from the FreeBSD Foundation.
   15  *
   16  * Portions of this software were developed by Konstantin Belousov
   17  * under sponsorship from the FreeBSD Foundation.
   18  */
   19 
   20 #include <sys/cdefs.h>
   21 __FBSDID("$FreeBSD$");
   22 
   23 #include "opt_ntp.h"
   24 #include "opt_ffclock.h"
   25 
   26 #include <sys/param.h>
   27 #include <sys/kernel.h>
   28 #include <sys/limits.h>
   29 #include <sys/lock.h>
   30 #include <sys/mutex.h>
   31 #include <sys/proc.h>
   32 #include <sys/sbuf.h>
   33 #include <sys/sleepqueue.h>
   34 #include <sys/sysctl.h>
   35 #include <sys/syslog.h>
   36 #include <sys/systm.h>
   37 #include <sys/timeffc.h>
   38 #include <sys/timepps.h>
   39 #include <sys/timetc.h>
   40 #include <sys/timex.h>
   41 #include <sys/vdso.h>
   42 
   43 /*
   44  * A large step happens on boot.  This constant detects such steps.
   45  * It is relatively small so that ntp_update_second gets called enough
   46  * in the typical 'missed a couple of seconds' case, but doesn't loop
   47  * forever when the time step is large.
   48  */
   49 #define LARGE_STEP      200
   50 
   51 /*
   52  * Implement a dummy timecounter which we can use until we get a real one
   53  * in the air.  This allows the console and other early stuff to use
   54  * time services.
   55  */
   56 
   57 static u_int
   58 dummy_get_timecount(struct timecounter *tc)
   59 {
   60         static u_int now;
   61 
   62         return (++now);
   63 }
   64 
   65 static struct timecounter dummy_timecounter = {
   66         dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000
   67 };
   68 
   69 struct timehands {
   70         /* These fields must be initialized by the driver. */
   71         struct timecounter      *th_counter;
   72         int64_t                 th_adjustment;
   73         uint64_t                th_scale;
   74         u_int                   th_large_delta;
   75         u_int                   th_offset_count;
   76         struct bintime          th_offset;
   77         struct bintime          th_bintime;
   78         struct timeval          th_microtime;
   79         struct timespec         th_nanotime;
   80         struct bintime          th_boottime;
   81         /* Fields not to be copied in tc_windup start with th_generation. */
   82         u_int                   th_generation;
   83         struct timehands        *th_next;
   84 };
   85 
   86 static struct timehands ths[16] = {
   87     [0] =  {
   88         .th_counter = &dummy_timecounter,
   89         .th_scale = (uint64_t)-1 / 1000000,
   90         .th_large_delta = 1000000,
   91         .th_offset = { .sec = 1 },
   92         .th_generation = 1,
   93     },
   94 };
   95 
   96 static struct timehands *volatile timehands = &ths[0];
   97 struct timecounter *timecounter = &dummy_timecounter;
   98 static struct timecounter *timecounters = &dummy_timecounter;
   99 
  100 /* Mutex to protect the timecounter list. */
  101 static struct mtx tc_lock;
  102 
  103 int tc_min_ticktock_freq = 1;
  104 
  105 volatile time_t time_second = 1;
  106 volatile time_t time_uptime = 1;
  107 
  108 static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS);
  109 SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime,
  110     CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
  111     sysctl_kern_boottime, "S,timeval",
  112     "System boottime");
  113 
  114 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
  115     "");
  116 static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc,
  117     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
  118     "");
  119 
  120 static int timestepwarnings;
  121 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RWTUN,
  122     &timestepwarnings, 0, "Log time steps");
  123 
  124 static int timehands_count = 2;
  125 SYSCTL_INT(_kern_timecounter, OID_AUTO, timehands_count,
  126     CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  127     &timehands_count, 0, "Count of timehands in rotation");
  128 
  129 struct bintime bt_timethreshold;
  130 struct bintime bt_tickthreshold;
  131 sbintime_t sbt_timethreshold;
  132 sbintime_t sbt_tickthreshold;
  133 struct bintime tc_tick_bt;
  134 sbintime_t tc_tick_sbt;
  135 int tc_precexp;
  136 int tc_timepercentage = TC_DEFAULTPERC;
  137 static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS);
  138 SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation,
  139     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0,
  140     sysctl_kern_timecounter_adjprecision, "I",
  141     "Allowed time interval deviation in percents");
  142 
  143 volatile int rtc_generation = 1;
  144 
  145 static int tc_chosen;   /* Non-zero if a specific tc was chosen via sysctl. */
  146 static char tc_from_tunable[16];
  147 
  148 static void tc_windup(struct bintime *new_boottimebin);
  149 static void cpu_tick_calibrate(int);
  150 
  151 void dtrace_getnanotime(struct timespec *tsp);
  152 void dtrace_getnanouptime(struct timespec *tsp);
  153 
  154 static int
  155 sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
  156 {
  157         struct timeval boottime;
  158 
  159         getboottime(&boottime);
  160 
  161 /* i386 is the only arch which uses a 32bits time_t */
  162 #ifdef __amd64__
  163 #ifdef SCTL_MASK32
  164         int tv[2];
  165 
  166         if (req->flags & SCTL_MASK32) {
  167                 tv[0] = boottime.tv_sec;
  168                 tv[1] = boottime.tv_usec;
  169                 return (SYSCTL_OUT(req, tv, sizeof(tv)));
  170         }
  171 #endif
  172 #endif
  173         return (SYSCTL_OUT(req, &boottime, sizeof(boottime)));
  174 }
  175 
  176 static int
  177 sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS)
  178 {
  179         u_int ncount;
  180         struct timecounter *tc = arg1;
  181 
  182         ncount = tc->tc_get_timecount(tc);
  183         return (sysctl_handle_int(oidp, &ncount, 0, req));
  184 }
  185 
  186 static int
  187 sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS)
  188 {
  189         uint64_t freq;
  190         struct timecounter *tc = arg1;
  191 
  192         freq = tc->tc_frequency;
  193         return (sysctl_handle_64(oidp, &freq, 0, req));
  194 }
  195 
  196 /*
  197  * Return the difference between the timehands' counter value now and what
  198  * was when we copied it to the timehands' offset_count.
  199  */
  200 static __inline u_int
  201 tc_delta(struct timehands *th)
  202 {
  203         struct timecounter *tc;
  204 
  205         tc = th->th_counter;
  206         return ((tc->tc_get_timecount(tc) - th->th_offset_count) &
  207             tc->tc_counter_mask);
  208 }
  209 
  210 static __inline void
  211 bintime_add_tc_delta(struct bintime *bt, uint64_t scale,
  212     uint64_t large_delta, uint64_t delta)
  213 {
  214         uint64_t x;
  215 
  216         if (__predict_false(delta >= large_delta)) {
  217                 /* Avoid overflow for scale * delta. */
  218                 x = (scale >> 32) * delta;
  219                 bt->sec += x >> 32;
  220                 bintime_addx(bt, x << 32);
  221                 bintime_addx(bt, (scale & 0xffffffff) * delta);
  222         } else {
  223                 bintime_addx(bt, scale * delta);
  224         }
  225 }
  226 
  227 /*
  228  * Functions for reading the time.  We have to loop until we are sure that
  229  * the timehands that we operated on was not updated under our feet.  See
  230  * the comment in <sys/time.h> for a description of these 12 functions.
  231  */
  232 
  233 static __inline void
  234 bintime_off(struct bintime *bt, u_int off)
  235 {
  236         struct timehands *th;
  237         struct bintime *btp;
  238         uint64_t scale;
  239         u_int delta, gen, large_delta;
  240 
  241         do {
  242                 th = timehands;
  243                 gen = atomic_load_acq_int(&th->th_generation);
  244                 btp = (struct bintime *)((vm_offset_t)th + off);
  245                 *bt = *btp;
  246                 scale = th->th_scale;
  247                 delta = tc_delta(th);
  248                 large_delta = th->th_large_delta;
  249                 atomic_thread_fence_acq();
  250         } while (gen == 0 || gen != th->th_generation);
  251 
  252         bintime_add_tc_delta(bt, scale, large_delta, delta);
  253 }
  254 #define GETTHBINTIME(dst, member)                                       \
  255 do {                                                                    \
  256         _Static_assert(_Generic(((struct timehands *)NULL)->member,     \
  257             struct bintime: 1, default: 0) == 1,                        \
  258             "struct timehands member is not of struct bintime type");   \
  259         bintime_off(dst, __offsetof(struct timehands, member));         \
  260 } while (0)
  261 
  262 static __inline void
  263 getthmember(void *out, size_t out_size, u_int off)
  264 {
  265         struct timehands *th;
  266         u_int gen;
  267 
  268         do {
  269                 th = timehands;
  270                 gen = atomic_load_acq_int(&th->th_generation);
  271                 memcpy(out, (char *)th + off, out_size);
  272                 atomic_thread_fence_acq();
  273         } while (gen == 0 || gen != th->th_generation);
  274 }
  275 #define GETTHMEMBER(dst, member)                                        \
  276 do {                                                                    \
  277         _Static_assert(_Generic(*dst,                                   \
  278             __typeof(((struct timehands *)NULL)->member): 1,            \
  279             default: 0) == 1,                                           \
  280             "*dst and struct timehands member have different types");   \
  281         getthmember(dst, sizeof(*dst), __offsetof(struct timehands,     \
  282             member));                                                   \
  283 } while (0)
  284 
  285 #ifdef FFCLOCK
  286 void
  287 fbclock_binuptime(struct bintime *bt)
  288 {
  289 
  290         GETTHBINTIME(bt, th_offset);
  291 }
  292 
  293 void
  294 fbclock_nanouptime(struct timespec *tsp)
  295 {
  296         struct bintime bt;
  297 
  298         fbclock_binuptime(&bt);
  299         bintime2timespec(&bt, tsp);
  300 }
  301 
  302 void
  303 fbclock_microuptime(struct timeval *tvp)
  304 {
  305         struct bintime bt;
  306 
  307         fbclock_binuptime(&bt);
  308         bintime2timeval(&bt, tvp);
  309 }
  310 
  311 void
  312 fbclock_bintime(struct bintime *bt)
  313 {
  314 
  315         GETTHBINTIME(bt, th_bintime);
  316 }
  317 
  318 void
  319 fbclock_nanotime(struct timespec *tsp)
  320 {
  321         struct bintime bt;
  322 
  323         fbclock_bintime(&bt);
  324         bintime2timespec(&bt, tsp);
  325 }
  326 
  327 void
  328 fbclock_microtime(struct timeval *tvp)
  329 {
  330         struct bintime bt;
  331 
  332         fbclock_bintime(&bt);
  333         bintime2timeval(&bt, tvp);
  334 }
  335 
  336 void
  337 fbclock_getbinuptime(struct bintime *bt)
  338 {
  339 
  340         GETTHMEMBER(bt, th_offset);
  341 }
  342 
  343 void
  344 fbclock_getnanouptime(struct timespec *tsp)
  345 {
  346         struct bintime bt;
  347 
  348         GETTHMEMBER(&bt, th_offset);
  349         bintime2timespec(&bt, tsp);
  350 }
  351 
  352 void
  353 fbclock_getmicrouptime(struct timeval *tvp)
  354 {
  355         struct bintime bt;
  356 
  357         GETTHMEMBER(&bt, th_offset);
  358         bintime2timeval(&bt, tvp);
  359 }
  360 
  361 void
  362 fbclock_getbintime(struct bintime *bt)
  363 {
  364 
  365         GETTHMEMBER(bt, th_bintime);
  366 }
  367 
  368 void
  369 fbclock_getnanotime(struct timespec *tsp)
  370 {
  371 
  372         GETTHMEMBER(tsp, th_nanotime);
  373 }
  374 
  375 void
  376 fbclock_getmicrotime(struct timeval *tvp)
  377 {
  378 
  379         GETTHMEMBER(tvp, th_microtime);
  380 }
  381 #else /* !FFCLOCK */
  382 
  383 void
  384 binuptime(struct bintime *bt)
  385 {
  386 
  387         GETTHBINTIME(bt, th_offset);
  388 }
  389 
  390 void
  391 nanouptime(struct timespec *tsp)
  392 {
  393         struct bintime bt;
  394 
  395         binuptime(&bt);
  396         bintime2timespec(&bt, tsp);
  397 }
  398 
  399 void
  400 microuptime(struct timeval *tvp)
  401 {
  402         struct bintime bt;
  403 
  404         binuptime(&bt);
  405         bintime2timeval(&bt, tvp);
  406 }
  407 
  408 void
  409 bintime(struct bintime *bt)
  410 {
  411 
  412         GETTHBINTIME(bt, th_bintime);
  413 }
  414 
  415 void
  416 nanotime(struct timespec *tsp)
  417 {
  418         struct bintime bt;
  419 
  420         bintime(&bt);
  421         bintime2timespec(&bt, tsp);
  422 }
  423 
  424 void
  425 microtime(struct timeval *tvp)
  426 {
  427         struct bintime bt;
  428 
  429         bintime(&bt);
  430         bintime2timeval(&bt, tvp);
  431 }
  432 
  433 void
  434 getbinuptime(struct bintime *bt)
  435 {
  436 
  437         GETTHMEMBER(bt, th_offset);
  438 }
  439 
  440 void
  441 getnanouptime(struct timespec *tsp)
  442 {
  443         struct bintime bt;
  444 
  445         GETTHMEMBER(&bt, th_offset);
  446         bintime2timespec(&bt, tsp);
  447 }
  448 
  449 void
  450 getmicrouptime(struct timeval *tvp)
  451 {
  452         struct bintime bt;
  453 
  454         GETTHMEMBER(&bt, th_offset);
  455         bintime2timeval(&bt, tvp);
  456 }
  457 
  458 void
  459 getbintime(struct bintime *bt)
  460 {
  461 
  462         GETTHMEMBER(bt, th_bintime);
  463 }
  464 
  465 void
  466 getnanotime(struct timespec *tsp)
  467 {
  468 
  469         GETTHMEMBER(tsp, th_nanotime);
  470 }
  471 
  472 void
  473 getmicrotime(struct timeval *tvp)
  474 {
  475 
  476         GETTHMEMBER(tvp, th_microtime);
  477 }
  478 #endif /* FFCLOCK */
  479 
  480 void
  481 getboottime(struct timeval *boottime)
  482 {
  483         struct bintime boottimebin;
  484 
  485         getboottimebin(&boottimebin);
  486         bintime2timeval(&boottimebin, boottime);
  487 }
  488 
  489 void
  490 getboottimebin(struct bintime *boottimebin)
  491 {
  492 
  493         GETTHMEMBER(boottimebin, th_boottime);
  494 }
  495 
  496 #ifdef FFCLOCK
  497 /*
  498  * Support for feed-forward synchronization algorithms. This is heavily inspired
  499  * by the timehands mechanism but kept independent from it. *_windup() functions
  500  * have some connection to avoid accessing the timecounter hardware more than
  501  * necessary.
  502  */
  503 
  504 /* Feed-forward clock estimates kept updated by the synchronization daemon. */
  505 struct ffclock_estimate ffclock_estimate;
  506 struct bintime ffclock_boottime;        /* Feed-forward boot time estimate. */
  507 uint32_t ffclock_status;                /* Feed-forward clock status. */
  508 int8_t ffclock_updated;                 /* New estimates are available. */
  509 struct mtx ffclock_mtx;                 /* Mutex on ffclock_estimate. */
  510 
  511 struct fftimehands {
  512         struct ffclock_estimate cest;
  513         struct bintime          tick_time;
  514         struct bintime          tick_time_lerp;
  515         ffcounter               tick_ffcount;
  516         uint64_t                period_lerp;
  517         volatile uint8_t        gen;
  518         struct fftimehands      *next;
  519 };
  520 
  521 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
  522 
  523 static struct fftimehands ffth[10];
  524 static struct fftimehands *volatile fftimehands = ffth;
  525 
  526 static void
  527 ffclock_init(void)
  528 {
  529         struct fftimehands *cur;
  530         struct fftimehands *last;
  531 
  532         memset(ffth, 0, sizeof(ffth));
  533 
  534         last = ffth + NUM_ELEMENTS(ffth) - 1;
  535         for (cur = ffth; cur < last; cur++)
  536                 cur->next = cur + 1;
  537         last->next = ffth;
  538 
  539         ffclock_updated = 0;
  540         ffclock_status = FFCLOCK_STA_UNSYNC;
  541         mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF);
  542 }
  543 
  544 /*
  545  * Reset the feed-forward clock estimates. Called from inittodr() to get things
  546  * kick started and uses the timecounter nominal frequency as a first period
  547  * estimate. Note: this function may be called several time just after boot.
  548  * Note: this is the only function that sets the value of boot time for the
  549  * monotonic (i.e. uptime) version of the feed-forward clock.
  550  */
  551 void
  552 ffclock_reset_clock(struct timespec *ts)
  553 {
  554         struct timecounter *tc;
  555         struct ffclock_estimate cest;
  556 
  557         tc = timehands->th_counter;
  558         memset(&cest, 0, sizeof(struct ffclock_estimate));
  559 
  560         timespec2bintime(ts, &ffclock_boottime);
  561         timespec2bintime(ts, &(cest.update_time));
  562         ffclock_read_counter(&cest.update_ffcount);
  563         cest.leapsec_next = 0;
  564         cest.period = ((1ULL << 63) / tc->tc_frequency) << 1;
  565         cest.errb_abs = 0;
  566         cest.errb_rate = 0;
  567         cest.status = FFCLOCK_STA_UNSYNC;
  568         cest.leapsec_total = 0;
  569         cest.leapsec = 0;
  570 
  571         mtx_lock(&ffclock_mtx);
  572         bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
  573         ffclock_updated = INT8_MAX;
  574         mtx_unlock(&ffclock_mtx);
  575 
  576         printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name,
  577             (unsigned long long)tc->tc_frequency, (long)ts->tv_sec,
  578             (unsigned long)ts->tv_nsec);
  579 }
  580 
  581 /*
  582  * Sub-routine to convert a time interval measured in RAW counter units to time
  583  * in seconds stored in bintime format.
  584  * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be
  585  * larger than the max value of u_int (on 32 bit architecture). Loop to consume
  586  * extra cycles.
  587  */
  588 static void
  589 ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt)
  590 {
  591         struct bintime bt2;
  592         ffcounter delta, delta_max;
  593 
  594         delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1;
  595         bintime_clear(bt);
  596         do {
  597                 if (ffdelta > delta_max)
  598                         delta = delta_max;
  599                 else
  600                         delta = ffdelta;
  601                 bt2.sec = 0;
  602                 bt2.frac = period;
  603                 bintime_mul(&bt2, (unsigned int)delta);
  604                 bintime_add(bt, &bt2);
  605                 ffdelta -= delta;
  606         } while (ffdelta > 0);
  607 }
  608 
  609 /*
  610  * Update the fftimehands.
  611  * Push the tick ffcount and time(s) forward based on current clock estimate.
  612  * The conversion from ffcounter to bintime relies on the difference clock
  613  * principle, whose accuracy relies on computing small time intervals. If a new
  614  * clock estimate has been passed by the synchronisation daemon, make it
  615  * current, and compute the linear interpolation for monotonic time if needed.
  616  */
  617 static void
  618 ffclock_windup(unsigned int delta)
  619 {
  620         struct ffclock_estimate *cest;
  621         struct fftimehands *ffth;
  622         struct bintime bt, gap_lerp;
  623         ffcounter ffdelta;
  624         uint64_t frac;
  625         unsigned int polling;
  626         uint8_t forward_jump, ogen;
  627 
  628         /*
  629          * Pick the next timehand, copy current ffclock estimates and move tick
  630          * times and counter forward.
  631          */
  632         forward_jump = 0;
  633         ffth = fftimehands->next;
  634         ogen = ffth->gen;
  635         ffth->gen = 0;
  636         cest = &ffth->cest;
  637         bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate));
  638         ffdelta = (ffcounter)delta;
  639         ffth->period_lerp = fftimehands->period_lerp;
  640 
  641         ffth->tick_time = fftimehands->tick_time;
  642         ffclock_convert_delta(ffdelta, cest->period, &bt);
  643         bintime_add(&ffth->tick_time, &bt);
  644 
  645         ffth->tick_time_lerp = fftimehands->tick_time_lerp;
  646         ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt);
  647         bintime_add(&ffth->tick_time_lerp, &bt);
  648 
  649         ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta;
  650 
  651         /*
  652          * Assess the status of the clock, if the last update is too old, it is
  653          * likely the synchronisation daemon is dead and the clock is free
  654          * running.
  655          */
  656         if (ffclock_updated == 0) {
  657                 ffdelta = ffth->tick_ffcount - cest->update_ffcount;
  658                 ffclock_convert_delta(ffdelta, cest->period, &bt);
  659                 if (bt.sec > 2 * FFCLOCK_SKM_SCALE)
  660                         ffclock_status |= FFCLOCK_STA_UNSYNC;
  661         }
  662 
  663         /*
  664          * If available, grab updated clock estimates and make them current.
  665          * Recompute time at this tick using the updated estimates. The clock
  666          * estimates passed the feed-forward synchronisation daemon may result
  667          * in time conversion that is not monotonically increasing (just after
  668          * the update). time_lerp is a particular linear interpolation over the
  669          * synchronisation algo polling period that ensures monotonicity for the
  670          * clock ids requesting it.
  671          */
  672         if (ffclock_updated > 0) {
  673                 bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate));
  674                 ffdelta = ffth->tick_ffcount - cest->update_ffcount;
  675                 ffth->tick_time = cest->update_time;
  676                 ffclock_convert_delta(ffdelta, cest->period, &bt);
  677                 bintime_add(&ffth->tick_time, &bt);
  678 
  679                 /* ffclock_reset sets ffclock_updated to INT8_MAX */
  680                 if (ffclock_updated == INT8_MAX)
  681                         ffth->tick_time_lerp = ffth->tick_time;
  682 
  683                 if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >))
  684                         forward_jump = 1;
  685                 else
  686                         forward_jump = 0;
  687 
  688                 bintime_clear(&gap_lerp);
  689                 if (forward_jump) {
  690                         gap_lerp = ffth->tick_time;
  691                         bintime_sub(&gap_lerp, &ffth->tick_time_lerp);
  692                 } else {
  693                         gap_lerp = ffth->tick_time_lerp;
  694                         bintime_sub(&gap_lerp, &ffth->tick_time);
  695                 }
  696 
  697                 /*
  698                  * The reset from the RTC clock may be far from accurate, and
  699                  * reducing the gap between real time and interpolated time
  700                  * could take a very long time if the interpolated clock insists
  701                  * on strict monotonicity. The clock is reset under very strict
  702                  * conditions (kernel time is known to be wrong and
  703                  * synchronization daemon has been restarted recently.
  704                  * ffclock_boottime absorbs the jump to ensure boot time is
  705                  * correct and uptime functions stay consistent.
  706                  */
  707                 if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) &&
  708                     ((cest->status & FFCLOCK_STA_UNSYNC) == 0) &&
  709                     ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) {
  710                         if (forward_jump)
  711                                 bintime_add(&ffclock_boottime, &gap_lerp);
  712                         else
  713                                 bintime_sub(&ffclock_boottime, &gap_lerp);
  714                         ffth->tick_time_lerp = ffth->tick_time;
  715                         bintime_clear(&gap_lerp);
  716                 }
  717 
  718                 ffclock_status = cest->status;
  719                 ffth->period_lerp = cest->period;
  720 
  721                 /*
  722                  * Compute corrected period used for the linear interpolation of
  723                  * time. The rate of linear interpolation is capped to 5000PPM
  724                  * (5ms/s).
  725                  */
  726                 if (bintime_isset(&gap_lerp)) {
  727                         ffdelta = cest->update_ffcount;
  728                         ffdelta -= fftimehands->cest.update_ffcount;
  729                         ffclock_convert_delta(ffdelta, cest->period, &bt);
  730                         polling = bt.sec;
  731                         bt.sec = 0;
  732                         bt.frac = 5000000 * (uint64_t)18446744073LL;
  733                         bintime_mul(&bt, polling);
  734                         if (bintime_cmp(&gap_lerp, &bt, >))
  735                                 gap_lerp = bt;
  736 
  737                         /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */
  738                         frac = 0;
  739                         if (gap_lerp.sec > 0) {
  740                                 frac -= 1;
  741                                 frac /= ffdelta / gap_lerp.sec;
  742                         }
  743                         frac += gap_lerp.frac / ffdelta;
  744 
  745                         if (forward_jump)
  746                                 ffth->period_lerp += frac;
  747                         else
  748                                 ffth->period_lerp -= frac;
  749                 }
  750 
  751                 ffclock_updated = 0;
  752         }
  753         if (++ogen == 0)
  754                 ogen = 1;
  755         ffth->gen = ogen;
  756         fftimehands = ffth;
  757 }
  758 
  759 /*
  760  * Adjust the fftimehands when the timecounter is changed. Stating the obvious,
  761  * the old and new hardware counter cannot be read simultaneously. tc_windup()
  762  * does read the two counters 'back to back', but a few cycles are effectively
  763  * lost, and not accumulated in tick_ffcount. This is a fairly radical
  764  * operation for a feed-forward synchronization daemon, and it is its job to not
  765  * pushing irrelevant data to the kernel. Because there is no locking here,
  766  * simply force to ignore pending or next update to give daemon a chance to
  767  * realize the counter has changed.
  768  */
  769 static void
  770 ffclock_change_tc(struct timehands *th)
  771 {
  772         struct fftimehands *ffth;
  773         struct ffclock_estimate *cest;
  774         struct timecounter *tc;
  775         uint8_t ogen;
  776 
  777         tc = th->th_counter;
  778         ffth = fftimehands->next;
  779         ogen = ffth->gen;
  780         ffth->gen = 0;
  781 
  782         cest = &ffth->cest;
  783         bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate));
  784         cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1;
  785         cest->errb_abs = 0;
  786         cest->errb_rate = 0;
  787         cest->status |= FFCLOCK_STA_UNSYNC;
  788 
  789         ffth->tick_ffcount = fftimehands->tick_ffcount;
  790         ffth->tick_time_lerp = fftimehands->tick_time_lerp;
  791         ffth->tick_time = fftimehands->tick_time;
  792         ffth->period_lerp = cest->period;
  793 
  794         /* Do not lock but ignore next update from synchronization daemon. */
  795         ffclock_updated--;
  796 
  797         if (++ogen == 0)
  798                 ogen = 1;
  799         ffth->gen = ogen;
  800         fftimehands = ffth;
  801 }
  802 
  803 /*
  804  * Retrieve feed-forward counter and time of last kernel tick.
  805  */
  806 void
  807 ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags)
  808 {
  809         struct fftimehands *ffth;
  810         uint8_t gen;
  811 
  812         /*
  813          * No locking but check generation has not changed. Also need to make
  814          * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
  815          */
  816         do {
  817                 ffth = fftimehands;
  818                 gen = ffth->gen;
  819                 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP)
  820                         *bt = ffth->tick_time_lerp;
  821                 else
  822                         *bt = ffth->tick_time;
  823                 *ffcount = ffth->tick_ffcount;
  824         } while (gen == 0 || gen != ffth->gen);
  825 }
  826 
  827 /*
  828  * Absolute clock conversion. Low level function to convert ffcounter to
  829  * bintime. The ffcounter is converted using the current ffclock period estimate
  830  * or the "interpolated period" to ensure monotonicity.
  831  * NOTE: this conversion may have been deferred, and the clock updated since the
  832  * hardware counter has been read.
  833  */
  834 void
  835 ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags)
  836 {
  837         struct fftimehands *ffth;
  838         struct bintime bt2;
  839         ffcounter ffdelta;
  840         uint8_t gen;
  841 
  842         /*
  843          * No locking but check generation has not changed. Also need to make
  844          * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
  845          */
  846         do {
  847                 ffth = fftimehands;
  848                 gen = ffth->gen;
  849                 if (ffcount > ffth->tick_ffcount)
  850                         ffdelta = ffcount - ffth->tick_ffcount;
  851                 else
  852                         ffdelta = ffth->tick_ffcount - ffcount;
  853 
  854                 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) {
  855                         *bt = ffth->tick_time_lerp;
  856                         ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2);
  857                 } else {
  858                         *bt = ffth->tick_time;
  859                         ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2);
  860                 }
  861 
  862                 if (ffcount > ffth->tick_ffcount)
  863                         bintime_add(bt, &bt2);
  864                 else
  865                         bintime_sub(bt, &bt2);
  866         } while (gen == 0 || gen != ffth->gen);
  867 }
  868 
  869 /*
  870  * Difference clock conversion.
  871  * Low level function to Convert a time interval measured in RAW counter units
  872  * into bintime. The difference clock allows measuring small intervals much more
  873  * reliably than the absolute clock.
  874  */
  875 void
  876 ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt)
  877 {
  878         struct fftimehands *ffth;
  879         uint8_t gen;
  880 
  881         /* No locking but check generation has not changed. */
  882         do {
  883                 ffth = fftimehands;
  884                 gen = ffth->gen;
  885                 ffclock_convert_delta(ffdelta, ffth->cest.period, bt);
  886         } while (gen == 0 || gen != ffth->gen);
  887 }
  888 
  889 /*
  890  * Access to current ffcounter value.
  891  */
  892 void
  893 ffclock_read_counter(ffcounter *ffcount)
  894 {
  895         struct timehands *th;
  896         struct fftimehands *ffth;
  897         unsigned int gen, delta;
  898 
  899         /*
  900          * ffclock_windup() called from tc_windup(), safe to rely on
  901          * th->th_generation only, for correct delta and ffcounter.
  902          */
  903         do {
  904                 th = timehands;
  905                 gen = atomic_load_acq_int(&th->th_generation);
  906                 ffth = fftimehands;
  907                 delta = tc_delta(th);
  908                 *ffcount = ffth->tick_ffcount;
  909                 atomic_thread_fence_acq();
  910         } while (gen == 0 || gen != th->th_generation);
  911 
  912         *ffcount += delta;
  913 }
  914 
  915 void
  916 binuptime(struct bintime *bt)
  917 {
  918 
  919         binuptime_fromclock(bt, sysclock_active);
  920 }
  921 
  922 void
  923 nanouptime(struct timespec *tsp)
  924 {
  925 
  926         nanouptime_fromclock(tsp, sysclock_active);
  927 }
  928 
  929 void
  930 microuptime(struct timeval *tvp)
  931 {
  932 
  933         microuptime_fromclock(tvp, sysclock_active);
  934 }
  935 
  936 void
  937 bintime(struct bintime *bt)
  938 {
  939 
  940         bintime_fromclock(bt, sysclock_active);
  941 }
  942 
  943 void
  944 nanotime(struct timespec *tsp)
  945 {
  946 
  947         nanotime_fromclock(tsp, sysclock_active);
  948 }
  949 
  950 void
  951 microtime(struct timeval *tvp)
  952 {
  953 
  954         microtime_fromclock(tvp, sysclock_active);
  955 }
  956 
  957 void
  958 getbinuptime(struct bintime *bt)
  959 {
  960 
  961         getbinuptime_fromclock(bt, sysclock_active);
  962 }
  963 
  964 void
  965 getnanouptime(struct timespec *tsp)
  966 {
  967 
  968         getnanouptime_fromclock(tsp, sysclock_active);
  969 }
  970 
  971 void
  972 getmicrouptime(struct timeval *tvp)
  973 {
  974 
  975         getmicrouptime_fromclock(tvp, sysclock_active);
  976 }
  977 
  978 void
  979 getbintime(struct bintime *bt)
  980 {
  981 
  982         getbintime_fromclock(bt, sysclock_active);
  983 }
  984 
  985 void
  986 getnanotime(struct timespec *tsp)
  987 {
  988 
  989         getnanotime_fromclock(tsp, sysclock_active);
  990 }
  991 
  992 void
  993 getmicrotime(struct timeval *tvp)
  994 {
  995 
  996         getmicrouptime_fromclock(tvp, sysclock_active);
  997 }
  998 
  999 #endif /* FFCLOCK */
 1000 
 1001 /*
 1002  * This is a clone of getnanotime and used for walltimestamps.
 1003  * The dtrace_ prefix prevents fbt from creating probes for
 1004  * it so walltimestamp can be safely used in all fbt probes.
 1005  */
 1006 void
 1007 dtrace_getnanotime(struct timespec *tsp)
 1008 {
 1009 
 1010         GETTHMEMBER(tsp, th_nanotime);
 1011 }
 1012 
 1013 /*
 1014  * This is a clone of getnanouptime used for time since boot.
 1015  * The dtrace_ prefix prevents fbt from creating probes for
 1016  * it so an uptime that can be safely used in all fbt probes.
 1017  */
 1018 void
 1019 dtrace_getnanouptime(struct timespec *tsp)
 1020 {
 1021         struct bintime bt;
 1022 
 1023         GETTHMEMBER(&bt, th_offset);
 1024         bintime2timespec(&bt, tsp);
 1025 }
 1026 
 1027 /*
 1028  * System clock currently providing time to the system. Modifiable via sysctl
 1029  * when the FFCLOCK option is defined.
 1030  */
 1031 int sysclock_active = SYSCLOCK_FBCK;
 1032 
 1033 /* Internal NTP status and error estimates. */
 1034 extern int time_status;
 1035 extern long time_esterror;
 1036 
 1037 /*
 1038  * Take a snapshot of sysclock data which can be used to compare system clocks
 1039  * and generate timestamps after the fact.
 1040  */
 1041 void
 1042 sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
 1043 {
 1044         struct fbclock_info *fbi;
 1045         struct timehands *th;
 1046         struct bintime bt;
 1047         unsigned int delta, gen;
 1048 #ifdef FFCLOCK
 1049         ffcounter ffcount;
 1050         struct fftimehands *ffth;
 1051         struct ffclock_info *ffi;
 1052         struct ffclock_estimate cest;
 1053 
 1054         ffi = &clock_snap->ff_info;
 1055 #endif
 1056 
 1057         fbi = &clock_snap->fb_info;
 1058         delta = 0;
 1059 
 1060         do {
 1061                 th = timehands;
 1062                 gen = atomic_load_acq_int(&th->th_generation);
 1063                 fbi->th_scale = th->th_scale;
 1064                 fbi->tick_time = th->th_offset;
 1065 #ifdef FFCLOCK
 1066                 ffth = fftimehands;
 1067                 ffi->tick_time = ffth->tick_time_lerp;
 1068                 ffi->tick_time_lerp = ffth->tick_time_lerp;
 1069                 ffi->period = ffth->cest.period;
 1070                 ffi->period_lerp = ffth->period_lerp;
 1071                 clock_snap->ffcount = ffth->tick_ffcount;
 1072                 cest = ffth->cest;
 1073 #endif
 1074                 if (!fast)
 1075                         delta = tc_delta(th);
 1076                 atomic_thread_fence_acq();
 1077         } while (gen == 0 || gen != th->th_generation);
 1078 
 1079         clock_snap->delta = delta;
 1080         clock_snap->sysclock_active = sysclock_active;
 1081 
 1082         /* Record feedback clock status and error. */
 1083         clock_snap->fb_info.status = time_status;
 1084         /* XXX: Very crude estimate of feedback clock error. */
 1085         bt.sec = time_esterror / 1000000;
 1086         bt.frac = ((time_esterror - bt.sec) * 1000000) *
 1087             (uint64_t)18446744073709ULL;
 1088         clock_snap->fb_info.error = bt;
 1089 
 1090 #ifdef FFCLOCK
 1091         if (!fast)
 1092                 clock_snap->ffcount += delta;
 1093 
 1094         /* Record feed-forward clock leap second adjustment. */
 1095         ffi->leapsec_adjustment = cest.leapsec_total;
 1096         if (clock_snap->ffcount > cest.leapsec_next)
 1097                 ffi->leapsec_adjustment -= cest.leapsec;
 1098 
 1099         /* Record feed-forward clock status and error. */
 1100         clock_snap->ff_info.status = cest.status;
 1101         ffcount = clock_snap->ffcount - cest.update_ffcount;
 1102         ffclock_convert_delta(ffcount, cest.period, &bt);
 1103         /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */
 1104         bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL);
 1105         /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */
 1106         bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL);
 1107         clock_snap->ff_info.error = bt;
 1108 #endif
 1109 }
 1110 
 1111 /*
 1112  * Convert a sysclock snapshot into a struct bintime based on the specified
 1113  * clock source and flags.
 1114  */
 1115 int
 1116 sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt,
 1117     int whichclock, uint32_t flags)
 1118 {
 1119         struct bintime boottimebin;
 1120 #ifdef FFCLOCK
 1121         struct bintime bt2;
 1122         uint64_t period;
 1123 #endif
 1124 
 1125         switch (whichclock) {
 1126         case SYSCLOCK_FBCK:
 1127                 *bt = cs->fb_info.tick_time;
 1128 
 1129                 /* If snapshot was created with !fast, delta will be >0. */
 1130                 if (cs->delta > 0)
 1131                         bintime_addx(bt, cs->fb_info.th_scale * cs->delta);
 1132 
 1133                 if ((flags & FBCLOCK_UPTIME) == 0) {
 1134                         getboottimebin(&boottimebin);
 1135                         bintime_add(bt, &boottimebin);
 1136                 }
 1137                 break;
 1138 #ifdef FFCLOCK
 1139         case SYSCLOCK_FFWD:
 1140                 if (flags & FFCLOCK_LERP) {
 1141                         *bt = cs->ff_info.tick_time_lerp;
 1142                         period = cs->ff_info.period_lerp;
 1143                 } else {
 1144                         *bt = cs->ff_info.tick_time;
 1145                         period = cs->ff_info.period;
 1146                 }
 1147 
 1148                 /* If snapshot was created with !fast, delta will be >0. */
 1149                 if (cs->delta > 0) {
 1150                         ffclock_convert_delta(cs->delta, period, &bt2);
 1151                         bintime_add(bt, &bt2);
 1152                 }
 1153 
 1154                 /* Leap second adjustment. */
 1155                 if (flags & FFCLOCK_LEAPSEC)
 1156                         bt->sec -= cs->ff_info.leapsec_adjustment;
 1157 
 1158                 /* Boot time adjustment, for uptime/monotonic clocks. */
 1159                 if (flags & FFCLOCK_UPTIME)
 1160                         bintime_sub(bt, &ffclock_boottime);
 1161                 break;
 1162 #endif
 1163         default:
 1164                 return (EINVAL);
 1165                 break;
 1166         }
 1167 
 1168         return (0);
 1169 }
 1170 
 1171 /*
 1172  * Initialize a new timecounter and possibly use it.
 1173  */
 1174 void
 1175 tc_init(struct timecounter *tc)
 1176 {
 1177         u_int u;
 1178         struct sysctl_oid *tc_root;
 1179 
 1180         u = tc->tc_frequency / tc->tc_counter_mask;
 1181         /* XXX: We need some margin here, 10% is a guess */
 1182         u *= 11;
 1183         u /= 10;
 1184         if (u > hz && tc->tc_quality >= 0) {
 1185                 tc->tc_quality = -2000;
 1186                 if (bootverbose) {
 1187                         printf("Timecounter \"%s\" frequency %ju Hz",
 1188                             tc->tc_name, (uintmax_t)tc->tc_frequency);
 1189                         printf(" -- Insufficient hz, needs at least %u\n", u);
 1190                 }
 1191         } else if (tc->tc_quality >= 0 || bootverbose) {
 1192                 printf("Timecounter \"%s\" frequency %ju Hz quality %d\n",
 1193                     tc->tc_name, (uintmax_t)tc->tc_frequency,
 1194                     tc->tc_quality);
 1195         }
 1196 
 1197         /*
 1198          * Set up sysctl tree for this counter.
 1199          */
 1200         tc_root = SYSCTL_ADD_NODE_WITH_LABEL(NULL,
 1201             SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name,
 1202             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 1203             "timecounter description", "timecounter");
 1204         SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
 1205             "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0,
 1206             "mask for implemented bits");
 1207         SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
 1208             "counter", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, tc,
 1209             sizeof(*tc), sysctl_kern_timecounter_get, "IU",
 1210             "current timecounter value");
 1211         SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
 1212             "frequency", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, tc,
 1213             sizeof(*tc), sysctl_kern_timecounter_freq, "QU",
 1214             "timecounter frequency");
 1215         SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
 1216             "quality", CTLFLAG_RD, &(tc->tc_quality), 0,
 1217             "goodness of time counter");
 1218 
 1219         mtx_lock(&tc_lock);
 1220         tc->tc_next = timecounters;
 1221         timecounters = tc;
 1222 
 1223         /*
 1224          * Do not automatically switch if the current tc was specifically
 1225          * chosen.  Never automatically use a timecounter with negative quality.
 1226          * Even though we run on the dummy counter, switching here may be
 1227          * worse since this timecounter may not be monotonic.
 1228          */
 1229         if (tc_chosen)
 1230                 goto unlock;
 1231         if (tc->tc_quality < 0)
 1232                 goto unlock;
 1233         if (tc_from_tunable[0] != '\0' &&
 1234             strcmp(tc->tc_name, tc_from_tunable) == 0) {
 1235                 tc_chosen = 1;
 1236                 tc_from_tunable[0] = '\0';
 1237         } else {
 1238                 if (tc->tc_quality < timecounter->tc_quality)
 1239                         goto unlock;
 1240                 if (tc->tc_quality == timecounter->tc_quality &&
 1241                     tc->tc_frequency < timecounter->tc_frequency)
 1242                         goto unlock;
 1243         }
 1244         (void)tc->tc_get_timecount(tc);
 1245         timecounter = tc;
 1246 unlock:
 1247         mtx_unlock(&tc_lock);
 1248 }
 1249 
 1250 /* Report the frequency of the current timecounter. */
 1251 uint64_t
 1252 tc_getfrequency(void)
 1253 {
 1254 
 1255         return (timehands->th_counter->tc_frequency);
 1256 }
 1257 
 1258 static bool
 1259 sleeping_on_old_rtc(struct thread *td)
 1260 {
 1261 
 1262         /*
 1263          * td_rtcgen is modified by curthread when it is running,
 1264          * and by other threads in this function.  By finding the thread
 1265          * on a sleepqueue and holding the lock on the sleepqueue
 1266          * chain, we guarantee that the thread is not running and that
 1267          * modifying td_rtcgen is safe.  Setting td_rtcgen to zero informs
 1268          * the thread that it was woken due to a real-time clock adjustment.
 1269          * (The declaration of td_rtcgen refers to this comment.)
 1270          */
 1271         if (td->td_rtcgen != 0 && td->td_rtcgen != rtc_generation) {
 1272                 td->td_rtcgen = 0;
 1273                 return (true);
 1274         }
 1275         return (false);
 1276 }
 1277 
 1278 static struct mtx tc_setclock_mtx;
 1279 MTX_SYSINIT(tc_setclock_init, &tc_setclock_mtx, "tcsetc", MTX_SPIN);
 1280 
 1281 /*
 1282  * Step our concept of UTC.  This is done by modifying our estimate of
 1283  * when we booted.
 1284  */
 1285 void
 1286 tc_setclock(struct timespec *ts)
 1287 {
 1288         struct timespec tbef, taft;
 1289         struct bintime bt, bt2;
 1290 
 1291         timespec2bintime(ts, &bt);
 1292         nanotime(&tbef);
 1293         mtx_lock_spin(&tc_setclock_mtx);
 1294         cpu_tick_calibrate(1);
 1295         binuptime(&bt2);
 1296         bintime_sub(&bt, &bt2);
 1297 
 1298         /* XXX fiddle all the little crinkly bits around the fiords... */
 1299         tc_windup(&bt);
 1300         mtx_unlock_spin(&tc_setclock_mtx);
 1301 
 1302         /* Avoid rtc_generation == 0, since td_rtcgen == 0 is special. */
 1303         atomic_add_rel_int(&rtc_generation, 2);
 1304         sleepq_chains_remove_matching(sleeping_on_old_rtc);
 1305         if (timestepwarnings) {
 1306                 nanotime(&taft);
 1307                 log(LOG_INFO,
 1308                     "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n",
 1309                     (intmax_t)tbef.tv_sec, tbef.tv_nsec,
 1310                     (intmax_t)taft.tv_sec, taft.tv_nsec,
 1311                     (intmax_t)ts->tv_sec, ts->tv_nsec);
 1312         }
 1313 }
 1314 
 1315 /*
 1316  * Recalculate the scaling factor.  We want the number of 1/2^64
 1317  * fractions of a second per period of the hardware counter, taking
 1318  * into account the th_adjustment factor which the NTP PLL/adjtime(2)
 1319  * processing provides us with.
 1320  *
 1321  * The th_adjustment is nanoseconds per second with 32 bit binary
 1322  * fraction and we want 64 bit binary fraction of second:
 1323  *
 1324  *       x = a * 2^32 / 10^9 = a * 4.294967296
 1325  *
 1326  * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
 1327  * we can only multiply by about 850 without overflowing, that
 1328  * leaves no suitably precise fractions for multiply before divide.
 1329  *
 1330  * Divide before multiply with a fraction of 2199/512 results in a
 1331  * systematic undercompensation of 10PPM of th_adjustment.  On a
 1332  * 5000PPM adjustment this is a 0.05PPM error.  This is acceptable.
 1333  *
 1334  * We happily sacrifice the lowest of the 64 bits of our result
 1335  * to the goddess of code clarity.
 1336  */
 1337 static void
 1338 recalculate_scaling_factor_and_large_delta(struct timehands *th)
 1339 {
 1340         uint64_t scale;
 1341 
 1342         scale = (uint64_t)1 << 63;
 1343         scale += (th->th_adjustment / 1024) * 2199;
 1344         scale /= th->th_counter->tc_frequency;
 1345         th->th_scale = scale * 2;
 1346         th->th_large_delta = MIN(((uint64_t)1 << 63) / scale, UINT_MAX);
 1347 }
 1348 
 1349 /*
 1350  * Initialize the next struct timehands in the ring and make
 1351  * it the active timehands.  Along the way we might switch to a different
 1352  * timecounter and/or do seconds processing in NTP.  Slightly magic.
 1353  */
 1354 static void
 1355 tc_windup(struct bintime *new_boottimebin)
 1356 {
 1357         struct bintime bt;
 1358         struct timecounter *tc;
 1359         struct timehands *th, *tho;
 1360         u_int delta, ncount, ogen;
 1361         int i;
 1362         time_t t;
 1363 
 1364         /*
 1365          * Make the next timehands a copy of the current one, but do
 1366          * not overwrite the generation or next pointer.  While we
 1367          * update the contents, the generation must be zero.  We need
 1368          * to ensure that the zero generation is visible before the
 1369          * data updates become visible, which requires release fence.
 1370          * For similar reasons, re-reading of the generation after the
 1371          * data is read should use acquire fence.
 1372          */
 1373         tho = timehands;
 1374         th = tho->th_next;
 1375         ogen = th->th_generation;
 1376         th->th_generation = 0;
 1377         atomic_thread_fence_rel();
 1378         memcpy(th, tho, offsetof(struct timehands, th_generation));
 1379         if (new_boottimebin != NULL)
 1380                 th->th_boottime = *new_boottimebin;
 1381 
 1382         /*
 1383          * Capture a timecounter delta on the current timecounter and if
 1384          * changing timecounters, a counter value from the new timecounter.
 1385          * Update the offset fields accordingly.
 1386          */
 1387         tc = atomic_load_ptr(&timecounter);
 1388         delta = tc_delta(th);
 1389         if (th->th_counter != tc)
 1390                 ncount = tc->tc_get_timecount(tc);
 1391         else
 1392                 ncount = 0;
 1393 #ifdef FFCLOCK
 1394         ffclock_windup(delta);
 1395 #endif
 1396         th->th_offset_count += delta;
 1397         th->th_offset_count &= th->th_counter->tc_counter_mask;
 1398         bintime_add_tc_delta(&th->th_offset, th->th_scale,
 1399             th->th_large_delta, delta);
 1400 
 1401         /*
 1402          * Hardware latching timecounters may not generate interrupts on
 1403          * PPS events, so instead we poll them.  There is a finite risk that
 1404          * the hardware might capture a count which is later than the one we
 1405          * got above, and therefore possibly in the next NTP second which might
 1406          * have a different rate than the current NTP second.  It doesn't
 1407          * matter in practice.
 1408          */
 1409         if (tho->th_counter->tc_poll_pps)
 1410                 tho->th_counter->tc_poll_pps(tho->th_counter);
 1411 
 1412         /*
 1413          * Deal with NTP second processing.  The loop normally
 1414          * iterates at most once, but in extreme situations it might
 1415          * keep NTP sane if timeouts are not run for several seconds.
 1416          * At boot, the time step can be large when the TOD hardware
 1417          * has been read, so on really large steps, we call
 1418          * ntp_update_second only twice.  We need to call it twice in
 1419          * case we missed a leap second.
 1420          */
 1421         bt = th->th_offset;
 1422         bintime_add(&bt, &th->th_boottime);
 1423         i = bt.sec - tho->th_microtime.tv_sec;
 1424         if (i > 0) {
 1425                 if (i > LARGE_STEP)
 1426                         i = 2;
 1427 
 1428                 do {
 1429                         t = bt.sec;
 1430                         ntp_update_second(&th->th_adjustment, &bt.sec);
 1431                         if (bt.sec != t)
 1432                                 th->th_boottime.sec += bt.sec - t;
 1433                         --i;
 1434                 } while (i > 0);
 1435 
 1436                 recalculate_scaling_factor_and_large_delta(th);
 1437         }
 1438 
 1439         /* Update the UTC timestamps used by the get*() functions. */
 1440         th->th_bintime = bt;
 1441         bintime2timeval(&bt, &th->th_microtime);
 1442         bintime2timespec(&bt, &th->th_nanotime);
 1443 
 1444         /* Now is a good time to change timecounters. */
 1445         if (th->th_counter != tc) {
 1446 #ifndef __arm__
 1447                 if ((tc->tc_flags & TC_FLAGS_C2STOP) != 0)
 1448                         cpu_disable_c2_sleep++;
 1449                 if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0)
 1450                         cpu_disable_c2_sleep--;
 1451 #endif
 1452                 th->th_counter = tc;
 1453                 th->th_offset_count = ncount;
 1454                 tc_min_ticktock_freq = max(1, tc->tc_frequency /
 1455                     (((uint64_t)tc->tc_counter_mask + 1) / 3));
 1456                 recalculate_scaling_factor_and_large_delta(th);
 1457 #ifdef FFCLOCK
 1458                 ffclock_change_tc(th);
 1459 #endif
 1460         }
 1461 
 1462         /*
 1463          * Now that the struct timehands is again consistent, set the new
 1464          * generation number, making sure to not make it zero.
 1465          */
 1466         if (++ogen == 0)
 1467                 ogen = 1;
 1468         atomic_store_rel_int(&th->th_generation, ogen);
 1469 
 1470         /* Go live with the new struct timehands. */
 1471 #ifdef FFCLOCK
 1472         switch (sysclock_active) {
 1473         case SYSCLOCK_FBCK:
 1474 #endif
 1475                 time_second = th->th_microtime.tv_sec;
 1476                 time_uptime = th->th_offset.sec;
 1477 #ifdef FFCLOCK
 1478                 break;
 1479         case SYSCLOCK_FFWD:
 1480                 time_second = fftimehands->tick_time_lerp.sec;
 1481                 time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec;
 1482                 break;
 1483         }
 1484 #endif
 1485 
 1486         timehands = th;
 1487         timekeep_push_vdso();
 1488 }
 1489 
 1490 /* Report or change the active timecounter hardware. */
 1491 static int
 1492 sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
 1493 {
 1494         char newname[32];
 1495         struct timecounter *newtc, *tc;
 1496         int error;
 1497 
 1498         mtx_lock(&tc_lock);
 1499         tc = timecounter;
 1500         strlcpy(newname, tc->tc_name, sizeof(newname));
 1501         mtx_unlock(&tc_lock);
 1502 
 1503         error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req);
 1504         if (error != 0 || req->newptr == NULL)
 1505                 return (error);
 1506 
 1507         mtx_lock(&tc_lock);
 1508         /* Record that the tc in use now was specifically chosen. */
 1509         tc_chosen = 1;
 1510         if (strcmp(newname, tc->tc_name) == 0) {
 1511                 mtx_unlock(&tc_lock);
 1512                 return (0);
 1513         }
 1514         for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) {
 1515                 if (strcmp(newname, newtc->tc_name) != 0)
 1516                         continue;
 1517 
 1518                 /* Warm up new timecounter. */
 1519                 (void)newtc->tc_get_timecount(newtc);
 1520 
 1521                 timecounter = newtc;
 1522 
 1523                 /*
 1524                  * The vdso timehands update is deferred until the next
 1525                  * 'tc_windup()'.
 1526                  *
 1527                  * This is prudent given that 'timekeep_push_vdso()' does not
 1528                  * use any locking and that it can be called in hard interrupt
 1529                  * context via 'tc_windup()'.
 1530                  */
 1531                 break;
 1532         }
 1533         mtx_unlock(&tc_lock);
 1534         return (newtc != NULL ? 0 : EINVAL);
 1535 }
 1536 SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware,
 1537     CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, 0, 0,
 1538     sysctl_kern_timecounter_hardware, "A",
 1539     "Timecounter hardware selected");
 1540 
 1541 /* Report the available timecounter hardware. */
 1542 static int
 1543 sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS)
 1544 {
 1545         struct sbuf sb;
 1546         struct timecounter *tc;
 1547         int error;
 1548 
 1549         error = sysctl_wire_old_buffer(req, 0);
 1550         if (error != 0)
 1551                 return (error);
 1552         sbuf_new_for_sysctl(&sb, NULL, 0, req);
 1553         mtx_lock(&tc_lock);
 1554         for (tc = timecounters; tc != NULL; tc = tc->tc_next) {
 1555                 if (tc != timecounters)
 1556                         sbuf_putc(&sb, ' ');
 1557                 sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality);
 1558         }
 1559         mtx_unlock(&tc_lock);
 1560         error = sbuf_finish(&sb);
 1561         sbuf_delete(&sb);
 1562         return (error);
 1563 }
 1564 
 1565 SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice,
 1566     CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0,
 1567     sysctl_kern_timecounter_choice, "A",
 1568     "Timecounter hardware detected");
 1569 
 1570 /*
 1571  * RFC 2783 PPS-API implementation.
 1572  */
 1573 
 1574 /*
 1575  *  Return true if the driver is aware of the abi version extensions in the
 1576  *  pps_state structure, and it supports at least the given abi version number.
 1577  */
 1578 static inline int
 1579 abi_aware(struct pps_state *pps, int vers)
 1580 {
 1581 
 1582         return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers);
 1583 }
 1584 
 1585 static int
 1586 pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps)
 1587 {
 1588         int err, timo;
 1589         pps_seq_t aseq, cseq;
 1590         struct timeval tv;
 1591 
 1592         if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC)
 1593                 return (EINVAL);
 1594 
 1595         /*
 1596          * If no timeout is requested, immediately return whatever values were
 1597          * most recently captured.  If timeout seconds is -1, that's a request
 1598          * to block without a timeout.  WITNESS won't let us sleep forever
 1599          * without a lock (we really don't need a lock), so just repeatedly
 1600          * sleep a long time.
 1601          */
 1602         if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) {
 1603                 if (fapi->timeout.tv_sec == -1)
 1604                         timo = 0x7fffffff;
 1605                 else {
 1606                         tv.tv_sec = fapi->timeout.tv_sec;
 1607                         tv.tv_usec = fapi->timeout.tv_nsec / 1000;
 1608                         timo = tvtohz(&tv);
 1609                 }
 1610                 aseq = atomic_load_int(&pps->ppsinfo.assert_sequence);
 1611                 cseq = atomic_load_int(&pps->ppsinfo.clear_sequence);
 1612                 while (aseq == atomic_load_int(&pps->ppsinfo.assert_sequence) &&
 1613                     cseq == atomic_load_int(&pps->ppsinfo.clear_sequence)) {
 1614                         if (abi_aware(pps, 1) && pps->driver_mtx != NULL) {
 1615                                 if (pps->flags & PPSFLAG_MTX_SPIN) {
 1616                                         err = msleep_spin(pps, pps->driver_mtx,
 1617                                             "ppsfch", timo);
 1618                                 } else {
 1619                                         err = msleep(pps, pps->driver_mtx, PCATCH,
 1620                                             "ppsfch", timo);
 1621                                 }
 1622                         } else {
 1623                                 err = tsleep(pps, PCATCH, "ppsfch", timo);
 1624                         }
 1625                         if (err == EWOULDBLOCK) {
 1626                                 if (fapi->timeout.tv_sec == -1) {
 1627                                         continue;
 1628                                 } else {
 1629                                         return (ETIMEDOUT);
 1630                                 }
 1631                         } else if (err != 0) {
 1632                                 return (err);
 1633                         }
 1634                 }
 1635         }
 1636 
 1637         pps->ppsinfo.current_mode = pps->ppsparam.mode;
 1638         fapi->pps_info_buf = pps->ppsinfo;
 1639 
 1640         return (0);
 1641 }
 1642 
 1643 int
 1644 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
 1645 {
 1646         pps_params_t *app;
 1647         struct pps_fetch_args *fapi;
 1648 #ifdef FFCLOCK
 1649         struct pps_fetch_ffc_args *fapi_ffc;
 1650 #endif
 1651 #ifdef PPS_SYNC
 1652         struct pps_kcbind_args *kapi;
 1653 #endif
 1654 
 1655         KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl"));
 1656         switch (cmd) {
 1657         case PPS_IOC_CREATE:
 1658                 return (0);
 1659         case PPS_IOC_DESTROY:
 1660                 return (0);
 1661         case PPS_IOC_SETPARAMS:
 1662                 app = (pps_params_t *)data;
 1663                 if (app->mode & ~pps->ppscap)
 1664                         return (EINVAL);
 1665 #ifdef FFCLOCK
 1666                 /* Ensure only a single clock is selected for ffc timestamp. */
 1667                 if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK)
 1668                         return (EINVAL);
 1669 #endif
 1670                 pps->ppsparam = *app;
 1671                 return (0);
 1672         case PPS_IOC_GETPARAMS:
 1673                 app = (pps_params_t *)data;
 1674                 *app = pps->ppsparam;
 1675                 app->api_version = PPS_API_VERS_1;
 1676                 return (0);
 1677         case PPS_IOC_GETCAP:
 1678                 *(int*)data = pps->ppscap;
 1679                 return (0);
 1680         case PPS_IOC_FETCH:
 1681                 fapi = (struct pps_fetch_args *)data;
 1682                 return (pps_fetch(fapi, pps));
 1683 #ifdef FFCLOCK
 1684         case PPS_IOC_FETCH_FFCOUNTER:
 1685                 fapi_ffc = (struct pps_fetch_ffc_args *)data;
 1686                 if (fapi_ffc->tsformat && fapi_ffc->tsformat !=
 1687                     PPS_TSFMT_TSPEC)
 1688                         return (EINVAL);
 1689                 if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec)
 1690                         return (EOPNOTSUPP);
 1691                 pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode;
 1692                 fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc;
 1693                 /* Overwrite timestamps if feedback clock selected. */
 1694                 switch (pps->ppsparam.mode & PPS_TSCLK_MASK) {
 1695                 case PPS_TSCLK_FBCK:
 1696                         fapi_ffc->pps_info_buf_ffc.assert_timestamp =
 1697                             pps->ppsinfo.assert_timestamp;
 1698                         fapi_ffc->pps_info_buf_ffc.clear_timestamp =
 1699                             pps->ppsinfo.clear_timestamp;
 1700                         break;
 1701                 case PPS_TSCLK_FFWD:
 1702                         break;
 1703                 default:
 1704                         break;
 1705                 }
 1706                 return (0);
 1707 #endif /* FFCLOCK */
 1708         case PPS_IOC_KCBIND:
 1709 #ifdef PPS_SYNC
 1710                 kapi = (struct pps_kcbind_args *)data;
 1711                 /* XXX Only root should be able to do this */
 1712                 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC)
 1713                         return (EINVAL);
 1714                 if (kapi->kernel_consumer != PPS_KC_HARDPPS)
 1715                         return (EINVAL);
 1716                 if (kapi->edge & ~pps->ppscap)
 1717                         return (EINVAL);
 1718                 pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) |
 1719                     (pps->kcmode & KCMODE_ABIFLAG);
 1720                 return (0);
 1721 #else
 1722                 return (EOPNOTSUPP);
 1723 #endif
 1724         default:
 1725                 return (ENOIOCTL);
 1726         }
 1727 }
 1728 
 1729 void
 1730 pps_init(struct pps_state *pps)
 1731 {
 1732         pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT;
 1733         if (pps->ppscap & PPS_CAPTUREASSERT)
 1734                 pps->ppscap |= PPS_OFFSETASSERT;
 1735         if (pps->ppscap & PPS_CAPTURECLEAR)
 1736                 pps->ppscap |= PPS_OFFSETCLEAR;
 1737 #ifdef FFCLOCK
 1738         pps->ppscap |= PPS_TSCLK_MASK;
 1739 #endif
 1740         pps->kcmode &= ~KCMODE_ABIFLAG;
 1741 }
 1742 
 1743 void
 1744 pps_init_abi(struct pps_state *pps)
 1745 {
 1746 
 1747         pps_init(pps);
 1748         if (pps->driver_abi > 0) {
 1749                 pps->kcmode |= KCMODE_ABIFLAG;
 1750                 pps->kernel_abi = PPS_ABI_VERSION;
 1751         }
 1752 }
 1753 
 1754 void
 1755 pps_capture(struct pps_state *pps)
 1756 {
 1757         struct timehands *th;
 1758 
 1759         KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
 1760         th = timehands;
 1761         pps->capgen = atomic_load_acq_int(&th->th_generation);
 1762         pps->capth = th;
 1763 #ifdef FFCLOCK
 1764         pps->capffth = fftimehands;
 1765 #endif
 1766         pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
 1767         atomic_thread_fence_acq();
 1768         if (pps->capgen != th->th_generation)
 1769                 pps->capgen = 0;
 1770 }
 1771 
 1772 void
 1773 pps_event(struct pps_state *pps, int event)
 1774 {
 1775         struct bintime bt;
 1776         struct timespec ts, *tsp, *osp;
 1777         u_int tcount, *pcount;
 1778         int foff;
 1779         pps_seq_t *pseq;
 1780 #ifdef FFCLOCK
 1781         struct timespec *tsp_ffc;
 1782         pps_seq_t *pseq_ffc;
 1783         ffcounter *ffcount;
 1784 #endif
 1785 #ifdef PPS_SYNC
 1786         int fhard;
 1787 #endif
 1788 
 1789         KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
 1790         /* Nothing to do if not currently set to capture this event type. */
 1791         if ((event & pps->ppsparam.mode) == 0)
 1792                 return;
 1793         /* If the timecounter was wound up underneath us, bail out. */
 1794         if (pps->capgen == 0 || pps->capgen !=
 1795             atomic_load_acq_int(&pps->capth->th_generation))
 1796                 return;
 1797 
 1798         /* Things would be easier with arrays. */
 1799         if (event == PPS_CAPTUREASSERT) {
 1800                 tsp = &pps->ppsinfo.assert_timestamp;
 1801                 osp = &pps->ppsparam.assert_offset;
 1802                 foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
 1803 #ifdef PPS_SYNC
 1804                 fhard = pps->kcmode & PPS_CAPTUREASSERT;
 1805 #endif
 1806                 pcount = &pps->ppscount[0];
 1807                 pseq = &pps->ppsinfo.assert_sequence;
 1808 #ifdef FFCLOCK
 1809                 ffcount = &pps->ppsinfo_ffc.assert_ffcount;
 1810                 tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp;
 1811                 pseq_ffc = &pps->ppsinfo_ffc.assert_sequence;
 1812 #endif
 1813         } else {
 1814                 tsp = &pps->ppsinfo.clear_timestamp;
 1815                 osp = &pps->ppsparam.clear_offset;
 1816                 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
 1817 #ifdef PPS_SYNC
 1818                 fhard = pps->kcmode & PPS_CAPTURECLEAR;
 1819 #endif
 1820                 pcount = &pps->ppscount[1];
 1821                 pseq = &pps->ppsinfo.clear_sequence;
 1822 #ifdef FFCLOCK
 1823                 ffcount = &pps->ppsinfo_ffc.clear_ffcount;
 1824                 tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp;
 1825                 pseq_ffc = &pps->ppsinfo_ffc.clear_sequence;
 1826 #endif
 1827         }
 1828 
 1829         /*
 1830          * If the timecounter changed, we cannot compare the count values, so
 1831          * we have to drop the rest of the PPS-stuff until the next event.
 1832          */
 1833         if (pps->ppstc != pps->capth->th_counter) {
 1834                 pps->ppstc = pps->capth->th_counter;
 1835                 *pcount = pps->capcount;
 1836                 pps->ppscount[2] = pps->capcount;
 1837                 return;
 1838         }
 1839 
 1840         /* Convert the count to a timespec. */
 1841         tcount = pps->capcount - pps->capth->th_offset_count;
 1842         tcount &= pps->capth->th_counter->tc_counter_mask;
 1843         bt = pps->capth->th_bintime;
 1844         bintime_addx(&bt, pps->capth->th_scale * tcount);
 1845         bintime2timespec(&bt, &ts);
 1846 
 1847         /* If the timecounter was wound up underneath us, bail out. */
 1848         atomic_thread_fence_acq();
 1849         if (pps->capgen != pps->capth->th_generation)
 1850                 return;
 1851 
 1852         *pcount = pps->capcount;
 1853         (*pseq)++;
 1854         *tsp = ts;
 1855 
 1856         if (foff) {
 1857                 timespecadd(tsp, osp, tsp);
 1858                 if (tsp->tv_nsec < 0) {
 1859                         tsp->tv_nsec += 1000000000;
 1860                         tsp->tv_sec -= 1;
 1861                 }
 1862         }
 1863 
 1864 #ifdef FFCLOCK
 1865         *ffcount = pps->capffth->tick_ffcount + tcount;
 1866         bt = pps->capffth->tick_time;
 1867         ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt);
 1868         bintime_add(&bt, &pps->capffth->tick_time);
 1869         bintime2timespec(&bt, &ts);
 1870         (*pseq_ffc)++;
 1871         *tsp_ffc = ts;
 1872 #endif
 1873 
 1874 #ifdef PPS_SYNC
 1875         if (fhard) {
 1876                 uint64_t scale;
 1877 
 1878                 /*
 1879                  * Feed the NTP PLL/FLL.
 1880                  * The FLL wants to know how many (hardware) nanoseconds
 1881                  * elapsed since the previous event.
 1882                  */
 1883                 tcount = pps->capcount - pps->ppscount[2];
 1884                 pps->ppscount[2] = pps->capcount;
 1885                 tcount &= pps->capth->th_counter->tc_counter_mask;
 1886                 scale = (uint64_t)1 << 63;
 1887                 scale /= pps->capth->th_counter->tc_frequency;
 1888                 scale *= 2;
 1889                 bt.sec = 0;
 1890                 bt.frac = 0;
 1891                 bintime_addx(&bt, scale * tcount);
 1892                 bintime2timespec(&bt, &ts);
 1893                 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec);
 1894         }
 1895 #endif
 1896 
 1897         /* Wakeup anyone sleeping in pps_fetch().  */
 1898         wakeup(pps);
 1899 }
 1900 
 1901 /*
 1902  * Timecounters need to be updated every so often to prevent the hardware
 1903  * counter from overflowing.  Updating also recalculates the cached values
 1904  * used by the get*() family of functions, so their precision depends on
 1905  * the update frequency.
 1906  */
 1907 
 1908 static int tc_tick;
 1909 SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0,
 1910     "Approximate number of hardclock ticks in a millisecond");
 1911 
 1912 void
 1913 tc_ticktock(int cnt)
 1914 {
 1915         static int count;
 1916 
 1917         if (mtx_trylock_spin(&tc_setclock_mtx)) {
 1918                 count += cnt;
 1919                 if (count >= tc_tick) {
 1920                         count = 0;
 1921                         tc_windup(NULL);
 1922                 }
 1923                 mtx_unlock_spin(&tc_setclock_mtx);
 1924         }
 1925 }
 1926 
 1927 static void __inline
 1928 tc_adjprecision(void)
 1929 {
 1930         int t;
 1931 
 1932         if (tc_timepercentage > 0) {
 1933                 t = (99 + tc_timepercentage) / tc_timepercentage;
 1934                 tc_precexp = fls(t + (t >> 1)) - 1;
 1935                 FREQ2BT(hz / tc_tick, &bt_timethreshold);
 1936                 FREQ2BT(hz, &bt_tickthreshold);
 1937                 bintime_shift(&bt_timethreshold, tc_precexp);
 1938                 bintime_shift(&bt_tickthreshold, tc_precexp);
 1939         } else {
 1940                 tc_precexp = 31;
 1941                 bt_timethreshold.sec = INT_MAX;
 1942                 bt_timethreshold.frac = ~(uint64_t)0;
 1943                 bt_tickthreshold = bt_timethreshold;
 1944         }
 1945         sbt_timethreshold = bttosbt(bt_timethreshold);
 1946         sbt_tickthreshold = bttosbt(bt_tickthreshold);
 1947 }
 1948 
 1949 static int
 1950 sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS)
 1951 {
 1952         int error, val;
 1953 
 1954         val = tc_timepercentage;
 1955         error = sysctl_handle_int(oidp, &val, 0, req);
 1956         if (error != 0 || req->newptr == NULL)
 1957                 return (error);
 1958         tc_timepercentage = val;
 1959         if (cold)
 1960                 goto done;
 1961         tc_adjprecision();
 1962 done:
 1963         return (0);
 1964 }
 1965 
 1966 /* Set up the requested number of timehands. */
 1967 static void
 1968 inittimehands(void *dummy)
 1969 {
 1970         struct timehands *thp;
 1971         int i;
 1972 
 1973         TUNABLE_INT_FETCH("kern.timecounter.timehands_count",
 1974             &timehands_count);
 1975         if (timehands_count < 1)
 1976                 timehands_count = 1;
 1977         if (timehands_count > nitems(ths))
 1978                 timehands_count = nitems(ths);
 1979         for (i = 1, thp = &ths[0]; i < timehands_count;  thp = &ths[i++])
 1980                 thp->th_next = &ths[i];
 1981         thp->th_next = &ths[0];
 1982 
 1983         TUNABLE_STR_FETCH("kern.timecounter.hardware", tc_from_tunable,
 1984             sizeof(tc_from_tunable));
 1985 
 1986         mtx_init(&tc_lock, "tc", NULL, MTX_DEF);
 1987 }
 1988 SYSINIT(timehands, SI_SUB_TUNABLES, SI_ORDER_ANY, inittimehands, NULL);
 1989 
 1990 static void
 1991 inittimecounter(void *dummy)
 1992 {
 1993         u_int p;
 1994         int tick_rate;
 1995 
 1996         /*
 1997          * Set the initial timeout to
 1998          * max(1, <approx. number of hardclock ticks in a millisecond>).
 1999          * People should probably not use the sysctl to set the timeout
 2000          * to smaller than its initial value, since that value is the
 2001          * smallest reasonable one.  If they want better timestamps they
 2002          * should use the non-"get"* functions.
 2003          */
 2004         if (hz > 1000)
 2005                 tc_tick = (hz + 500) / 1000;
 2006         else
 2007                 tc_tick = 1;
 2008         tc_adjprecision();
 2009         FREQ2BT(hz, &tick_bt);
 2010         tick_sbt = bttosbt(tick_bt);
 2011         tick_rate = hz / tc_tick;
 2012         FREQ2BT(tick_rate, &tc_tick_bt);
 2013         tc_tick_sbt = bttosbt(tc_tick_bt);
 2014         p = (tc_tick * 1000000) / hz;
 2015         printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000);
 2016 
 2017 #ifdef FFCLOCK
 2018         ffclock_init();
 2019 #endif
 2020 
 2021         /* warm up new timecounter (again) and get rolling. */
 2022         (void)timecounter->tc_get_timecount(timecounter);
 2023         mtx_lock_spin(&tc_setclock_mtx);
 2024         tc_windup(NULL);
 2025         mtx_unlock_spin(&tc_setclock_mtx);
 2026 }
 2027 
 2028 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL);
 2029 
 2030 /* Cpu tick handling -------------------------------------------------*/
 2031 
 2032 static bool cpu_tick_variable;
 2033 static uint64_t cpu_tick_frequency;
 2034 
 2035 DPCPU_DEFINE_STATIC(uint64_t, tc_cpu_ticks_base);
 2036 DPCPU_DEFINE_STATIC(unsigned, tc_cpu_ticks_last);
 2037 
 2038 static uint64_t
 2039 tc_cpu_ticks(void)
 2040 {
 2041         struct timecounter *tc;
 2042         uint64_t res, *base;
 2043         unsigned u, *last;
 2044 
 2045         critical_enter();
 2046         base = DPCPU_PTR(tc_cpu_ticks_base);
 2047         last = DPCPU_PTR(tc_cpu_ticks_last);
 2048         tc = timehands->th_counter;
 2049         u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
 2050         if (u < *last)
 2051                 *base += (uint64_t)tc->tc_counter_mask + 1;
 2052         *last = u;
 2053         res = u + *base;
 2054         critical_exit();
 2055         return (res);
 2056 }
 2057 
 2058 void
 2059 cpu_tick_calibration(void)
 2060 {
 2061         static time_t last_calib;
 2062 
 2063         if (time_uptime != last_calib && !(time_uptime & 0xf)) {
 2064                 cpu_tick_calibrate(0);
 2065                 last_calib = time_uptime;
 2066         }
 2067 }
 2068 
 2069 /*
 2070  * This function gets called every 16 seconds on only one designated
 2071  * CPU in the system from hardclock() via cpu_tick_calibration()().
 2072  *
 2073  * Whenever the real time clock is stepped we get called with reset=1
 2074  * to make sure we handle suspend/resume and similar events correctly.
 2075  */
 2076 
 2077 static void
 2078 cpu_tick_calibrate(int reset)
 2079 {
 2080         static uint64_t c_last;
 2081         uint64_t c_this, c_delta;
 2082         static struct bintime  t_last;
 2083         struct bintime t_this, t_delta;
 2084         uint32_t divi;
 2085 
 2086         if (reset) {
 2087                 /* The clock was stepped, abort & reset */
 2088                 t_last.sec = 0;
 2089                 return;
 2090         }
 2091 
 2092         /* we don't calibrate fixed rate cputicks */
 2093         if (!cpu_tick_variable)
 2094                 return;
 2095 
 2096         getbinuptime(&t_this);
 2097         c_this = cpu_ticks();
 2098         if (t_last.sec != 0) {
 2099                 c_delta = c_this - c_last;
 2100                 t_delta = t_this;
 2101                 bintime_sub(&t_delta, &t_last);
 2102                 /*
 2103                  * Headroom:
 2104                  *      2^(64-20) / 16[s] =
 2105                  *      2^(44) / 16[s] =
 2106                  *      17.592.186.044.416 / 16 =
 2107                  *      1.099.511.627.776 [Hz]
 2108                  */
 2109                 divi = t_delta.sec << 20;
 2110                 divi |= t_delta.frac >> (64 - 20);
 2111                 c_delta <<= 20;
 2112                 c_delta /= divi;
 2113                 if (c_delta > cpu_tick_frequency) {
 2114                         if (0 && bootverbose)
 2115                                 printf("cpu_tick increased to %ju Hz\n",
 2116                                     c_delta);
 2117                         cpu_tick_frequency = c_delta;
 2118                 }
 2119         }
 2120         c_last = c_this;
 2121         t_last = t_this;
 2122 }
 2123 
 2124 void
 2125 set_cputicker(cpu_tick_f *func, uint64_t freq, bool isvariable)
 2126 {
 2127 
 2128         if (func == NULL) {
 2129                 cpu_ticks = tc_cpu_ticks;
 2130         } else {
 2131                 cpu_tick_frequency = freq;
 2132                 cpu_tick_variable = isvariable;
 2133                 cpu_ticks = func;
 2134         }
 2135 }
 2136 
 2137 uint64_t
 2138 cpu_tickrate(void)
 2139 {
 2140 
 2141         if (cpu_ticks == tc_cpu_ticks) 
 2142                 return (tc_getfrequency());
 2143         return (cpu_tick_frequency);
 2144 }
 2145 
 2146 /*
 2147  * We need to be slightly careful converting cputicks to microseconds.
 2148  * There is plenty of margin in 64 bits of microseconds (half a million
 2149  * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply
 2150  * before divide conversion (to retain precision) we find that the
 2151  * margin shrinks to 1.5 hours (one millionth of 146y).
 2152  * With a three prong approach we never lose significant bits, no
 2153  * matter what the cputick rate and length of timeinterval is.
 2154  */
 2155 
 2156 uint64_t
 2157 cputick2usec(uint64_t tick)
 2158 {
 2159 
 2160         if (tick > 18446744073709551LL)         /* floor(2^64 / 1000) */
 2161                 return (tick / (cpu_tickrate() / 1000000LL));
 2162         else if (tick > 18446744073709LL)       /* floor(2^64 / 1000000) */
 2163                 return ((tick * 1000LL) / (cpu_tickrate() / 1000LL));
 2164         else
 2165                 return ((tick * 1000000LL) / cpu_tickrate());
 2166 }
 2167 
 2168 cpu_tick_f      *cpu_ticks = tc_cpu_ticks;
 2169 
 2170 static int vdso_th_enable = 1;
 2171 static int
 2172 sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
 2173 {
 2174         int old_vdso_th_enable, error;
 2175 
 2176         old_vdso_th_enable = vdso_th_enable;
 2177         error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req);
 2178         if (error != 0)
 2179                 return (error);
 2180         vdso_th_enable = old_vdso_th_enable;
 2181         return (0);
 2182 }
 2183 SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
 2184     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 2185     NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day");
 2186 
 2187 uint32_t
 2188 tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
 2189 {
 2190         struct timehands *th;
 2191         uint32_t enabled;
 2192 
 2193         th = timehands;
 2194         vdso_th->th_scale = th->th_scale;
 2195         vdso_th->th_offset_count = th->th_offset_count;
 2196         vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
 2197         vdso_th->th_offset = th->th_offset;
 2198         vdso_th->th_boottime = th->th_boottime;
 2199         if (th->th_counter->tc_fill_vdso_timehands != NULL) {
 2200                 enabled = th->th_counter->tc_fill_vdso_timehands(vdso_th,
 2201                     th->th_counter);
 2202         } else
 2203                 enabled = 0;
 2204         if (!vdso_th_enable)
 2205                 enabled = 0;
 2206         return (enabled);
 2207 }
 2208 
 2209 #ifdef COMPAT_FREEBSD32
 2210 uint32_t
 2211 tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
 2212 {
 2213         struct timehands *th;
 2214         uint32_t enabled;
 2215 
 2216         th = timehands;
 2217         *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale;
 2218         vdso_th32->th_offset_count = th->th_offset_count;
 2219         vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask;
 2220         vdso_th32->th_offset.sec = th->th_offset.sec;
 2221         *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
 2222         vdso_th32->th_boottime.sec = th->th_boottime.sec;
 2223         *(uint64_t *)&vdso_th32->th_boottime.frac[0] = th->th_boottime.frac;
 2224         if (th->th_counter->tc_fill_vdso_timehands32 != NULL) {
 2225                 enabled = th->th_counter->tc_fill_vdso_timehands32(vdso_th32,
 2226                     th->th_counter);
 2227         } else
 2228                 enabled = 0;
 2229         if (!vdso_th_enable)
 2230                 enabled = 0;
 2231         return (enabled);
 2232 }
 2233 #endif
 2234 
 2235 #include "opt_ddb.h"
 2236 #ifdef DDB
 2237 #include <ddb/ddb.h>
 2238 
 2239 DB_SHOW_COMMAND(timecounter, db_show_timecounter)
 2240 {
 2241         struct timehands *th;
 2242         struct timecounter *tc;
 2243         u_int val1, val2;
 2244 
 2245         th = timehands;
 2246         tc = th->th_counter;
 2247         val1 = tc->tc_get_timecount(tc);
 2248         __compiler_membar();
 2249         val2 = tc->tc_get_timecount(tc);
 2250 
 2251         db_printf("timecounter %p %s\n", tc, tc->tc_name);
 2252         db_printf("  mask %#x freq %ju qual %d flags %#x priv %p\n",
 2253             tc->tc_counter_mask, (uintmax_t)tc->tc_frequency, tc->tc_quality,
 2254             tc->tc_flags, tc->tc_priv);
 2255         db_printf("  val %#x %#x\n", val1, val2);
 2256         db_printf("timehands adj %#jx scale %#jx ldelta %d off_cnt %d gen %d\n",
 2257             (uintmax_t)th->th_adjustment, (uintmax_t)th->th_scale,
 2258             th->th_large_delta, th->th_offset_count, th->th_generation);
 2259         db_printf("  offset %jd %jd boottime %jd %jd\n",
 2260             (intmax_t)th->th_offset.sec, (uintmax_t)th->th_offset.frac,
 2261             (intmax_t)th->th_boottime.sec, (uintmax_t)th->th_boottime.frac);
 2262 }
 2263 #endif

Cache object: 5fafc3db13d1b8727b25c110e27f9b21


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.