FreeBSD/Linux Kernel Cross Reference
sys/common/os/clock.c

     /*
      * CDDL HEADER START
      *
      * The contents of this file are subject to the terms of the
      * Common Development and Distribution License (the "License").
      * You may not use this file except in compliance with the License.
      *
      * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      * or http://www.opensolaris.org/os/licensing.
     * See the License for the specific language governing permissions
     * and limitations under the License.
     *
     * When distributing Covered Code, include this CDDL HEADER in each
     * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     * If applicable, add the following below this CDDL HEADER, with the
     * fields enclosed by brackets "[]" replaced with your own identifying
     * information: Portions Copyright [yyyy] [name of copyright owner]
     *
     * CDDL HEADER END
     */
    /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
    /*        All Rights Reserved   */
    
    /*
     * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
     */
    
    #include <sys/param.h>
    #include <sys/t_lock.h>
    #include <sys/types.h>
    #include <sys/tuneable.h>
    #include <sys/sysmacros.h>
    #include <sys/systm.h>
    #include <sys/cpuvar.h>
    #include <sys/lgrp.h>
    #include <sys/user.h>
    #include <sys/proc.h>
    #include <sys/callo.h>
    #include <sys/kmem.h>
    #include <sys/var.h>
    #include <sys/cmn_err.h>
    #include <sys/swap.h>
    #include <sys/vmsystm.h>
    #include <sys/class.h>
    #include <sys/time.h>
    #include <sys/debug.h>
    #include <sys/vtrace.h>
    #include <sys/spl.h>
    #include <sys/atomic.h>
    #include <sys/dumphdr.h>
    #include <sys/archsystm.h>
    #include <sys/fs/swapnode.h>
    #include <sys/panic.h>
    #include <sys/disp.h>
    #include <sys/msacct.h>
    #include <sys/mem_cage.h>
    
    #include <vm/page.h>
    #include <vm/anon.h>
    #include <vm/rm.h>
    #include <sys/cyclic.h>
    #include <sys/cpupart.h>
    #include <sys/rctl.h>
    #include <sys/task.h>
    #include <sys/sdt.h>
    #include <sys/ddi_timer.h>
    #include <sys/random.h>
    #include <sys/modctl.h>
    
    /*
     * for NTP support
     */
    #include <sys/timex.h>
    #include <sys/inttypes.h>
    
    #include <sys/sunddi.h>
    #include <sys/clock_impl.h>
    
    /*
     * clock() is called straight from the clock cyclic; see clock_init().
     *
     * Functions:
     *      reprime clock
     *      maintain date
     *      jab the scheduler
     */
    
    extern kcondvar_t       fsflush_cv;
    extern sysinfo_t        sysinfo;
    extern vminfo_t vminfo;
    extern int      idleswtch;      /* flag set while idle in pswtch() */
    extern hrtime_t volatile devinfo_freeze;
    
    /*
     * high-precision avenrun values.  These are needed to make the
     * regular avenrun values accurate.
     */
    static uint64_t hp_avenrun[3];
    int     avenrun[3];             /* FSCALED average run queue lengths */
   time_t  time;   /* time in seconds since 1970 - for compatibility only */
   
   static struct loadavg_s loadavg;
   /*
    * Phase/frequency-lock loop (PLL/FLL) definitions
    *
    * The following variables are read and set by the ntp_adjtime() system
    * call.
    *
    * time_state shows the state of the system clock, with values defined
    * in the timex.h header file.
    *
    * time_status shows the status of the system clock, with bits defined
    * in the timex.h header file.
    *
    * time_offset is used by the PLL/FLL to adjust the system time in small
    * increments.
    *
    * time_constant determines the bandwidth or "stiffness" of the PLL.
    *
    * time_tolerance determines maximum frequency error or tolerance of the
    * CPU clock oscillator and is a property of the architecture; however,
    * in principle it could change as result of the presence of external
    * discipline signals, for instance.
    *
    * time_precision is usually equal to the kernel tick variable; however,
    * in cases where a precision clock counter or external clock is
    * available, the resolution can be much less than this and depend on
    * whether the external clock is working or not.
    *
    * time_maxerror is initialized by a ntp_adjtime() call and increased by
    * the kernel once each second to reflect the maximum error bound
    * growth.
    *
    * time_esterror is set and read by the ntp_adjtime() call, but
    * otherwise not used by the kernel.
    */
   int32_t time_state = TIME_OK;   /* clock state */
   int32_t time_status = STA_UNSYNC;       /* clock status bits */
   int32_t time_offset = 0;                /* time offset (us) */
   int32_t time_constant = 0;              /* pll time constant */
   int32_t time_tolerance = MAXFREQ;       /* frequency tolerance (scaled ppm) */
   int32_t time_precision = 1;     /* clock precision (us) */
   int32_t time_maxerror = MAXPHASE;       /* maximum error (us) */
   int32_t time_esterror = MAXPHASE;       /* estimated error (us) */
   
   /*
    * The following variables establish the state of the PLL/FLL and the
    * residual time and frequency offset of the local clock. The scale
    * factors are defined in the timex.h header file.
    *
    * time_phase and time_freq are the phase increment and the frequency
    * increment, respectively, of the kernel time variable.
    *
    * time_freq is set via ntp_adjtime() from a value stored in a file when
    * the synchronization daemon is first started. Its value is retrieved
    * via ntp_adjtime() and written to the file about once per hour by the
    * daemon.
    *
    * time_adj is the adjustment added to the value of tick at each timer
    * interrupt and is recomputed from time_phase and time_freq at each
    * seconds rollover.
    *
    * time_reftime is the second's portion of the system time at the last
    * call to ntp_adjtime(). It is used to adjust the time_freq variable
    * and to increase the time_maxerror as the time since last update
    * increases.
    */
   int32_t time_phase = 0;         /* phase offset (scaled us) */
   int32_t time_freq = 0;          /* frequency offset (scaled ppm) */
   int32_t time_adj = 0;           /* tick adjust (scaled 1 / hz) */
   int32_t time_reftime = 0;               /* time at last adjustment (s) */
   
   /*
    * The scale factors of the following variables are defined in the
    * timex.h header file.
    *
    * pps_time contains the time at each calibration interval, as read by
    * microtime(). pps_count counts the seconds of the calibration
    * interval, the duration of which is nominally pps_shift in powers of
    * two.
    *
    * pps_offset is the time offset produced by the time median filter
    * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
    * this filter.
    *
    * pps_freq is the frequency offset produced by the frequency median
    * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
    * by this filter.
    *
    * pps_usec is latched from a high resolution counter or external clock
    * at pps_time. Here we want the hardware counter contents only, not the
    * contents plus the time_tv.usec as usual.
    *
    * pps_valid counts the number of seconds since the last PPS update. It
    * is used as a watchdog timer to disable the PPS discipline should the
    * PPS signal be lost.
    *
    * pps_glitch counts the number of seconds since the beginning of an
    * offset burst more than tick/2 from current nominal offset. It is used
    * mainly to suppress error bursts due to priority conflicts between the
    * PPS interrupt and timer interrupt.
    *
    * pps_intcnt counts the calibration intervals for use in the interval-
    * adaptation algorithm. It's just too complicated for words.
    */
   struct timeval pps_time;        /* kernel time at last interval */
   int32_t pps_tf[] = {0, 0, 0};   /* pps time offset median filter (us) */
   int32_t pps_offset = 0;         /* pps time offset (us) */
   int32_t pps_jitter = MAXTIME;   /* time dispersion (jitter) (us) */
   int32_t pps_ff[] = {0, 0, 0};   /* pps frequency offset median filter */
   int32_t pps_freq = 0;           /* frequency offset (scaled ppm) */
   int32_t pps_stabil = MAXFREQ;   /* frequency dispersion (scaled ppm) */
   int32_t pps_usec = 0;           /* microsec counter at last interval */
   int32_t pps_valid = PPS_VALID;  /* pps signal watchdog counter */
   int32_t pps_glitch = 0;         /* pps signal glitch counter */
   int32_t pps_count = 0;          /* calibration interval counter (s) */
   int32_t pps_shift = PPS_SHIFT;  /* interval duration (s) (shift) */
   int32_t pps_intcnt = 0;         /* intervals at current duration */
   
   /*
    * PPS signal quality monitors
    *
    * pps_jitcnt counts the seconds that have been discarded because the
    * jitter measured by the time median filter exceeds the limit MAXTIME
    * (100 us).
    *
    * pps_calcnt counts the frequency calibration intervals, which are
    * variable from 4 s to 256 s.
    *
    * pps_errcnt counts the calibration intervals which have been discarded
    * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
    * calibration interval jitter exceeds two ticks.
    *
    * pps_stbcnt counts the calibration intervals that have been discarded
    * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
    */
   int32_t pps_jitcnt = 0;         /* jitter limit exceeded */
   int32_t pps_calcnt = 0;         /* calibration intervals */
   int32_t pps_errcnt = 0;         /* calibration errors */
   int32_t pps_stbcnt = 0;         /* stability limit exceeded */
   
   kcondvar_t lbolt_cv;
   
   /*
    * Hybrid lbolt implementation:
    *
    * The service historically provided by the lbolt and lbolt64 variables has
    * been replaced by the ddi_get_lbolt() and ddi_get_lbolt64() routines, and the
    * original symbols removed from the system. The once clock driven variables are
    * now implemented in an event driven fashion, backed by gethrtime() coarsed to
    * the appropriate clock resolution. The default event driven implementation is
    * complemented by a cyclic driven one, active only during periods of intense
    * activity around the DDI lbolt routines, when a lbolt specific cyclic is
    * reprogramed to fire at a clock tick interval to serve consumers of lbolt who
    * rely on the original low cost of consulting a memory position.
    *
    * The implementation uses the number of calls to these routines and the
    * frequency of these to determine when to transition from event to cyclic
    * driven and vice-versa. These values are kept on a per CPU basis for
    * scalability reasons and to prevent CPUs from constantly invalidating a single
    * cache line when modifying a global variable. The transition from event to
    * cyclic mode happens once the thresholds are crossed, and activity on any CPU
    * can cause such transition.
    *
    * The lbolt_hybrid function pointer is called by ddi_get_lbolt() and
    * ddi_get_lbolt64(), and will point to lbolt_event_driven() or
    * lbolt_cyclic_driven() according to the current mode. When the thresholds
    * are exceeded, lbolt_event_driven() will reprogram the lbolt cyclic to
    * fire at a nsec_per_tick interval and increment an internal variable at
    * each firing. lbolt_hybrid will then point to lbolt_cyclic_driven(), which
    * will simply return the value of such variable. lbolt_cyclic() will attempt
    * to shut itself off at each threshold interval (sampling period for calls
    * to the DDI lbolt routines), and return to the event driven mode, but will
    * be prevented from doing so if lbolt_cyclic_driven() is being heavily used.
    *
    * lbolt_bootstrap is used during boot to serve lbolt consumers who don't wait
    * for the cyclic subsystem to be intialized.
    *
    */
   int64_t lbolt_bootstrap(void);
   int64_t lbolt_event_driven(void);
   int64_t lbolt_cyclic_driven(void);
   int64_t (*lbolt_hybrid)(void) = lbolt_bootstrap;
   uint_t lbolt_ev_to_cyclic(caddr_t, caddr_t);
   
   /*
    * lbolt's cyclic, installed by clock_init().
    */
   static void lbolt_cyclic(void);
   
   /*
    * Tunable to keep lbolt in cyclic driven mode. This will prevent the system
    * from switching back to event driven, once it reaches cyclic mode.
    */
   static boolean_t lbolt_cyc_only = B_FALSE;
   
   /*
    * Cache aligned, per CPU structure with lbolt usage statistics.
    */
   static lbolt_cpu_t *lb_cpu;
   
   /*
    * Single, cache aligned, structure with all the information required by
    * the lbolt implementation.
    */
   lbolt_info_t *lb_info;
   
   
   int one_sec = 1; /* turned on once every second */
   static int fsflushcnt;  /* counter for t_fsflushr */
   int     dosynctodr = 1; /* patchable; enable/disable sync to TOD chip */
   int     tod_needsync = 0;       /* need to sync tod chip with software time */
   static int tod_broken = 0;      /* clock chip doesn't work */
   time_t  boot_time = 0;          /* Boot time in seconds since 1970 */
   cyclic_id_t clock_cyclic;       /* clock()'s cyclic_id */
   cyclic_id_t deadman_cyclic;     /* deadman()'s cyclic_id */
   cyclic_id_t ddi_timer_cyclic;   /* cyclic_timer()'s cyclic_id */
   
   extern void     clock_tick_schedule(int);
   
   static int lgrp_ticks;          /* counter to schedule lgrp load calcs */
   
   /*
    * for tod fault detection
    */
   #define TOD_REF_FREQ            ((longlong_t)(NANOSEC))
   #define TOD_STALL_THRESHOLD     (TOD_REF_FREQ * 3 / 2)
   #define TOD_JUMP_THRESHOLD      (TOD_REF_FREQ / 2)
   #define TOD_FILTER_N            4
   #define TOD_FILTER_SETTLE       (4 * TOD_FILTER_N)
   static int tod_faulted = TOD_NOFAULT;
   
   static int tod_status_flag = 0;         /* used by tod_validate() */
   
   static hrtime_t prev_set_tick = 0;      /* gethrtime() prior to tod_set() */
   static time_t prev_set_tod = 0;         /* tv_sec value passed to tod_set() */
   
   /* patchable via /etc/system */
   int tod_validate_enable = 1;
   
   /* Diagnose/Limit messages about delay(9F) called from interrupt context */
   int                     delay_from_interrupt_diagnose = 0;
   volatile uint32_t       delay_from_interrupt_msg = 20;
   
   /*
    * On non-SPARC systems, TOD validation must be deferred until gethrtime
    * returns non-zero values (after mach_clkinit's execution).
    * On SPARC systems, it must be deferred until after hrtime_base
    * and hres_last_tick are set (in the first invocation of hres_tick).
    * Since in both cases the prerequisites occur before the invocation of
    * tod_get() in clock(), the deferment is lifted there.
    */
   static boolean_t tod_validate_deferred = B_TRUE;
   
   /*
    * tod_fault_table[] must be aligned with
    * enum tod_fault_type in systm.h
    */
   static char *tod_fault_table[] = {
           "Reversed",                     /* TOD_REVERSED */
           "Stalled",                      /* TOD_STALLED */
           "Jumped",                       /* TOD_JUMPED */
           "Changed in Clock Rate",        /* TOD_RATECHANGED */
           "Is Read-Only"                  /* TOD_RDONLY */
           /*
            * no strings needed for TOD_NOFAULT
            */
   };
   
   /*
    * test hook for tod broken detection in tod_validate
    */
   int tod_unit_test = 0;
   time_t tod_test_injector;
   
   #define CLOCK_ADJ_HIST_SIZE     4
   
   static int      adj_hist_entry;
   
   int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];
   
   static void calcloadavg(int, uint64_t *);
   static int genloadavg(struct loadavg_s *);
   static void loadavg_update();
   
   void (*cmm_clock_callout)() = NULL;
   void (*cpucaps_clock_callout)() = NULL;
   
   extern clock_t clock_tick_proc_max;
   
   static int64_t deadman_counter = 0;
   
   static void
   clock(void)
   {
           kthread_t       *t;
           uint_t  nrunnable;
           uint_t  w_io;
           cpu_t   *cp;
           cpupart_t *cpupart;
           extern  void    set_freemem();
           void    (*funcp)();
           int32_t ltemp;
           int64_t lltemp;
           int s;
           int do_lgrp_load;
           int i;
           clock_t now = LBOLT_NO_ACCOUNT; /* current tick */
   
           if (panicstr)
                   return;
   
           /*
            * Make sure that 'freemem' do not drift too far from the truth
            */
           set_freemem();
   
   
           /*
            * Before the section which is repeated is executed, we do
            * the time delta processing which occurs every clock tick
            *
            * There is additional processing which happens every time
            * the nanosecond counter rolls over which is described
            * below - see the section which begins with : if (one_sec)
            *
            * This section marks the beginning of the precision-kernel
            * code fragment.
            *
            * First, compute the phase adjustment. If the low-order bits
            * (time_phase) of the update overflow, bump the higher order
            * bits (time_update).
            */
           time_phase += time_adj;
           if (time_phase <= -FINEUSEC) {
                   ltemp = -time_phase / SCALE_PHASE;
                   time_phase += ltemp * SCALE_PHASE;
                   s = hr_clock_lock();
                   timedelta -= ltemp * (NANOSEC/MICROSEC);
                   hr_clock_unlock(s);
           } else if (time_phase >= FINEUSEC) {
                   ltemp = time_phase / SCALE_PHASE;
                   time_phase -= ltemp * SCALE_PHASE;
                   s = hr_clock_lock();
                   timedelta += ltemp * (NANOSEC/MICROSEC);
                   hr_clock_unlock(s);
           }
   
           /*
            * End of precision-kernel code fragment which is processed
            * every timer interrupt.
            *
            * Continue with the interrupt processing as scheduled.
            */
           /*
            * Count the number of runnable threads and the number waiting
            * for some form of I/O to complete -- gets added to
            * sysinfo.waiting.  To know the state of the system, must add
            * wait counts from all CPUs.  Also add up the per-partition
            * statistics.
            */
           w_io = 0;
           nrunnable = 0;
   
           /*
            * keep track of when to update lgrp/part loads
            */
   
           do_lgrp_load = 0;
           if (lgrp_ticks++ >= hz / 10) {
                   lgrp_ticks = 0;
                   do_lgrp_load = 1;
           }
   
           if (one_sec) {
                   loadavg_update();
                   deadman_counter++;
           }
   
           /*
            * First count the threads waiting on kpreempt queues in each
            * CPU partition.
            */
   
           cpupart = cp_list_head;
           do {
                   uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;
   
                   cpupart->cp_updates++;
                   nrunnable += cpupart_nrunnable;
                   cpupart->cp_nrunnable_cum += cpupart_nrunnable;
                   if (one_sec) {
                           cpupart->cp_nrunning = 0;
                           cpupart->cp_nrunnable = cpupart_nrunnable;
                   }
           } while ((cpupart = cpupart->cp_next) != cp_list_head);
   
   
           /* Now count the per-CPU statistics. */
           cp = cpu_list;
           do {
                   uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;
   
                   nrunnable += cpu_nrunnable;
                   cpupart = cp->cpu_part;
                   cpupart->cp_nrunnable_cum += cpu_nrunnable;
                   if (one_sec) {
                           cpupart->cp_nrunnable += cpu_nrunnable;
                           /*
                            * Update user, system, and idle cpu times.
                            */
                           cpupart->cp_nrunning++;
                           /*
                            * w_io is used to update sysinfo.waiting during
                            * one_second processing below.  Only gather w_io
                            * information when we walk the list of cpus if we're
                            * going to perform one_second processing.
                            */
                           w_io += CPU_STATS(cp, sys.iowait);
                   }
   
                   if (one_sec && (cp->cpu_flags & CPU_EXISTS)) {
                           int i, load, change;
                           hrtime_t intracct, intrused;
                           const hrtime_t maxnsec = 1000000000;
                           const int precision = 100;
   
                           /*
                            * Estimate interrupt load on this cpu each second.
                            * Computes cpu_intrload as %utilization (0-99).
                            */
   
                           /* add up interrupt time from all micro states */
                           for (intracct = 0, i = 0; i < NCMSTATES; i++)
                                   intracct += cp->cpu_intracct[i];
                           scalehrtime(&intracct);
   
                           /* compute nsec used in the past second */
                           intrused = intracct - cp->cpu_intrlast;
                           cp->cpu_intrlast = intracct;
   
                           /* limit the value for safety (and the first pass) */
                           if (intrused >= maxnsec)
                                   intrused = maxnsec - 1;
   
                           /* calculate %time in interrupt */
                           load = (precision * intrused) / maxnsec;
                           ASSERT(load >= 0 && load < precision);
                           change = cp->cpu_intrload - load;
   
                           /* jump to new max, or decay the old max */
                           if (change < 0)
                                   cp->cpu_intrload = load;
                           else if (change > 0)
                                   cp->cpu_intrload -= (change + 3) / 4;
   
                           DTRACE_PROBE3(cpu_intrload,
                               cpu_t *, cp,
                               hrtime_t, intracct,
                               hrtime_t, intrused);
                   }
   
                   if (do_lgrp_load &&
                       (cp->cpu_flags & CPU_EXISTS)) {
                           /*
                            * When updating the lgroup's load average,
                            * account for the thread running on the CPU.
                            * If the CPU is the current one, then we need
                            * to account for the underlying thread which
                            * got the clock interrupt not the thread that is
                            * handling the interrupt and caculating the load
                            * average
                            */
                           t = cp->cpu_thread;
                           if (CPU == cp)
                                   t = t->t_intr;
   
                           /*
                            * Account for the load average for this thread if
                            * it isn't the idle thread or it is on the interrupt
                            * stack and not the current CPU handling the clock
                            * interrupt
                            */
                           if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
                               CPU_ON_INTR(cp))) {
                                   if (t->t_lpl == cp->cpu_lpl) {
                                           /* local thread */
                                           cpu_nrunnable++;
                                   } else {
                                           /*
                                            * This is a remote thread, charge it
                                            * against its home lgroup.  Note that
                                            * we notice that a thread is remote
                                            * only if it's currently executing.
                                            * This is a reasonable approximation,
                                            * since queued remote threads are rare.
                                            * Note also that if we didn't charge
                                            * it to its home lgroup, remote
                                            * execution would often make a system
                                            * appear balanced even though it was
                                            * not, and thread placement/migration
                                            * would often not be done correctly.
                                            */
                                           lgrp_loadavg(t->t_lpl,
                                               LGRP_LOADAVG_IN_THREAD_MAX, 0);
                                   }
                           }
                           lgrp_loadavg(cp->cpu_lpl,
                               cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
                   }
           } while ((cp = cp->cpu_next) != cpu_list);
   
           clock_tick_schedule(one_sec);
   
           /*
            * Check for a callout that needs be called from the clock
            * thread to support the membership protocol in a clustered
            * system.  Copy the function pointer so that we can reset
            * this to NULL if needed.
            */
           if ((funcp = cmm_clock_callout) != NULL)
                   (*funcp)();
   
           if ((funcp = cpucaps_clock_callout) != NULL)
                   (*funcp)();
   
           /*
            * Wakeup the cageout thread waiters once per second.
            */
           if (one_sec)
                   kcage_tick();
   
           if (one_sec) {
   
                   int drift, absdrift;
                   timestruc_t tod;
                   int s;
   
                   /*
                    * Beginning of precision-kernel code fragment executed
                    * every second.
                    *
                    * On rollover of the second the phase adjustment to be
                    * used for the next second is calculated.  Also, the
                    * maximum error is increased by the tolerance.  If the
                    * PPS frequency discipline code is present, the phase is
                    * increased to compensate for the CPU clock oscillator
                    * frequency error.
                    *
                    * On a 32-bit machine and given parameters in the timex.h
                    * header file, the maximum phase adjustment is +-512 ms
                    * and maximum frequency offset is (a tad less than)
                    * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
                    */
                   time_maxerror += time_tolerance / SCALE_USEC;
   
                   /*
                    * Leap second processing. If in leap-insert state at
                    * the end of the day, the system clock is set back one
                    * second; if in leap-delete state, the system clock is
                    * set ahead one second. The microtime() routine or
                    * external clock driver will insure that reported time
                    * is always monotonic. The ugly divides should be
                    * replaced.
                    */
                   switch (time_state) {
   
                   case TIME_OK:
                           if (time_status & STA_INS)
                                   time_state = TIME_INS;
                           else if (time_status & STA_DEL)
                                   time_state = TIME_DEL;
                           break;
   
                   case TIME_INS:
                           if (hrestime.tv_sec % 86400 == 0) {
                                   s = hr_clock_lock();
                                   hrestime.tv_sec--;
                                   hr_clock_unlock(s);
                                   time_state = TIME_OOP;
                           }
                           break;
   
                   case TIME_DEL:
                           if ((hrestime.tv_sec + 1) % 86400 == 0) {
                                   s = hr_clock_lock();
                                   hrestime.tv_sec++;
                                   hr_clock_unlock(s);
                                   time_state = TIME_WAIT;
                           }
                           break;
   
                   case TIME_OOP:
                           time_state = TIME_WAIT;
                           break;
   
                   case TIME_WAIT:
                           if (!(time_status & (STA_INS | STA_DEL)))
                                   time_state = TIME_OK;
                   default:
                           break;
                   }
   
                   /*
                    * Compute the phase adjustment for the next second. In
                    * PLL mode, the offset is reduced by a fixed factor
                    * times the time constant. In FLL mode the offset is
                    * used directly. In either mode, the maximum phase
                    * adjustment for each second is clamped so as to spread
                    * the adjustment over not more than the number of
                    * seconds between updates.
                    */
                   if (time_offset == 0)
                           time_adj = 0;
                   else if (time_offset < 0) {
                           lltemp = -time_offset;
                           if (!(time_status & STA_FLL)) {
                                   if ((1 << time_constant) >= SCALE_KG)
                                           lltemp *= (1 << time_constant) /
                                               SCALE_KG;
                                   else
                                           lltemp = (lltemp / SCALE_KG) >>
                                               time_constant;
                           }
                           if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
                                   lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
                           time_offset += lltemp;
                           time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
                   } else {
                           lltemp = time_offset;
                           if (!(time_status & STA_FLL)) {
                                   if ((1 << time_constant) >= SCALE_KG)
                                           lltemp *= (1 << time_constant) /
                                               SCALE_KG;
                                   else
                                           lltemp = (lltemp / SCALE_KG) >>
                                               time_constant;
                           }
                           if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
                                   lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
                           time_offset -= lltemp;
                           time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
                   }
   
                   /*
                    * Compute the frequency estimate and additional phase
                    * adjustment due to frequency error for the next
                    * second. When the PPS signal is engaged, gnaw on the
                    * watchdog counter and update the frequency computed by
                    * the pll and the PPS signal.
                    */
                   pps_valid++;
                   if (pps_valid == PPS_VALID) {
                           pps_jitter = MAXTIME;
                           pps_stabil = MAXFREQ;
                           time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
                               STA_PPSWANDER | STA_PPSERROR);
                   }
                   lltemp = time_freq + pps_freq;
   
                   if (lltemp)
                           time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);
   
                   /*
                    * End of precision kernel-code fragment
                    *
                    * The section below should be modified if we are planning
                    * to use NTP for synchronization.
                    *
                    * Note: the clock synchronization code now assumes
                    * the following:
                    *   - if dosynctodr is 1, then compute the drift between
                    *      the tod chip and software time and adjust one or
                    *      the other depending on the circumstances
                    *
                    *   - if dosynctodr is 0, then the tod chip is independent
                    *      of the software clock and should not be adjusted,
                    *      but allowed to free run.  this allows NTP to sync.
                    *      hrestime without any interference from the tod chip.
                    */
   
                   tod_validate_deferred = B_FALSE;
                   mutex_enter(&tod_lock);
                   tod = tod_get();
                   drift = tod.tv_sec - hrestime.tv_sec;
                   absdrift = (drift >= 0) ? drift : -drift;
                   if (tod_needsync || absdrift > 1) {
                           int s;
                           if (absdrift > 2) {
                                   if (!tod_broken && tod_faulted == TOD_NOFAULT) {
                                           s = hr_clock_lock();
                                           hrestime = tod;
                                           membar_enter(); /* hrestime visible */
                                           timedelta = 0;
                                           timechanged++;
                                           tod_needsync = 0;
                                           hr_clock_unlock(s);
                                           callout_hrestime();
   
                                   }
                           } else {
                                   if (tod_needsync || !dosynctodr) {
                                           gethrestime(&tod);
                                           tod_set(tod);
                                           s = hr_clock_lock();
                                           if (timedelta == 0)
                                                   tod_needsync = 0;
                                           hr_clock_unlock(s);
                                   } else {
                                           /*
                                            * If the drift is 2 seconds on the
                                            * money, then the TOD is adjusting
                                            * the clock;  record that.
                                            */
                                           clock_adj_hist[adj_hist_entry++ %
                                               CLOCK_ADJ_HIST_SIZE] = now;
                                           s = hr_clock_lock();
                                           timedelta = (int64_t)drift*NANOSEC;
                                           hr_clock_unlock(s);
                                   }
                           }
                   }
                   one_sec = 0;
                   time = gethrestime_sec();  /* for crusty old kmem readers */
                   mutex_exit(&tod_lock);
   
                   /*
                    * Some drivers still depend on this... XXX
                    */
                   cv_broadcast(&lbolt_cv);
   
                   vminfo.freemem += freemem;
                   {
                           pgcnt_t maxswap, resv, free;
                           pgcnt_t avail =
                               MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);
   
                           maxswap = k_anoninfo.ani_mem_resv +
                               k_anoninfo.ani_max +avail;
                           /* Update ani_free */
                           set_anoninfo();
                           free = k_anoninfo.ani_free + avail;
                           resv = k_anoninfo.ani_phys_resv +
                               k_anoninfo.ani_mem_resv;
   
                           vminfo.swap_resv += resv;
                           /* number of reserved and allocated pages */
   #ifdef  DEBUG
                           if (maxswap < free)
                                   cmn_err(CE_WARN, "clock: maxswap < free");
                           if (maxswap < resv)
                                   cmn_err(CE_WARN, "clock: maxswap < resv");
   #endif
                           vminfo.swap_alloc += maxswap - free;
                           vminfo.swap_avail += maxswap - resv;
                           vminfo.swap_free += free;
                   }
                   vminfo.updates++;
                   if (nrunnable) {
                           sysinfo.runque += nrunnable;
                           sysinfo.runocc++;
                   }
                   if (nswapped) {
                           sysinfo.swpque += nswapped;
                           sysinfo.swpocc++;
                   }
                   sysinfo.waiting += w_io;
                   sysinfo.updates++;
   
                   /*
                    * Wake up fsflush to write out DELWRI
                    * buffers, dirty pages and other cached
                    * administrative data, e.g. inodes.
                    */
                   if (--fsflushcnt <= 0) {
                           fsflushcnt = tune.t_fsflushr;
                           cv_signal(&fsflush_cv);
                   }
   
                   vmmeter();
                   calcloadavg(genloadavg(&loadavg), hp_avenrun);
                   for (i = 0; i < 3; i++)
                           /*
                            * At the moment avenrun[] can only hold 31
                            * bits of load average as it is a signed
                            * int in the API. We need to ensure that
                            * hp_avenrun[i] >> (16 - FSHIFT) will not be
                            * too large. If it is, we put the largest value
                            * that we can use into avenrun[i]. This is
                            * kludgey, but about all we can do until we
                            * avenrun[] is declared as an array of uint64[]
                            */
                           if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
                                   avenrun[i] = (int32_t)(hp_avenrun[i] >>
                                       (16 - FSHIFT));
                           else
                                   avenrun[i] = 0x7fffffff;
   
                   cpupart = cp_list_head;
                   do {
                           calcloadavg(genloadavg(&cpupart->cp_loadavg),
                               cpupart->cp_hp_avenrun);
                   } while ((cpupart = cpupart->cp_next) != cp_list_head);
   
                   /*
                    * Wake up the swapper thread if necessary.
                    */
                   if (runin ||
                       (runout && (avefree < desfree || wake_sched_sec))) {
                           t = &t0;
                           thread_lock(t);
                           if (t->t_state == TS_STOPPED) {
                                   runin = runout = 0;
                                   wake_sched_sec = 0;
                                   t->t_whystop = 0;
                                   t->t_whatstop = 0;
                                   t->t_schedflag &= ~TS_ALLSTART;
                                   THREAD_TRANSITION(t);
                                   setfrontdq(t);
                           }
                           thread_unlock(t);
                   }
           }
   
           /*
            * Wake up the swapper if any high priority swapped-out threads
            * became runable during the last tick.
            */
           if (wake_sched) {
                   t = &t0;
                   thread_lock(t);
                   if (t->t_state == TS_STOPPED) {
                           runin = runout = 0;
                           wake_sched = 0;
                           t->t_whystop = 0;
                           t->t_whatstop = 0;
                           t->t_schedflag &= ~TS_ALLSTART;
                           THREAD_TRANSITION(t);
                           setfrontdq(t);
                   }
                   thread_unlock(t);
           }
   }
   
   void
   clock_init(void)
   {
           cyc_handler_t clk_hdlr, timer_hdlr, lbolt_hdlr;
           cyc_time_t clk_when, lbolt_when;
           int i, sz;
           intptr_t buf;
   
           /*
            * Setup handler and timer for the clock cyclic.
            */
           clk_hdlr.cyh_func = (cyc_func_t)clock;
           clk_hdlr.cyh_level = CY_LOCK_LEVEL;
           clk_hdlr.cyh_arg = NULL;
   
           clk_when.cyt_when = 0;
           clk_when.cyt_interval = nsec_per_tick;
   
           /*
            * cyclic_timer is dedicated to the ddi interface, which
            * uses the same clock resolution as the system one.
            */
           timer_hdlr.cyh_func = (cyc_func_t)cyclic_timer;
           timer_hdlr.cyh_level = CY_LOCK_LEVEL;
           timer_hdlr.cyh_arg = NULL;
   
           /*
            * The lbolt cyclic will be reprogramed to fire at a nsec_per_tick
            * interval to satisfy performance needs of the DDI lbolt consumers.
            * It is off by default.
            */
           lbolt_hdlr.cyh_func = (cyc_func_t)lbolt_cyclic;
           lbolt_hdlr.cyh_level = CY_LOCK_LEVEL;
           lbolt_hdlr.cyh_arg = NULL;
   
           lbolt_when.cyt_interval = nsec_per_tick;
   
           /*
            * Allocate cache line aligned space for the per CPU lbolt data and
            * lbolt info structures, and initialize them with their default
            * values. Note that these structures are also cache line sized.
            */
           sz = sizeof (lbolt_info_t) + CPU_CACHE_COHERENCE_SIZE;
           buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
           lb_info = (lbolt_info_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
   
           if (hz != HZ_DEFAULT)
                   lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL *
                       hz/HZ_DEFAULT;
           else
                   lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL;
   
           lb_info->lbi_thresh_calls = LBOLT_THRESH_CALLS;
   
           sz = (sizeof (lbolt_cpu_t) * max_ncpus) + CPU_CACHE_COHERENCE_SIZE;
          buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
          lb_cpu = (lbolt_cpu_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
  
          for (i = 0; i < max_ncpus; i++)
                  lb_cpu[i].lbc_counter = lb_info->lbi_thresh_calls;
  
          /*
           * Install the softint used to switch between event and cyclic driven
           * lbolt. We use a soft interrupt to make sure the context of the
           * cyclic reprogram call is safe.
           */
          lbolt_softint_add();
  
          /*
           * Since the hybrid lbolt implementation is based on a hardware counter
           * that is reset at every hardware reboot and that we'd like to have
           * the lbolt value starting at zero after both a hardware and a fast
           * reboot, we calculate the number of clock ticks the system's been up
           * and store it in the lbi_debug_time field of the lbolt info structure.
           * The value of this field will be subtracted from lbolt before
           * returning it.
           */
          lb_info->lbi_internal = lb_info->lbi_debug_time =
              (gethrtime()/nsec_per_tick);
  
          /*
           * lbolt_hybrid points at lbolt_bootstrap until now. The LBOLT_* macros
           * and lbolt_debug_{enter,return} use this value as an indication that
           * the initializaion above hasn't been completed. Setting lbolt_hybrid
           * to either lbolt_{cyclic,event}_driven here signals those code paths
           * that the lbolt related structures can be used.
           */
          if (lbolt_cyc_only) {
                  lbolt_when.cyt_when = 0;
                  lbolt_hybrid = lbolt_cyclic_driven;
          } else {
                  lbolt_when.cyt_when = CY_INFINITY;
                  lbolt_hybrid = lbolt_event_driven;
          }
  
          /*
           * Grab cpu_lock and install all three cyclics.
           */
          mutex_enter(&cpu_lock);
  
          clock_cyclic = cyclic_add(&clk_hdlr, &clk_when);
          ddi_timer_cyclic = cyclic_add(&timer_hdlr, &clk_when);
          lb_info->id.lbi_cyclic_id = cyclic_add(&lbolt_hdlr, &lbolt_when);
  
          mutex_exit(&cpu_lock);
  }
  
  /*
   * Called before calcloadavg to get 10-sec moving loadavg together
   */
  
  static int
  genloadavg(struct loadavg_s *avgs)
  {
          int avg;
          int spos; /* starting position */
          int cpos; /* moving current position */
          int i;
          int slen;
          hrtime_t hr_avg;
  
          /* 10-second snapshot, calculate first positon */
          if (avgs->lg_len == 0) {
                  return (0);
          }
          slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;
  
          spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
              S_LOADAVG_SZ + (avgs->lg_cur - 1);
          for (i = hr_avg = 0; i < slen; i++) {
                  cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
                  hr_avg += avgs->lg_loads[cpos];
          }
  
          hr_avg = hr_avg / slen;
          avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);
  
          return (avg);
  }
  
  /*
   * Run every second from clock () to update the loadavg count available to the
   * system and cpu-partitions.
   *
   * This works by sampling the previous usr, sys, wait time elapsed,
   * computing a delta, and adding that delta to the elapsed usr, sys,
   * wait increase.
   */
  
  static void
  loadavg_update()
  {
          cpu_t *cp;
          cpupart_t *cpupart;
          hrtime_t cpu_total;
          int prev;
  
          cp = cpu_list;
          loadavg.lg_total = 0;
  
          /*
           * first pass totals up per-cpu statistics for system and cpu
           * partitions
           */
  
          do {
                  struct loadavg_s *lavg;
  
                  lavg = &cp->cpu_loadavg;
  
                  cpu_total = cp->cpu_acct[CMS_USER] +
                      cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
                  /* compute delta against last total */
                  scalehrtime(&cpu_total);
                  prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
                      S_LOADAVG_SZ + (lavg->lg_cur - 1);
                  if (lavg->lg_loads[prev] <= 0) {
                          lavg->lg_loads[lavg->lg_cur] = cpu_total;
                          cpu_total = 0;
                  } else {
                          lavg->lg_loads[lavg->lg_cur] = cpu_total;
                          cpu_total = cpu_total - lavg->lg_loads[prev];
                          if (cpu_total < 0)
                                  cpu_total = 0;
                  }
  
                  lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
                  lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
                      lavg->lg_len + 1 : S_LOADAVG_SZ;
  
                  loadavg.lg_total += cpu_total;
                  cp->cpu_part->cp_loadavg.lg_total += cpu_total;
  
          } while ((cp = cp->cpu_next) != cpu_list);
  
          loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
          loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
          loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
              loadavg.lg_len + 1 : S_LOADAVG_SZ;
          /*
           * Second pass updates counts
           */
          cpupart = cp_list_head;
  
          do {
                  struct loadavg_s *lavg;
  
                  lavg = &cpupart->cp_loadavg;
                  lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
                  lavg->lg_total = 0;
                  lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
                  lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
                      lavg->lg_len + 1 : S_LOADAVG_SZ;
  
          } while ((cpupart = cpupart->cp_next) != cp_list_head);
  
  }
  
  /*
   * clock_update() - local clock update
   *
   * This routine is called by ntp_adjtime() to update the local clock
   * phase and frequency. The implementation is of an
   * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
   * routine computes new time and frequency offset estimates for each
   * call.  The PPS signal itself determines the new time offset,
   * instead of the calling argument.  Presumably, calls to
   * ntp_adjtime() occur only when the caller believes the local clock
   * is valid within some bound (+-128 ms with NTP). If the caller's
   * time is far different than the PPS time, an argument will ensue,
   * and it's not clear who will lose.
   *
   * For uncompensated quartz crystal oscillatores and nominal update
   * intervals less than 1024 s, operation should be in phase-lock mode
   * (STA_FLL = 0), where the loop is disciplined to phase. For update
   * intervals greater than this, operation should be in frequency-lock
   * mode (STA_FLL = 1), where the loop is disciplined to frequency.
   *
   * Note: mutex(&tod_lock) is in effect.
   */
  void
  clock_update(int offset)
  {
          int ltemp, mtemp, s;
  
          ASSERT(MUTEX_HELD(&tod_lock));
  
          if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
                  return;
          ltemp = offset;
          if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
                  ltemp = pps_offset;
  
          /*
           * Scale the phase adjustment and clamp to the operating range.
           */
          if (ltemp > MAXPHASE)
                  time_offset = MAXPHASE * SCALE_UPDATE;
          else if (ltemp < -MAXPHASE)
                  time_offset = -(MAXPHASE * SCALE_UPDATE);
          else
                  time_offset = ltemp * SCALE_UPDATE;
  
          /*
           * Select whether the frequency is to be controlled and in which
           * mode (PLL or FLL). Clamp to the operating range. Ugly
           * multiply/divide should be replaced someday.
           */
          if (time_status & STA_FREQHOLD || time_reftime == 0)
                  time_reftime = hrestime.tv_sec;
  
          mtemp = hrestime.tv_sec - time_reftime;
          time_reftime = hrestime.tv_sec;
  
          if (time_status & STA_FLL) {
                  if (mtemp >= MINSEC) {
                          ltemp = ((time_offset / mtemp) * (SCALE_USEC /
                              SCALE_UPDATE));
                          if (ltemp)
                                  time_freq += ltemp / SCALE_KH;
                  }
          } else {
                  if (mtemp < MAXSEC) {
                          ltemp *= mtemp;
                          if (ltemp)
                                  time_freq += (int)(((int64_t)ltemp *
                                      SCALE_USEC) / SCALE_KF)
                                      / (1 << (time_constant * 2));
                  }
          }
          if (time_freq > time_tolerance)
                  time_freq = time_tolerance;
          else if (time_freq < -time_tolerance)
                  time_freq = -time_tolerance;
  
          s = hr_clock_lock();
          tod_needsync = 1;
          hr_clock_unlock(s);
  }
  
  /*
   * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
   *
   * This routine is called at each PPS interrupt in order to discipline
   * the CPU clock oscillator to the PPS signal. It measures the PPS phase
   * and leaves it in a handy spot for the clock() routine. It
   * integrates successive PPS phase differences and calculates the
   * frequency offset. This is used in clock() to discipline the CPU
   * clock oscillator so that intrinsic frequency error is cancelled out.
   * The code requires the caller to capture the time and hardware counter
   * value at the on-time PPS signal transition.
   *
   * Note that, on some Unix systems, this routine runs at an interrupt
   * priority level higher than the timer interrupt routine clock().
   * Therefore, the variables used are distinct from the clock()
   * variables, except for certain exceptions: The PPS frequency pps_freq
   * and phase pps_offset variables are determined by this routine and
   * updated atomically. The time_tolerance variable can be considered a
   * constant, since it is infrequently changed, and then only when the
   * PPS signal is disabled. The watchdog counter pps_valid is updated
   * once per second by clock() and is atomically cleared in this
   * routine.
   *
   * tvp is the time of the last tick; usec is a microsecond count since the
   * last tick.
   *
   * Note: In Solaris systems, the tick value is actually given by
   *       usec_per_tick.  This is called from the serial driver cdintr(),
   *       or equivalent, at a high PIL.  Because the kernel keeps a
   *       highresolution time, the following code can accept either
   *       the traditional argument pair, or the current highres timestamp
   *       in tvp and zero in usec.
   */
  void
  ddi_hardpps(struct timeval *tvp, int usec)
  {
          int u_usec, v_usec, bigtick;
          time_t cal_sec;
          int cal_usec;
  
          /*
           * An occasional glitch can be produced when the PPS interrupt
           * occurs in the clock() routine before the time variable is
           * updated. Here the offset is discarded when the difference
           * between it and the last one is greater than tick/2, but not
           * if the interval since the first discard exceeds 30 s.
           */
          time_status |= STA_PPSSIGNAL;
          time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
          pps_valid = 0;
          u_usec = -tvp->tv_usec;
          if (u_usec < -(MICROSEC/2))
                  u_usec += MICROSEC;
          v_usec = pps_offset - u_usec;
          if (v_usec < 0)
                  v_usec = -v_usec;
          if (v_usec > (usec_per_tick >> 1)) {
                  if (pps_glitch > MAXGLITCH) {
                          pps_glitch = 0;
                          pps_tf[2] = u_usec;
                          pps_tf[1] = u_usec;
                  } else {
                          pps_glitch++;
                          u_usec = pps_offset;
                  }
          } else
                  pps_glitch = 0;
  
          /*
           * A three-stage median filter is used to help deglitch the pps
           * time. The median sample becomes the time offset estimate; the
           * difference between the other two samples becomes the time
           * dispersion (jitter) estimate.
           */
          pps_tf[2] = pps_tf[1];
          pps_tf[1] = pps_tf[0];
          pps_tf[0] = u_usec;
          if (pps_tf[0] > pps_tf[1]) {
                  if (pps_tf[1] > pps_tf[2]) {
                          pps_offset = pps_tf[1];         /* 0 1 2 */
                          v_usec = pps_tf[0] - pps_tf[2];
                  } else if (pps_tf[2] > pps_tf[0]) {
                          pps_offset = pps_tf[0];         /* 2 0 1 */
                          v_usec = pps_tf[2] - pps_tf[1];
                  } else {
                          pps_offset = pps_tf[2];         /* 0 2 1 */
                          v_usec = pps_tf[0] - pps_tf[1];
                  }
          } else {
                  if (pps_tf[1] < pps_tf[2]) {
                          pps_offset = pps_tf[1];         /* 2 1 0 */
                          v_usec = pps_tf[2] - pps_tf[0];
                  } else  if (pps_tf[2] < pps_tf[0]) {
                          pps_offset = pps_tf[0];         /* 1 0 2 */
                          v_usec = pps_tf[1] - pps_tf[2];
                  } else {
                          pps_offset = pps_tf[2];         /* 1 2 0 */
                          v_usec = pps_tf[1] - pps_tf[0];
                  }
          }
          if (v_usec > MAXTIME)
                  pps_jitcnt++;
          v_usec = (v_usec << PPS_AVG) - pps_jitter;
          pps_jitter += v_usec / (1 << PPS_AVG);
          if (pps_jitter > (MAXTIME >> 1))
                  time_status |= STA_PPSJITTER;
  
          /*
           * During the calibration interval adjust the starting time when
           * the tick overflows. At the end of the interval compute the
           * duration of the interval and the difference of the hardware
           * counters at the beginning and end of the interval. This code
           * is deliciously complicated by the fact valid differences may
           * exceed the value of tick when using long calibration
           * intervals and small ticks. Note that the counter can be
           * greater than tick if caught at just the wrong instant, but
           * the values returned and used here are correct.
           */
          bigtick = (int)usec_per_tick * SCALE_USEC;
          pps_usec -= pps_freq;
          if (pps_usec >= bigtick)
                  pps_usec -= bigtick;
          if (pps_usec < 0)
                  pps_usec += bigtick;
          pps_time.tv_sec++;
          pps_count++;
          if (pps_count < (1 << pps_shift))
                  return;
          pps_count = 0;
          pps_calcnt++;
          u_usec = usec * SCALE_USEC;
          v_usec = pps_usec - u_usec;
          if (v_usec >= bigtick >> 1)
                  v_usec -= bigtick;
          if (v_usec < -(bigtick >> 1))
                  v_usec += bigtick;
          if (v_usec < 0)
                  v_usec = -(-v_usec >> pps_shift);
          else
                  v_usec = v_usec >> pps_shift;
          pps_usec = u_usec;
          cal_sec = tvp->tv_sec;
          cal_usec = tvp->tv_usec;
          cal_sec -= pps_time.tv_sec;
          cal_usec -= pps_time.tv_usec;
          if (cal_usec < 0) {
                  cal_usec += MICROSEC;
                  cal_sec--;
          }
          pps_time = *tvp;
  
          /*
           * Check for lost interrupts, noise, excessive jitter and
           * excessive frequency error. The number of timer ticks during
           * the interval may vary +-1 tick. Add to this a margin of one
           * tick for the PPS signal jitter and maximum frequency
           * deviation. If the limits are exceeded, the calibration
           * interval is reset to the minimum and we start over.
           */
          u_usec = (int)usec_per_tick << 1;
          if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
              (cal_sec == 0 && cal_usec < u_usec)) ||
              v_usec > time_tolerance || v_usec < -time_tolerance) {
                  pps_errcnt++;
                  pps_shift = PPS_SHIFT;
                  pps_intcnt = 0;
                  time_status |= STA_PPSERROR;
                  return;
          }
  
          /*
           * A three-stage median filter is used to help deglitch the pps
           * frequency. The median sample becomes the frequency offset
           * estimate; the difference between the other two samples
           * becomes the frequency dispersion (stability) estimate.
           */
          pps_ff[2] = pps_ff[1];
          pps_ff[1] = pps_ff[0];
          pps_ff[0] = v_usec;
          if (pps_ff[0] > pps_ff[1]) {
                  if (pps_ff[1] > pps_ff[2]) {
                          u_usec = pps_ff[1];             /* 0 1 2 */
                          v_usec = pps_ff[0] - pps_ff[2];
                  } else if (pps_ff[2] > pps_ff[0]) {
                          u_usec = pps_ff[0];             /* 2 0 1 */
                          v_usec = pps_ff[2] - pps_ff[1];
                  } else {
                          u_usec = pps_ff[2];             /* 0 2 1 */
                          v_usec = pps_ff[0] - pps_ff[1];
                  }
          } else {
                  if (pps_ff[1] < pps_ff[2]) {
                          u_usec = pps_ff[1];             /* 2 1 0 */
                          v_usec = pps_ff[2] - pps_ff[0];
                  } else  if (pps_ff[2] < pps_ff[0]) {
                          u_usec = pps_ff[0];             /* 1 0 2 */
                          v_usec = pps_ff[1] - pps_ff[2];
                  } else {
                          u_usec = pps_ff[2];             /* 1 2 0 */
                          v_usec = pps_ff[1] - pps_ff[0];
                  }
          }
  
          /*
           * Here the frequency dispersion (stability) is updated. If it
           * is less than one-fourth the maximum (MAXFREQ), the frequency
           * offset is updated as well, but clamped to the tolerance. It
           * will be processed later by the clock() routine.
           */
          v_usec = (v_usec >> 1) - pps_stabil;
          if (v_usec < 0)
                  pps_stabil -= -v_usec >> PPS_AVG;
          else
                  pps_stabil += v_usec >> PPS_AVG;
          if (pps_stabil > MAXFREQ >> 2) {
                  pps_stbcnt++;
                  time_status |= STA_PPSWANDER;
                  return;
          }
          if (time_status & STA_PPSFREQ) {
                  if (u_usec < 0) {
                          pps_freq -= -u_usec >> PPS_AVG;
                          if (pps_freq < -time_tolerance)
                                  pps_freq = -time_tolerance;
                          u_usec = -u_usec;
                  } else {
                          pps_freq += u_usec >> PPS_AVG;
                          if (pps_freq > time_tolerance)
                                  pps_freq = time_tolerance;
                  }
          }
  
          /*
           * Here the calibration interval is adjusted. If the maximum
           * time difference is greater than tick / 4, reduce the interval
           * by half. If this is not the case for four consecutive
           * intervals, double the interval.
           */
          if (u_usec << pps_shift > bigtick >> 2) {
                  pps_intcnt = 0;
                  if (pps_shift > PPS_SHIFT)
                          pps_shift--;
          } else if (pps_intcnt >= 4) {
                  pps_intcnt = 0;
                  if (pps_shift < PPS_SHIFTMAX)
                          pps_shift++;
          } else
                  pps_intcnt++;
  
          /*
           * If recovering from kmdb, then make sure the tod chip gets resynced.
           * If we took an early exit above, then we don't yet have a stable
           * calibration signal to lock onto, so don't mark the tod for sync
           * until we get all the way here.
           */
          {
                  int s = hr_clock_lock();
  
                  tod_needsync = 1;
                  hr_clock_unlock(s);
          }
  }
  
  /*
   * Handle clock tick processing for a thread.
   * Check for timer action, enforce CPU rlimit, do profiling etc.
   */
  void
  clock_tick(kthread_t *t, int pending)
  {
          struct proc *pp;
          klwp_id_t    lwp;
          struct as *as;
          clock_t ticks;
          int     poke = 0;               /* notify another CPU */
          int     user_mode;
          size_t   rss;
          int i, total_usec, usec;
          rctl_qty_t secs;
  
          ASSERT(pending > 0);
  
          /* Must be operating on a lwp/thread */
          if ((lwp = ttolwp(t)) == NULL) {
                  panic("clock_tick: no lwp");
                  /*NOTREACHED*/
          }
  
          for (i = 0; i < pending; i++) {
                  CL_TICK(t);     /* Class specific tick processing */
                  DTRACE_SCHED1(tick, kthread_t *, t);
          }
  
          pp = ttoproc(t);
  
          /* pp->p_lock makes sure that the thread does not exit */
          ASSERT(MUTEX_HELD(&pp->p_lock));
  
          user_mode = (lwp->lwp_state == LWP_USER);
  
          ticks = (pp->p_utime + pp->p_stime) % hz;
          /*
           * Update process times. Should use high res clock and state
           * changes instead of statistical sampling method. XXX
           */
          if (user_mode) {
                  pp->p_utime += pending;
          } else {
                  pp->p_stime += pending;
          }
  
          pp->p_ttime += pending;
          as = pp->p_as;
  
          /*
           * Update user profiling statistics. Get the pc from the
           * lwp when the AST happens.
           */
          if (pp->p_prof.pr_scale) {
                  atomic_add_32(&lwp->lwp_oweupc, (int32_t)pending);
                  if (user_mode) {
                          poke = 1;
                          aston(t);
                  }
          }
  
          /*
           * If CPU was in user state, process lwp-virtual time
           * interval timer. The value passed to itimerdecr() has to be
           * in microseconds and has to be less than one second. Hence
           * this loop.
           */
          total_usec = usec_per_tick * pending;
          while (total_usec > 0) {
                  usec = MIN(total_usec, (MICROSEC - 1));
                  if (user_mode &&
                      timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
                      itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec) == 0) {
                          poke = 1;
                          sigtoproc(pp, t, SIGVTALRM);
                  }
                  total_usec -= usec;
          }
  
          /*
           * If CPU was in user state, process lwp-profile
           * interval timer.
           */
          total_usec = usec_per_tick * pending;
          while (total_usec > 0) {
                  usec = MIN(total_usec, (MICROSEC - 1));
                  if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
                      itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec) == 0) {
                          poke = 1;
                          sigtoproc(pp, t, SIGPROF);
                  }
                  total_usec -= usec;
          }
  
          /*
           * Enforce CPU resource controls:
           *   (a) process.max-cpu-time resource control
           *
           * Perform the check only if we have accumulated more a second.
           */
          if ((ticks + pending) >= hz) {
                  (void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
                      (pp->p_utime + pp->p_stime)/hz, RCA_UNSAFE_SIGINFO);
          }
  
          /*
           *   (b) task.max-cpu-time resource control
           *
           * If we have accumulated enough ticks, increment the task CPU
           * time usage and test for the resource limit. This minimizes the
           * number of calls to the rct_test(). The task CPU time mutex
           * is highly contentious as many processes can be sharing a task.
           */
          if (pp->p_ttime >= clock_tick_proc_max) {
                  secs = task_cpu_time_incr(pp->p_task, pp->p_ttime);
                  pp->p_ttime = 0;
                  if (secs) {
                          (void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls,
                              pp, secs, RCA_UNSAFE_SIGINFO);
                  }
          }
  
          /*
           * Update memory usage for the currently running process.
           */
          rss = rm_asrss(as);
          PTOU(pp)->u_mem += rss;
          if (rss > PTOU(pp)->u_mem_max)
                  PTOU(pp)->u_mem_max = rss;
  
          /*
           * Notify the CPU the thread is running on.
           */
          if (poke && t->t_cpu != CPU)
                  poke_cpu(t->t_cpu->cpu_id);
  }
  
  void
  profil_tick(uintptr_t upc)
  {
          int ticks;
          proc_t *p = ttoproc(curthread);
          klwp_t *lwp = ttolwp(curthread);
          struct prof *pr = &p->p_prof;
  
          do {
                  ticks = lwp->lwp_oweupc;
          } while (cas32(&lwp->lwp_oweupc, ticks, 0) != ticks);
  
          mutex_enter(&p->p_pflock);
          if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
                  /*
                   * Old-style profiling
                   */
                  uint16_t *slot = pr->pr_base;
                  uint16_t old, new;
                  if (pr->pr_scale != 2) {
                          uintptr_t delta = upc - pr->pr_off;
                          uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
                              (((delta & 0xffff) * pr->pr_scale) >> 16);
                          if (byteoff >= (uintptr_t)pr->pr_size) {
                                  mutex_exit(&p->p_pflock);
                                  return;
                          }
                          slot += byteoff / sizeof (uint16_t);
                  }
                  if (fuword16(slot, &old) < 0 ||
                      (new = old + ticks) > SHRT_MAX ||
                      suword16(slot, new) < 0) {
                          pr->pr_scale = 0;
                  }
          } else if (pr->pr_scale == 1) {
                  /*
                   * PC Sampling
                   */
                  model_t model = lwp_getdatamodel(lwp);
                  int result;
  #ifdef __lint
                  model = model;
  #endif
                  while (ticks-- > 0) {
                          if (pr->pr_samples == pr->pr_size) {
                                  /* buffer full, turn off sampling */
                                  pr->pr_scale = 0;
                                  break;
                          }
                          switch (SIZEOF_PTR(model)) {
                          case sizeof (uint32_t):
                                  result = suword32(pr->pr_base, (uint32_t)upc);
                                  break;
  #ifdef _LP64
                          case sizeof (uint64_t):
                                  result = suword64(pr->pr_base, (uint64_t)upc);
                                  break;
  #endif
                          default:
                                  cmn_err(CE_WARN, "profil_tick: unexpected "
                                      "data model");
                                  result = -1;
                                  break;
                          }
                          if (result != 0) {
                                  pr->pr_scale = 0;
                                  break;
                          }
                          pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
                          pr->pr_samples++;
                  }
          }
          mutex_exit(&p->p_pflock);
  }
  
  static void
  delay_wakeup(void *arg)
  {
          kthread_t       *t = arg;
  
          mutex_enter(&t->t_delay_lock);
          cv_signal(&t->t_delay_cv);
          mutex_exit(&t->t_delay_lock);
  }
  
  /*
   * The delay(9F) man page indicates that it can only be called from user or
   * kernel context - detect and diagnose bad calls. The following macro will
   * produce a limited number of messages identifying bad callers.  This is done
   * in a macro so that caller() is meaningful. When a bad caller is identified,
   * switching to 'drv_usecwait(TICK_TO_USEC(ticks));' may be appropriate.
   */
  #define DELAY_CONTEXT_CHECK()   {                                       \
          uint32_t        m;                                              \
          char            *f;                                             \
          ulong_t         off;                                            \
                                                                          \
          m = delay_from_interrupt_msg;                                   \
          if (delay_from_interrupt_diagnose && servicing_interrupt() &&   \
              !panicstr && !devinfo_freeze &&                             \
              atomic_cas_32(&delay_from_interrupt_msg, m ? m : 1, m-1)) { \
                  f = modgetsymname((uintptr_t)caller(), &off);           \
                  cmn_err(CE_WARN, "delay(9F) called from "               \
                      "interrupt context: %s`%s",                         \
                      mod_containing_pc(caller()), f ? f : "...");        \
          }                                                               \
  }
  
  /*
   * delay_common: common delay code.
   */
  static void
  delay_common(clock_t ticks)
  {
          kthread_t       *t = curthread;
          clock_t         deadline;
          clock_t         timeleft;
          callout_id_t    id;
  
          /* If timeouts aren't running all we can do is spin. */
          if (panicstr || devinfo_freeze) {
                  /* Convert delay(9F) call into drv_usecwait(9F) call. */
                  if (ticks > 0)
                          drv_usecwait(TICK_TO_USEC(ticks));
                  return;
          }
  
          deadline = ddi_get_lbolt() + ticks;
          while ((timeleft = deadline - ddi_get_lbolt()) > 0) {
                  mutex_enter(&t->t_delay_lock);
                  id = timeout_default(delay_wakeup, t, timeleft);
                  cv_wait(&t->t_delay_cv, &t->t_delay_lock);
                  mutex_exit(&t->t_delay_lock);
                  (void) untimeout_default(id, 0);
          }
  }
  
  /*
   * Delay specified number of clock ticks.
   */
  void
  delay(clock_t ticks)
  {
          DELAY_CONTEXT_CHECK();
  
          delay_common(ticks);
  }
  
  /*
   * Delay a random number of clock ticks between 1 and ticks.
   */
  void
  delay_random(clock_t ticks)
  {
          int     r;
  
          DELAY_CONTEXT_CHECK();
  
          (void) random_get_pseudo_bytes((void *)&r, sizeof (r));
          if (ticks == 0)
                  ticks = 1;
          ticks = (r % ticks) + 1;
          delay_common(ticks);
  }
  
  /*
   * Like delay, but interruptible by a signal.
   */
  int
  delay_sig(clock_t ticks)
  {
          kthread_t       *t = curthread;
          clock_t         deadline;
          clock_t         rc;
  
          /* If timeouts aren't running all we can do is spin. */
          if (panicstr || devinfo_freeze) {
                  if (ticks > 0)
                          drv_usecwait(TICK_TO_USEC(ticks));
                  return (0);
          }
  
          deadline = ddi_get_lbolt() + ticks;
          mutex_enter(&t->t_delay_lock);
          do {
                  rc = cv_timedwait_sig(&t->t_delay_cv,
                      &t->t_delay_lock, deadline);
                  /* loop until past deadline or signaled */
          } while (rc > 0);
          mutex_exit(&t->t_delay_lock);
          if (rc == 0)
                  return (EINTR);
          return (0);
  }
  
  
  #define SECONDS_PER_DAY 86400
  
  /*
   * Initialize the system time based on the TOD chip.  approx is used as
   * an approximation of time (e.g. from the filesystem) in the event that
   * the TOD chip has been cleared or is unresponsive.  An approx of -1
   * means the filesystem doesn't keep time.
   */
  void
  clkset(time_t approx)
  {
          timestruc_t ts;
          int spl;
          int set_clock = 0;
  
          mutex_enter(&tod_lock);
          ts = tod_get();
  
          if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
                  /*
                   * If the TOD chip is reporting some time after 1971,
                   * then it probably didn't lose power or become otherwise
                   * cleared in the recent past;  check to assure that
                   * the time coming from the filesystem isn't in the future
                   * according to the TOD chip.
                   */
                  if (approx != -1 && approx > ts.tv_sec) {
                          cmn_err(CE_WARN, "Last shutdown is later "
                              "than time on time-of-day chip; check date.");
                  }
          } else {
                  /*
                   * If the TOD chip isn't giving correct time, set it to the
                   * greater of i) approx and ii) 1987. That way if approx
                   * is negative or is earlier than 1987, we set the clock
                   * back to a time when Oliver North, ALF and Dire Straits
                   * were all on the collective brain:  1987.
                   */
                  timestruc_t tmp;
                  time_t diagnose_date = (1987 - 1970) * 365 * SECONDS_PER_DAY;
                  ts.tv_sec = (approx > diagnose_date ? approx : diagnose_date);
                  ts.tv_nsec = 0;
  
                  /*
                   * Attempt to write the new time to the TOD chip.  Set spl high
                   * to avoid getting preempted between the tod_set and tod_get.
                   */
                  spl = splhi();
                  tod_set(ts);
                  tmp = tod_get();
                  splx(spl);
  
                  if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
                          tod_broken = 1;
                          dosynctodr = 0;
                          cmn_err(CE_WARN, "Time-of-day chip unresponsive.");
                  } else {
                          cmn_err(CE_WARN, "Time-of-day chip had "
                              "incorrect date; check and reset.");
                  }
                  set_clock = 1;
          }
  
          if (!boot_time) {
                  boot_time = ts.tv_sec;
                  set_clock = 1;
          }
  
          if (set_clock)
                  set_hrestime(&ts);
  
          mutex_exit(&tod_lock);
  }
  
  int     timechanged;    /* for testing if the system time has been reset */
  
  void
  set_hrestime(timestruc_t *ts)
  {
          int spl = hr_clock_lock();
          hrestime = *ts;
          membar_enter(); /* hrestime must be visible before timechanged++ */
          timedelta = 0;
          timechanged++;
          hr_clock_unlock(spl);
          callout_hrestime();
  }
  
  static uint_t deadman_seconds;
  static uint32_t deadman_panics;
  static int deadman_enabled = 0;
  static int deadman_panic_timers = 1;
  
  static void
  deadman(void)
  {
          if (panicstr) {
                  /*
                   * During panic, other CPUs besides the panic
                   * master continue to handle cyclics and some other
                   * interrupts.  The code below is intended to be
                   * single threaded, so any CPU other than the master
                   * must keep out.
                   */
                  if (CPU->cpu_id != panic_cpu.cpu_id)
                          return;
  
                  if (!deadman_panic_timers)
                          return; /* allow all timers to be manually disabled */
  
                  /*
                   * If we are generating a crash dump or syncing filesystems and
                   * the corresponding timer is set, decrement it and re-enter
                   * the panic code to abort it and advance to the next state.
                   * The panic states and triggers are explained in panic.c.
                   */
                  if (panic_dump) {
                          if (dump_timeleft && (--dump_timeleft == 0)) {
                                  panic("panic dump timeout");
                                  /*NOTREACHED*/
                          }
                  } else if (panic_sync) {
                          if (sync_timeleft && (--sync_timeleft == 0)) {
                                  panic("panic sync timeout");
                                  /*NOTREACHED*/
                          }
                  }
  
                  return;
          }
  
          if (deadman_counter != CPU->cpu_deadman_counter) {
                  CPU->cpu_deadman_counter = deadman_counter;
                  CPU->cpu_deadman_countdown = deadman_seconds;
                  return;
          }
  
          if (--CPU->cpu_deadman_countdown > 0)
                  return;
  
          /*
           * Regardless of whether or not we actually bring the system down,
           * bump the deadman_panics variable.
           *
           * N.B. deadman_panics is incremented once for each CPU that
           * passes through here.  It's expected that all the CPUs will
           * detect this condition within one second of each other, so
           * when deadman_enabled is off, deadman_panics will
           * typically be a multiple of the total number of CPUs in
           * the system.
           */
          atomic_add_32(&deadman_panics, 1);
  
          if (!deadman_enabled) {
                  CPU->cpu_deadman_countdown = deadman_seconds;
                  return;
          }
  
          /*
           * If we're here, we want to bring the system down.
           */
          panic("deadman: timed out after %d seconds of clock "
              "inactivity", deadman_seconds);
          /*NOTREACHED*/
  }
  
  /*ARGSUSED*/
  static void
  deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
  {
          cpu->cpu_deadman_counter = 0;
          cpu->cpu_deadman_countdown = deadman_seconds;
  
          hdlr->cyh_func = (cyc_func_t)deadman;
          hdlr->cyh_level = CY_HIGH_LEVEL;
          hdlr->cyh_arg = NULL;
  
          /*
           * Stagger the CPUs so that they don't all run deadman() at
           * the same time.  Simplest reason to do this is to make it
           * more likely that only one CPU will panic in case of a
           * timeout.  This is (strictly speaking) an aesthetic, not a
           * technical consideration.
           */
          when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
          when->cyt_interval = NANOSEC;
  }
  
  
  void
  deadman_init(void)
  {
          cyc_omni_handler_t hdlr;
  
          if (deadman_seconds == 0)
                  deadman_seconds = snoop_interval / MICROSEC;
  
          if (snooping)
                  deadman_enabled = 1;
  
          hdlr.cyo_online = deadman_online;
          hdlr.cyo_offline = NULL;
          hdlr.cyo_arg = NULL;
  
          mutex_enter(&cpu_lock);
          deadman_cyclic = cyclic_add_omni(&hdlr);
          mutex_exit(&cpu_lock);
  }
  
  /*
   * tod_fault() is for updating tod validate mechanism state:
   * (1) TOD_NOFAULT: for resetting the state to 'normal'.
   *     currently used for debugging only
   * (2) The following four cases detected by tod validate mechanism:
   *       TOD_REVERSED: current tod value is less than previous value.
   *       TOD_STALLED: current tod value hasn't advanced.
   *       TOD_JUMPED: current tod value advanced too far from previous value.
   *       TOD_RATECHANGED: the ratio between average tod delta and
   *       average tick delta has changed.
   * (3) TOD_RDONLY: when the TOD clock is not writeable e.g. because it is
   *     a virtual TOD provided by a hypervisor.
   */
  enum tod_fault_type
  tod_fault(enum tod_fault_type ftype, int off)
  {
          ASSERT(MUTEX_HELD(&tod_lock));
  
          if (tod_faulted != ftype) {
                  switch (ftype) {
                  case TOD_NOFAULT:
                          plat_tod_fault(TOD_NOFAULT);
                          cmn_err(CE_NOTE, "Restarted tracking "
                              "Time of Day clock.");
                          tod_faulted = ftype;
                          break;
                  case TOD_REVERSED:
                  case TOD_JUMPED:
                          if (tod_faulted == TOD_NOFAULT) {
                                  plat_tod_fault(ftype);
                                  cmn_err(CE_WARN, "Time of Day clock error: "
                                      "reason [%s by 0x%x]. -- "
                                      " Stopped tracking Time Of Day clock.",
                                      tod_fault_table[ftype], off);
                                  tod_faulted = ftype;
                          }
                          break;
                  case TOD_STALLED:
                  case TOD_RATECHANGED:
                          if (tod_faulted == TOD_NOFAULT) {
                                  plat_tod_fault(ftype);
                                  cmn_err(CE_WARN, "Time of Day clock error: "
                                      "reason [%s]. -- "
                                      " Stopped tracking Time Of Day clock.",
                                      tod_fault_table[ftype]);
                                  tod_faulted = ftype;
                          }
                          break;
                  case TOD_RDONLY:
                          if (tod_faulted == TOD_NOFAULT) {
                                  plat_tod_fault(ftype);
                                  cmn_err(CE_NOTE, "!Time of Day clock is "
                                      "Read-Only; set of Date/Time will not "
                                      "persist across reboot.");
                                  tod_faulted = ftype;
                          }
                          break;
                  default:
                          break;
                  }
          }
          return (tod_faulted);
  }
  
  /*
   * Two functions that allow tod_status_flag to be manipulated by functions
   * external to this file.
   */
  
  void
  tod_status_set(int tod_flag)
  {
          tod_status_flag |= tod_flag;
  }
  
  void
  tod_status_clear(int tod_flag)
  {
          tod_status_flag &= ~tod_flag;
  }
  
  /*
   * Record a timestamp and the value passed to tod_set().  The next call to
   * tod_validate() can use these values, prev_set_tick and prev_set_tod,
   * when checking the timestruc_t returned by tod_get().  Ordinarily,
   * tod_validate() will use prev_tick and prev_tod for this task but these
   * become obsolete, and will be re-assigned with the prev_set_* values,
   * in the case when the TOD is re-written.
   */
  void
  tod_set_prev(timestruc_t ts)
  {
          if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
              tod_validate_deferred) {
                  return;
          }
          prev_set_tick = gethrtime();
          /*
           * A negative value will be set to zero in utc_to_tod() so we fake
           * a zero here in such a case.  This would need to change if the
           * behavior of utc_to_tod() changes.
           */
          prev_set_tod = ts.tv_sec < 0 ? 0 : ts.tv_sec;
  }
  
  /*
   * tod_validate() is used for checking values returned by tod_get().
   * Four error cases can be detected by this routine:
   *   TOD_REVERSED: current tod value is less than previous.
   *   TOD_STALLED: current tod value hasn't advanced.
   *   TOD_JUMPED: current tod value advanced too far from previous value.
   *   TOD_RATECHANGED: the ratio between average tod delta and
   *   average tick delta has changed.
   */
  time_t
  tod_validate(time_t tod)
  {
          time_t diff_tod;
          hrtime_t diff_tick;
  
          long dtick;
          int dtick_delta;
  
          int off = 0;
          enum tod_fault_type tod_bad = TOD_NOFAULT;
  
          static int firsttime = 1;
  
          static time_t prev_tod = 0;
          static hrtime_t prev_tick = 0;
          static long dtick_avg = TOD_REF_FREQ;
  
          int cpr_resume_done = 0;
          int dr_resume_done = 0;
  
          hrtime_t tick = gethrtime();
  
          ASSERT(MUTEX_HELD(&tod_lock));
  
          /*
           * tod_validate_enable is patchable via /etc/system.
           * If TOD is already faulted, or if TOD validation is deferred,
           * there is nothing to do.
           */
          if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
              tod_validate_deferred) {
                  return (tod);
          }
  
          /*
           * If this is the first time through, we just need to save the tod
           * we were called with and hrtime so we can use them next time to
           * validate tod_get().
           */
          if (firsttime) {
                  firsttime = 0;
                  prev_tod = tod;
                  prev_tick = tick;
                  return (tod);
          }
  
          /*
           * Handle any flags that have been turned on by tod_status_set().
           * In the case where a tod_set() is done and then a subsequent
           * tod_get() fails (ie, both TOD_SET_DONE and TOD_GET_FAILED are
           * true), we treat the TOD_GET_FAILED with precedence by switching
           * off the flag, returning tod and leaving TOD_SET_DONE asserted
           * until such time as tod_get() completes successfully.
           */
          if (tod_status_flag & TOD_GET_FAILED) {
                  /*
                   * tod_get() has encountered an issue, possibly transitory,
                   * when reading TOD.  We'll just return the incoming tod
                   * value (which is actually hrestime.tv_sec in this case)
                   * and when we get a genuine tod, following a successful
                   * tod_get(), we can validate using prev_tod and prev_tick.
                   */
                  tod_status_flag &= ~TOD_GET_FAILED;
                  return (tod);
          } else if (tod_status_flag & TOD_SET_DONE) {
                  /*
                   * TOD has been modified.  Just before the TOD was written,
                   * tod_set_prev() saved tod and hrtime; we can now use
                   * those values, prev_set_tod and prev_set_tick, to validate
                   * the incoming tod that's just been read.
                   */
                  prev_tod = prev_set_tod;
                  prev_tick = prev_set_tick;
                  dtick_avg = TOD_REF_FREQ;
                  tod_status_flag &= ~TOD_SET_DONE;
                  /*
                   * If a tod_set() preceded a cpr_suspend() without an
                   * intervening tod_validate(), we need to ensure that a
                   * TOD_JUMPED condition is ignored.
                   * Note this isn't a concern in the case of DR as we've
                   * just reassigned dtick_avg, above.
                   */
                  if (tod_status_flag & TOD_CPR_RESUME_DONE) {
                          cpr_resume_done = 1;
                          tod_status_flag &= ~TOD_CPR_RESUME_DONE;
                  }
          } else if (tod_status_flag & TOD_CPR_RESUME_DONE) {
                  /*
                   * The system's coming back from a checkpoint resume.
                   */
                  cpr_resume_done = 1;
                  tod_status_flag &= ~TOD_CPR_RESUME_DONE;
                  /*
                   * We need to handle the possibility of a CPR suspend
                   * operation having been initiated whilst a DR event was
                   * in-flight.
                   */
                  if (tod_status_flag & TOD_DR_RESUME_DONE) {
                          dr_resume_done = 1;
                          tod_status_flag &= ~TOD_DR_RESUME_DONE;
                  }
          } else if (tod_status_flag & TOD_DR_RESUME_DONE) {
                  /*
                   * A Dynamic Reconfiguration event has taken place.
                   */
                  dr_resume_done = 1;
                  tod_status_flag &= ~TOD_DR_RESUME_DONE;
          }
  
          /* test hook */
          switch (tod_unit_test) {
          case 1: /* for testing jumping tod */
                  tod += tod_test_injector;
                  tod_unit_test = 0;
                  break;
          case 2: /* for testing stuck tod bit */
                  tod |= 1 << tod_test_injector;
                  tod_unit_test = 0;
                  break;
          case 3: /* for testing stalled tod */
                  tod = prev_tod;
                  tod_unit_test = 0;
                  break;
          case 4: /* reset tod fault status */
                  (void) tod_fault(TOD_NOFAULT, 0);
                  tod_unit_test = 0;
                  break;
          default:
                  break;
          }
  
          diff_tod = tod - prev_tod;
          diff_tick = tick - prev_tick;
  
          ASSERT(diff_tick >= 0);
  
          if (diff_tod < 0) {
                  /* ERROR - tod reversed */
                  tod_bad = TOD_REVERSED;
                  off = (int)(prev_tod - tod);
          } else if (diff_tod == 0) {
                  /* tod did not advance */
                  if (diff_tick > TOD_STALL_THRESHOLD) {
                          /* ERROR - tod stalled */
                          tod_bad = TOD_STALLED;
                  } else {
                          /*
                           * Make sure we don't update prev_tick
                           * so that diff_tick is calculated since
                           * the first diff_tod == 0
                           */
                          return (tod);
                  }
          } else {
                  /* calculate dtick */
                  dtick = diff_tick / diff_tod;
  
                  /* update dtick averages */
                  dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);
  
                  /*
                   * Calculate dtick_delta as
                   * variation from reference freq in quartiles
                   */
                  dtick_delta = (dtick_avg - TOD_REF_FREQ) /
                      (TOD_REF_FREQ >> 2);
  
                  /*
                   * Even with a perfectly functioning TOD device,
                   * when the number of elapsed seconds is low the
                   * algorithm can calculate a rate that is beyond
                   * tolerance, causing an error.  The algorithm is
                   * inaccurate when elapsed time is low (less than
                   * 5 seconds).
                   */
                  if (diff_tod > 4) {
                          if (dtick < TOD_JUMP_THRESHOLD) {
                                  /*
                                   * If we've just done a CPR resume, we detect
                                   * a jump in the TOD but, actually, what's
                                   * happened is that the TOD has been increasing
                                   * whilst the system was suspended and the tick
                                   * count hasn't kept up.  We consider the first
                                   * occurrence of this after a resume as normal
                                   * and ignore it; otherwise, in a non-resume
                                   * case, we regard it as a TOD problem.
                                   */
                                  if (!cpr_resume_done) {
                                          /* ERROR - tod jumped */
                                          tod_bad = TOD_JUMPED;
                                          off = (int)diff_tod;
                                  }
                          }
                          if (dtick_delta) {
                                  /*
                                   * If we've just done a DR resume, dtick_avg
                                   * can go a bit askew so we reset it and carry
                                   * on; otherwise, the TOD is in error.
                                   */
                                  if (dr_resume_done) {
                                          dtick_avg = TOD_REF_FREQ;
                                  } else {
                                          /* ERROR - change in clock rate */
                                          tod_bad = TOD_RATECHANGED;
                                  }
                          }
                  }
          }
  
          if (tod_bad != TOD_NOFAULT) {
                  (void) tod_fault(tod_bad, off);
  
                  /*
                   * Disable dosynctodr since we are going to fault
                   * the TOD chip anyway here
                   */
                  dosynctodr = 0;
  
                  /*
                   * Set tod to the correct value from hrestime
                   */
                  tod = hrestime.tv_sec;
          }
  
          prev_tod = tod;
          prev_tick = tick;
          return (tod);
  }
  
  static void
  calcloadavg(int nrun, uint64_t *hp_ave)
  {
          static int64_t f[3] = { 135, 27, 9 };
          uint_t i;
          int64_t q, r;
  
          /*
           * Compute load average over the last 1, 5, and 15 minutes
           * (60, 300, and 900 seconds).  The constants in f[3] are for
           * exponential decay:
           * (1 - exp(-1/60)) << 13 = 135,
           * (1 - exp(-1/300)) << 13 = 27,
           * (1 - exp(-1/900)) << 13 = 9.
           */
  
          /*
           * a little hoop-jumping to avoid integer overflow
           */
          for (i = 0; i < 3; i++) {
                  q = (hp_ave[i]  >> 16) << 7;
                  r = (hp_ave[i]  & 0xffff) << 7;
                  hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
          }
  }
  
  /*
   * lbolt_hybrid() is used by ddi_get_lbolt() and ddi_get_lbolt64() to
   * calculate the value of lbolt according to the current mode. In the event
   * driven mode (the default), lbolt is calculated by dividing the current hires
   * time by the number of nanoseconds per clock tick. In the cyclic driven mode
   * an internal variable is incremented at each firing of the lbolt cyclic
   * and returned by lbolt_cyclic_driven().
   *
   * The system will transition from event to cyclic driven mode when the number
   * of calls to lbolt_event_driven() exceeds the (per CPU) threshold within a
   * window of time. It does so by reprograming lbolt_cyclic from CY_INFINITY to
   * nsec_per_tick. The lbolt cyclic will remain ON while at least one CPU is
   * causing enough activity to cross the thresholds.
   */
  int64_t
  lbolt_bootstrap(void)
  {
          return (0);
  }
  
  /* ARGSUSED */
  uint_t
  lbolt_ev_to_cyclic(caddr_t arg1, caddr_t arg2)
  {
          hrtime_t ts, exp;
          int ret;
  
          ASSERT(lbolt_hybrid != lbolt_cyclic_driven);
  
          kpreempt_disable();
  
          ts = gethrtime();
          lb_info->lbi_internal = (ts/nsec_per_tick);
  
          /*
           * Align the next expiration to a clock tick boundary.
           */
          exp = ts + nsec_per_tick - 1;
          exp = (exp/nsec_per_tick) * nsec_per_tick;
  
          ret = cyclic_reprogram(lb_info->id.lbi_cyclic_id, exp);
          ASSERT(ret);
  
          lbolt_hybrid = lbolt_cyclic_driven;
          lb_info->lbi_cyc_deactivate = B_FALSE;
          lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
  
          kpreempt_enable();
  
          ret = atomic_dec_32_nv(&lb_info->lbi_token);
          ASSERT(ret == 0);
  
          return (1);
  }
  
  int64_t
  lbolt_event_driven(void)
  {
          hrtime_t ts;
          int64_t lb;
          int ret, cpu = CPU->cpu_seqid;
  
          ts = gethrtime();
          ASSERT(ts > 0);
  
          ASSERT(nsec_per_tick > 0);
          lb = (ts/nsec_per_tick);
  
          /*
           * Switch to cyclic mode if the number of calls to this routine
           * has reached the threshold within the interval.
           */
          if ((lb - lb_cpu[cpu].lbc_cnt_start) < lb_info->lbi_thresh_interval) {
  
                  if (--lb_cpu[cpu].lbc_counter == 0) {
                          /*
                           * Reached the threshold within the interval, reset
                           * the usage statistics.
                           */
                          lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
                          lb_cpu[cpu].lbc_cnt_start = lb;
  
                          /*
                           * Make sure only one thread reprograms the
                           * lbolt cyclic and changes the mode.
                           */
                          if (panicstr == NULL &&
                              atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
  
                                  if (lbolt_hybrid == lbolt_cyclic_driven) {
                                          ret = atomic_dec_32_nv(
                                              &lb_info->lbi_token);
                                          ASSERT(ret == 0);
                                  } else {
                                          lbolt_softint_post();
                                  }
                          }
                  }
          } else {
                  /*
                   * Exceeded the interval, reset the usage statistics.
                   */
                  lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
                  lb_cpu[cpu].lbc_cnt_start = lb;
          }
  
          ASSERT(lb >= lb_info->lbi_debug_time);
  
          return (lb - lb_info->lbi_debug_time);
  }
  
  int64_t
  lbolt_cyclic_driven(void)
  {
          int64_t lb = lb_info->lbi_internal;
          int cpu;
  
          /*
           * If a CPU has already prevented the lbolt cyclic from deactivating
           * itself, don't bother tracking the usage. Otherwise check if we're
           * within the interval and how the per CPU counter is doing.
           */
          if (lb_info->lbi_cyc_deactivate) {
                  cpu = CPU->cpu_seqid;
                  if ((lb - lb_cpu[cpu].lbc_cnt_start) <
                      lb_info->lbi_thresh_interval) {
  
                          if (lb_cpu[cpu].lbc_counter == 0)
                                  /*
                                   * Reached the threshold within the interval,
                                   * prevent the lbolt cyclic from turning itself
                                   * off.
                                   */
                                  lb_info->lbi_cyc_deactivate = B_FALSE;
                          else
                                  lb_cpu[cpu].lbc_counter--;
                  } else {
                          /*
                           * Only reset the usage statistics when we have
                           * exceeded the interval.
                           */
                          lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
                          lb_cpu[cpu].lbc_cnt_start = lb;
                  }
          }
  
          ASSERT(lb >= lb_info->lbi_debug_time);
  
          return (lb - lb_info->lbi_debug_time);
  }
  
  /*
   * The lbolt_cyclic() routine will fire at a nsec_per_tick interval to satisfy
   * performance needs of ddi_get_lbolt() and ddi_get_lbolt64() consumers.
   * It is inactive by default, and will be activated when switching from event
   * to cyclic driven lbolt. The cyclic will turn itself off unless signaled
   * by lbolt_cyclic_driven().
   */
  static void
  lbolt_cyclic(void)
  {
          int ret;
  
          lb_info->lbi_internal++;
  
          if (!lbolt_cyc_only) {
  
                  if (lb_info->lbi_cyc_deactivate) {
                          /*
                           * Switching from cyclic to event driven mode.
                           */
                          if (panicstr == NULL &&
                              atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
  
                                  if (lbolt_hybrid == lbolt_event_driven) {
                                          ret = atomic_dec_32_nv(
                                              &lb_info->lbi_token);
                                          ASSERT(ret == 0);
                                          return;
                                  }
  
                                  kpreempt_disable();
  
                                  lbolt_hybrid = lbolt_event_driven;
                                  ret = cyclic_reprogram(
                                      lb_info->id.lbi_cyclic_id,
                                      CY_INFINITY);
                                  ASSERT(ret);
  
                                  kpreempt_enable();
  
                                  ret = atomic_dec_32_nv(&lb_info->lbi_token);
                                  ASSERT(ret == 0);
                          }
                  }
  
                  /*
                   * The lbolt cyclic should not try to deactivate itself before
                   * the sampling period has elapsed.
                   */
                  if (lb_info->lbi_internal - lb_info->lbi_cyc_deac_start >=
                      lb_info->lbi_thresh_interval) {
                          lb_info->lbi_cyc_deactivate = B_TRUE;
                          lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
                  }
          }
  }
  
  /*
   * Since the lbolt service was historically cyclic driven, it must be 'stopped'
   * when the system drops into the kernel debugger. lbolt_debug_entry() is
   * called by the KDI system claim callbacks to record a hires timestamp at
   * debug enter time. lbolt_debug_return() is called by the sistem release
   * callbacks to account for the time spent in the debugger. The value is then
   * accumulated in the lb_info structure and used by lbolt_event_driven() and
   * lbolt_cyclic_driven(), as well as the mdb_get_lbolt() routine.
   */
  void
  lbolt_debug_entry(void)
  {
          if (lbolt_hybrid != lbolt_bootstrap) {
                  ASSERT(lb_info != NULL);
                  lb_info->lbi_debug_ts = gethrtime();
          }
  }
  
  /*
   * Calculate the time spent in the debugger and add it to the lbolt info
   * structure. We also update the internal lbolt value in case we were in
   * cyclic driven mode going in.
   */
  void
  lbolt_debug_return(void)
  {
          hrtime_t ts;
  
          if (lbolt_hybrid != lbolt_bootstrap) {
                  ASSERT(lb_info != NULL);
                  ASSERT(nsec_per_tick > 0);
  
                  ts = gethrtime();
                  lb_info->lbi_internal = (ts/nsec_per_tick);
                  lb_info->lbi_debug_time +=
                      ((ts - lb_info->lbi_debug_ts)/nsec_per_tick);
  
                  lb_info->lbi_debug_ts = 0;
          }
  }

Cache object: efc3b20281ac0d7895c2ec6f2970bdb5