FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_cctr.c
1 /* $NetBSD: kern_cctr.c,v 1.12 2020/10/10 18:18:04 thorpej Exp $ */
2
3 /*-
4 * Copyright (c) 2020 Jason R. Thorpe
5 * Copyright (c) 2018 Naruaki Etomi
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * Most of the following was adapted from the Linux/ia64 cycle counter
31 * synchronization algorithm:
32 *
33 * IA-64 Linux Kernel: Design and Implementation p356-p361
34 * (Hewlett-Packard Professional Books)
35 *
36 * Here's a rough description of how it works.
37 *
38 * The primary CPU is the reference monotonic counter. Each secondary
39 * CPU is responsible for knowing the offset of its own cycle counter
40 * relative to the primary's. When the time counter is read, the CC
41 * value is adjusted by this delta.
42 *
43 * Calibration happens periodically, and works like this:
44 *
45 * Secondary CPU Primary CPU
46 * Send IPI to publish reference CC
47 * --------->
48 * Indicate Primary Ready
49 * <----------------------------
50 * T0 = local CC
51 * Indicate Secondary Ready
52 * ----------------->
53 * (assume this happens at Tavg) Publish reference CC
54 * Indicate completion
55 * <------------------------
56 * Notice completion
57 * T1 = local CC
58 *
59 * Tavg = (T0 + T1) / 2
60 *
61 * Delta = Tavg - Published primary CC value
62 *
63 * "Notice completion" is performed by waiting for the primary to set
64 * the calibration state to FINISHED. This is a little unfortunate,
65 * because T0->Tavg involves a single store-release on the secondary, and
66 * Tavg->T1 involves a store-relaxed and a store-release. It would be
67 * better to simply wait for the reference CC to transition from 0 to
68 * non-0 (i.e. just wait for a single store-release from Tavg->T1), but
69 * if the cycle counter just happened to read back as 0 at that instant,
70 * we would never break out of the loop.
71 *
72 * We trigger calibration roughly once a second; the period is actually
73 * skewed based on the CPU index in order to avoid lock contention. The
74 * calibration interval does not need to be precise, and so this is fine.
75 */
76
77 #include <sys/cdefs.h>
78 __KERNEL_RCSID(0, "$NetBSD: kern_cctr.c,v 1.12 2020/10/10 18:18:04 thorpej Exp $");
79
80 #include <sys/param.h>
81 #include <sys/atomic.h>
82 #include <sys/systm.h>
83 #include <sys/sysctl.h>
84 #include <sys/timepps.h>
85 #include <sys/time.h>
86 #include <sys/timetc.h>
87 #include <sys/kernel.h>
88 #include <sys/power.h>
89 #include <sys/cpu.h>
90 #include <machine/cpu_counter.h>
91
92 /* XXX make cc_timecounter.tc_frequency settable by sysctl() */
93
94 #if defined(MULTIPROCESSOR)
95 static uint32_t cc_primary __cacheline_aligned;
96 static uint32_t cc_calibration_state __cacheline_aligned;
97 static kmutex_t cc_calibration_lock __cacheline_aligned;
98
99 #define CC_CAL_START 0 /* initial state */
100 #define CC_CAL_PRIMARY_READY 1 /* primary CPU ready to respond */
101 #define CC_CAL_SECONDARY_READY 2 /* secondary CPU ready to receive */
102 #define CC_CAL_FINISHED 3 /* calibration attempt complete */
103 #endif /* MULTIPROCESSOR */
104
105 static struct timecounter cc_timecounter = {
106 .tc_get_timecount = cc_get_timecount,
107 .tc_poll_pps = NULL,
108 .tc_counter_mask = ~0u,
109 .tc_frequency = 0,
110 .tc_name = "unknown cycle counter",
111 /*
112 * don't pick cycle counter automatically
113 * if frequency changes might affect cycle counter
114 */
115 .tc_quality = -100000,
116
117 .tc_priv = NULL,
118 .tc_next = NULL
119 };
120
121 /*
122 * Initialize cycle counter based timecounter. This must be done on the
123 * primary CPU.
124 */
125 struct timecounter *
126 cc_init(timecounter_get_t getcc, uint64_t freq, const char *name, int quality)
127 {
128 static bool cc_init_done __diagused;
129 struct cpu_info * const ci = curcpu();
130
131 KASSERT(!cc_init_done);
132 KASSERT(cold);
133 KASSERT(CPU_IS_PRIMARY(ci));
134
135 #if defined(MULTIPROCESSOR)
136 mutex_init(&cc_calibration_lock, MUTEX_DEFAULT, IPL_HIGH);
137 #endif
138
139 cc_init_done = true;
140
141 ci->ci_cc.cc_delta = 0;
142 ci->ci_cc.cc_ticks = 0;
143 ci->ci_cc.cc_cal_ticks = 0;
144
145 if (getcc != NULL)
146 cc_timecounter.tc_get_timecount = getcc;
147
148 cc_timecounter.tc_frequency = freq;
149 cc_timecounter.tc_name = name;
150 cc_timecounter.tc_quality = quality;
151 tc_init(&cc_timecounter);
152
153 return &cc_timecounter;
154 }
155
156 /*
157 * Initialize cycle counter timecounter calibration data on a secondary
158 * CPU. Must be called on that secondary CPU.
159 */
160 void
161 cc_init_secondary(struct cpu_info * const ci)
162 {
163 KASSERT(!CPU_IS_PRIMARY(curcpu()));
164 KASSERT(ci == curcpu());
165
166 ci->ci_cc.cc_ticks = 0;
167
168 /*
169 * It's not critical that calibration be performed in
170 * precise intervals, so skew when calibration is done
171 * on each secondary CPU based on it's CPU index to
172 * avoid contending on the calibration lock.
173 */
174 ci->ci_cc.cc_cal_ticks = hz - cpu_index(ci);
175 KASSERT(ci->ci_cc.cc_cal_ticks);
176
177 cc_calibrate_cpu(ci);
178 }
179
180 /*
181 * pick up tick count scaled to reference tick count
182 */
183 u_int
184 cc_get_timecount(struct timecounter *tc)
185 {
186 #if defined(MULTIPROCESSOR)
187 int64_t rcc, ncsw;
188
189 retry:
190 ncsw = curlwp->l_ncsw;
191
192 __insn_barrier();
193 /* N.B. the delta is always 0 on the primary. */
194 rcc = cpu_counter32() - curcpu()->ci_cc.cc_delta;
195 __insn_barrier();
196
197 if (ncsw != curlwp->l_ncsw) {
198 /* Was preempted */
199 goto retry;
200 }
201
202 return rcc;
203 #else
204 return cpu_counter32();
205 #endif /* MULTIPROCESSOR */
206 }
207
208 #if defined(MULTIPROCESSOR)
209 static inline bool
210 cc_get_delta(struct cpu_info * const ci)
211 {
212 int64_t t0, t1, tcenter = 0;
213
214 t0 = cpu_counter32();
215
216 atomic_store_release(&cc_calibration_state, CC_CAL_SECONDARY_READY);
217
218 for (;;) {
219 if (atomic_load_acquire(&cc_calibration_state) ==
220 CC_CAL_FINISHED) {
221 break;
222 }
223 }
224
225 t1 = cpu_counter32();
226
227 if (t1 < t0) {
228 /* Overflow! */
229 return false;
230 }
231
232 /* average t0 and t1 without overflow: */
233 tcenter = (t0 >> 1) + (t1 >> 1);
234 if ((t0 & 1) + (t1 & 1) == 2)
235 tcenter++;
236
237 ci->ci_cc.cc_delta = tcenter - cc_primary;
238
239 return true;
240 }
241 #endif /* MULTIPROCESSOR */
242
243 /*
244 * Called on secondary CPUs to calibrate their cycle counter offset
245 * relative to the primary CPU.
246 */
247 void
248 cc_calibrate_cpu(struct cpu_info * const ci)
249 {
250 #if defined(MULTIPROCESSOR)
251 KASSERT(!CPU_IS_PRIMARY(ci));
252
253 mutex_spin_enter(&cc_calibration_lock);
254
255 retry:
256 atomic_store_release(&cc_calibration_state, CC_CAL_START);
257
258 /* Trigger primary CPU. */
259 cc_get_primary_cc();
260
261 for (;;) {
262 if (atomic_load_acquire(&cc_calibration_state) ==
263 CC_CAL_PRIMARY_READY) {
264 break;
265 }
266 }
267
268 if (! cc_get_delta(ci)) {
269 goto retry;
270 }
271
272 mutex_exit(&cc_calibration_lock);
273 #endif /* MULTIPROCESSOR */
274 }
275
276 void
277 cc_primary_cc(void)
278 {
279 #if defined(MULTIPROCESSOR)
280 /* N.B. We expect all interrupts to be blocked. */
281
282 atomic_store_release(&cc_calibration_state, CC_CAL_PRIMARY_READY);
283
284 for (;;) {
285 if (atomic_load_acquire(&cc_calibration_state) ==
286 CC_CAL_SECONDARY_READY) {
287 break;
288 }
289 }
290
291 cc_primary = cpu_counter32();
292 atomic_store_release(&cc_calibration_state, CC_CAL_FINISHED);
293 #endif /* MULTIPROCESSOR */
294 }
Cache object: f8fab35716139c21c6c70c5918670312
|