1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
5 * Copyright (c) 2021 Dmitry Chagin <dchagin@FreeBSD.org>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 static int
30 fls(int mask)
31 {
32
33 if (mask == 0)
34 return (0);
35 return ((__builtin_clz(mask) ^ 0x1f) + 1);
36 }
37
38 #ifdef _LP64
39 static int
40 flsl(long mask)
41 {
42 int bit;
43
44 if (mask == 0)
45 return (0);
46 for (bit = 1; mask != 1; bit++)
47 mask = (unsigned long)mask >> 1;
48 return (bit);
49 }
50 #else
51 static int
52 flsll(long long mask)
53 {
54 int bit;
55
56 if (mask == 0)
57 return (0);
58 for (bit = 1; mask != 1; bit++)
59 mask = (unsigned long long)mask >> 1;
60 return (bit);
61 }
62 #endif
63
64 static int
65 __vdso_native_to_linux_timespec(struct l_timespec *lts,
66 struct timespec *nts)
67 {
68
69 #ifdef COMPAT_LINUX32
70 if (nts->tv_sec > INT_MAX || nts->tv_sec < INT_MIN)
71 return (LINUX_EOVERFLOW);
72 #endif
73 lts->tv_sec = nts->tv_sec;
74 lts->tv_nsec = nts->tv_nsec;
75 return (0);
76 }
77
78 static int
79 __vdso_native_to_linux_timeval(l_timeval *ltv,
80 struct timeval *ntv)
81 {
82
83 #ifdef COMPAT_LINUX32
84 if (ntv->tv_sec > INT_MAX || ntv->tv_sec < INT_MIN)
85 return (LINUX_EOVERFLOW);
86 #endif
87 ltv->tv_sec = ntv->tv_sec;
88 ltv->tv_usec = ntv->tv_usec;
89 return (0);
90 }
91
92
93 #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32))
94 static int
95 __vdso_native_to_linux_timespec64(struct l_timespec64 *lts,
96 struct timespec *nts)
97 {
98
99 lts->tv_sec = nts->tv_sec;
100 lts->tv_nsec = nts->tv_nsec;
101 return (0);
102 }
103 #endif
104
105 static int
106 __vdso_linux_to_native_clockid(clockid_t *n, clockid_t l)
107 {
108
109 switch (l) {
110 case LINUX_CLOCK_REALTIME:
111 *n = CLOCK_REALTIME;
112 break;
113 case LINUX_CLOCK_MONOTONIC:
114 *n = CLOCK_MONOTONIC;
115 break;
116 case LINUX_CLOCK_REALTIME_COARSE:
117 *n = CLOCK_REALTIME_FAST;
118 break;
119 case LINUX_CLOCK_MONOTONIC_COARSE:
120 case LINUX_CLOCK_MONOTONIC_RAW:
121 *n = CLOCK_MONOTONIC_FAST;
122 break;
123 case LINUX_CLOCK_BOOTTIME:
124 *n = CLOCK_UPTIME;
125 break;
126 default:
127 return (LINUX_EINVAL);
128 }
129 return (0);
130 }
131
132 /*
133 * The code below adapted from
134 * lib/libc/sys/__vdso_gettimeofday.c
135 */
136
137 static inline void
138 __vdso_gettimekeep(struct vdso_timekeep **tk)
139 {
140
141 *tk = (struct vdso_timekeep *)kern_timekeep_base;
142 }
143
144 static int
145 tc_delta(const struct vdso_timehands *th, u_int *delta)
146 {
147 int error;
148 u_int tc;
149
150 error = __vdso_gettc(th, &tc);
151 if (error == 0)
152 *delta = (tc - th->th_offset_count) & th->th_counter_mask;
153 return (error);
154 }
155
156 /*
157 * Calculate the absolute or boot-relative time from the
158 * machine-specific fast timecounter and the published timehands
159 * structure read from the shared page.
160 *
161 * The lockless reading scheme is similar to the one used to read the
162 * in-kernel timehands, see sys/kern/kern_tc.c:binuptime(). This code
163 * is based on the kernel implementation.
164 */
165 static int
166 freebsd_binuptime(struct bintime *bt, struct vdso_timekeep *tk, bool abs)
167 {
168 struct vdso_timehands *th;
169 uint32_t curr, gen;
170 uint64_t scale, x;
171 u_int delta, scale_bits;
172 int error;
173
174 do {
175 if (!tk->tk_enabled)
176 return (ENOSYS);
177
178 curr = atomic_load_acq_32(&tk->tk_current);
179 th = &tk->tk_th[curr];
180 gen = atomic_load_acq_32(&th->th_gen);
181 *bt = th->th_offset;
182 error = tc_delta(th, &delta);
183 if (error == EAGAIN)
184 continue;
185 if (error != 0)
186 return (error);
187 scale = th->th_scale;
188 #ifdef _LP64
189 scale_bits = flsl(scale);
190 #else
191 scale_bits = flsll(scale);
192 #endif
193 if (__predict_false(scale_bits + fls(delta) > 63)) {
194 x = (scale >> 32) * delta;
195 scale &= 0xffffffff;
196 bt->sec += x >> 32;
197 bintime_addx(bt, x << 32);
198 }
199 bintime_addx(bt, scale * delta);
200 if (abs)
201 bintime_add(bt, &th->th_boottime);
202
203 /*
204 * Ensure that the load of th_offset is completed
205 * before the load of th_gen.
206 */
207 atomic_thread_fence_acq();
208 } while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
209 return (0);
210 }
211
212 static int
213 freebsd_getnanouptime(struct bintime *bt, struct vdso_timekeep *tk)
214 {
215 struct vdso_timehands *th;
216 uint32_t curr, gen;
217
218 do {
219 if (!tk->tk_enabled)
220 return (ENOSYS);
221
222 curr = atomic_load_acq_32(&tk->tk_current);
223 th = &tk->tk_th[curr];
224 gen = atomic_load_acq_32(&th->th_gen);
225 *bt = th->th_offset;
226
227 /*
228 * Ensure that the load of th_offset is completed
229 * before the load of th_gen.
230 */
231 atomic_thread_fence_acq();
232 } while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
233 return (0);
234 }
235
236 static int
237 freebsd_gettimeofday(struct timeval *tv, struct timezone *tz)
238 {
239 struct vdso_timekeep *tk;
240 struct bintime bt;
241 int error;
242
243 if (tz != NULL)
244 return (ENOSYS);
245 __vdso_gettimekeep(&tk);
246 if (tk == NULL)
247 return (ENOSYS);
248 if (tk->tk_ver != VDSO_TK_VER_CURR)
249 return (ENOSYS);
250 error = freebsd_binuptime(&bt, tk, true);
251 if (error == 0)
252 bintime2timeval(&bt, tv);
253 return (error);
254 }
255
256 static int
257 freebsd_clock_gettime(clockid_t clock_id, struct timespec *ts)
258 {
259 struct vdso_timekeep *tk;
260 struct bintime bt;
261 int error;
262
263 __vdso_gettimekeep(&tk);
264 if (tk == NULL)
265 return (ENOSYS);
266 if (tk->tk_ver != VDSO_TK_VER_CURR)
267 return (ENOSYS);
268 switch (clock_id) {
269 case CLOCK_REALTIME:
270 case CLOCK_REALTIME_PRECISE:
271 case CLOCK_REALTIME_FAST:
272 error = freebsd_binuptime(&bt, tk, true);
273 break;
274 case CLOCK_MONOTONIC:
275 case CLOCK_MONOTONIC_PRECISE:
276 case CLOCK_UPTIME:
277 case CLOCK_UPTIME_PRECISE:
278 error = freebsd_binuptime(&bt, tk, false);
279 break;
280 case CLOCK_MONOTONIC_FAST:
281 case CLOCK_UPTIME_FAST:
282 error = freebsd_getnanouptime(&bt, tk);
283 break;
284 default:
285 error = ENOSYS;
286 break;
287 }
288 if (error == 0)
289 bintime2timespec(&bt, ts);
290 return (error);
291 }
292
293 /*
294 * Linux vDSO interfaces
295 *
296 */
297 int
298 __vdso_clock_gettime(clockid_t clock_id, struct l_timespec *lts)
299 {
300 struct timespec ts;
301 clockid_t which;
302 int error;
303
304 error = __vdso_linux_to_native_clockid(&which, clock_id);
305 if (error != 0)
306 return (__vdso_clock_gettime_fallback(clock_id, lts));
307 error = freebsd_clock_gettime(which, &ts);
308 if (error == 0)
309 return (-__vdso_native_to_linux_timespec(lts, &ts));
310 else
311 return (__vdso_clock_gettime_fallback(clock_id, lts));
312 }
313
314 int
315 __vdso_gettimeofday(l_timeval *ltv, struct timezone *tz)
316 {
317 struct timeval tv;
318 int error;
319
320 error = freebsd_gettimeofday(&tv, tz);
321 if (error != 0)
322 return (__vdso_gettimeofday_fallback(ltv, tz));
323 return (-__vdso_native_to_linux_timeval(ltv, &tv));
324 }
325
326 int
327 __vdso_clock_getres(clockid_t clock_id, struct l_timespec *lts)
328 {
329
330 return (__vdso_clock_getres_fallback(clock_id, lts));
331 }
332
333 #if defined(__i386__) || defined(COMPAT_LINUX32)
334 int
335 __vdso_clock_gettime64(clockid_t clock_id, struct l_timespec64 *lts)
336 {
337 struct timespec ts;
338 clockid_t which;
339 int error;
340
341 error = __vdso_linux_to_native_clockid(&which, clock_id);
342 if (error != 0)
343 return (__vdso_clock_gettime64_fallback(clock_id, lts));
344 error = freebsd_clock_gettime(which, &ts);
345 if (error == 0)
346 return(-__vdso_native_to_linux_timespec64(lts, &ts));
347 else
348 return(__vdso_clock_gettime64_fallback(clock_id, lts));
349 }
350
351 int clock_gettime64(clockid_t clock_id, struct l_timespec64 *lts)
352 __attribute__((weak, alias("__vdso_clock_gettime64")));
353 #endif
354
355 #if defined(__i386__) || defined(__amd64__)
356 int
357 __vdso_getcpu(uint32_t *cpu, uint32_t *node, void *cache)
358 {
359 int ret;
360
361 if (node != NULL)
362 return (__vdso_getcpu_fallback(cpu, node, cache));
363 ret = __vdso_getcpu_try();
364 if (ret < 0)
365 return (__vdso_getcpu_fallback(cpu, node, cache));
366 *cpu = ret;
367 return (0);
368 }
369 #endif
370
371 #if defined(__i386__) || defined(__amd64__)
372 int
373 __vdso_time(long *tm)
374 {
375 struct timeval tv;
376 int error;
377
378 error = freebsd_gettimeofday(&tv, NULL);
379 if (error != 0)
380 return (__vdso_time_fallback(tm));
381 if (tm != NULL)
382 *tm = tv.tv_sec;
383 return (tv.tv_sec);
384 }
385 #endif
Cache object: e8059ba02962bd3720d92fa26feaf275
|