1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Generic userspace implementations of gettimeofday() and similar.
4 */
5 #include <vdso/datapage.h>
6 #include <vdso/helpers.h>
7
8 #ifndef vdso_calc_delta
9 /*
10 * Default implementation which works for all sane clocksources. That
11 * obviously excludes x86/TSC.
12 */
13 static __always_inline
vdso_calc_delta(u64 cycles,u64 last,u64 mask,u32 mult)14 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
15 {
16 return ((cycles - last) & mask) * mult;
17 }
18 #endif
19
20 #ifndef vdso_shift_ns
vdso_shift_ns(u64 ns,u32 shift)21 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
22 {
23 return ns >> shift;
24 }
25 #endif
26
27 #ifndef __arch_vdso_hres_capable
__arch_vdso_hres_capable(void)28 static inline bool __arch_vdso_hres_capable(void)
29 {
30 return true;
31 }
32 #endif
33
34 #ifndef vdso_clocksource_ok
vdso_clocksource_ok(const struct vdso_data * vd)35 static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
36 {
37 return vd->clock_mode != VDSO_CLOCKMODE_NONE;
38 }
39 #endif
40
41 #ifndef vdso_cycles_ok
vdso_cycles_ok(u64 cycles)42 static inline bool vdso_cycles_ok(u64 cycles)
43 {
44 return true;
45 }
46 #endif
47
48 #ifdef CONFIG_TIME_NS
do_hres_timens(const struct vdso_data * vdns,clockid_t clk,struct __kernel_timespec * ts)49 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
50 struct __kernel_timespec *ts)
51 {
52 const struct vdso_data *vd = __arch_get_timens_vdso_data();
53 const struct timens_offset *offs = &vdns->offset[clk];
54 const struct vdso_timestamp *vdso_ts;
55 u64 cycles, last, ns;
56 u32 seq;
57 s64 sec;
58
59 if (clk != CLOCK_MONOTONIC_RAW)
60 vd = &vd[CS_HRES_COARSE];
61 else
62 vd = &vd[CS_RAW];
63 vdso_ts = &vd->basetime[clk];
64
65 do {
66 seq = vdso_read_begin(vd);
67
68 if (unlikely(!vdso_clocksource_ok(vd)))
69 return -1;
70
71 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72 if (unlikely(!vdso_cycles_ok(cycles)))
73 return -1;
74 ns = vdso_ts->nsec;
75 last = vd->cycle_last;
76 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
77 ns = vdso_shift_ns(ns, vd->shift);
78 sec = vdso_ts->sec;
79 } while (unlikely(vdso_read_retry(vd, seq)));
80
81 /* Add the namespace offset */
82 sec += offs->sec;
83 ns += offs->nsec;
84
85 /*
86 * Do this outside the loop: a race inside the loop could result
87 * in __iter_div_u64_rem() being extremely slow.
88 */
89 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
90 ts->tv_nsec = ns;
91
92 return 0;
93 }
94 #else
__arch_get_timens_vdso_data(void)95 static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
96 {
97 return NULL;
98 }
99
do_hres_timens(const struct vdso_data * vdns,clockid_t clk,struct __kernel_timespec * ts)100 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
101 struct __kernel_timespec *ts)
102 {
103 return -EINVAL;
104 }
105 #endif
106
do_hres(const struct vdso_data * vd,clockid_t clk,struct __kernel_timespec * ts)107 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
108 struct __kernel_timespec *ts)
109 {
110 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
111 u64 cycles, last, sec, ns;
112 u32 seq;
113
114 /* Allows to compile the high resolution parts out */
115 if (!__arch_vdso_hres_capable())
116 return -1;
117
118 do {
119 /*
120 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
121 * enabled tasks have a special VVAR page installed which
122 * has vd->seq set to 1 and vd->clock_mode set to
123 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
124 * this does not affect performance because if vd->seq is
125 * odd, i.e. a concurrent update is in progress the extra
126 * check for vd->clock_mode is just a few extra
127 * instructions while spin waiting for vd->seq to become
128 * even again.
129 */
130 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
131 if (IS_ENABLED(CONFIG_TIME_NS) &&
132 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
133 return do_hres_timens(vd, clk, ts);
134 cpu_relax();
135 }
136 smp_rmb();
137
138 if (unlikely(!vdso_clocksource_ok(vd)))
139 return -1;
140
141 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
142 if (unlikely(!vdso_cycles_ok(cycles)))
143 return -1;
144 ns = vdso_ts->nsec;
145 last = vd->cycle_last;
146 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
147 ns = vdso_shift_ns(ns, vd->shift);
148 sec = vdso_ts->sec;
149 } while (unlikely(vdso_read_retry(vd, seq)));
150
151 /*
152 * Do this outside the loop: a race inside the loop could result
153 * in __iter_div_u64_rem() being extremely slow.
154 */
155 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
156 ts->tv_nsec = ns;
157
158 return 0;
159 }
160
161 #ifdef CONFIG_TIME_NS
do_coarse_timens(const struct vdso_data * vdns,clockid_t clk,struct __kernel_timespec * ts)162 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
163 struct __kernel_timespec *ts)
164 {
165 const struct vdso_data *vd = __arch_get_timens_vdso_data();
166 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
167 const struct timens_offset *offs = &vdns->offset[clk];
168 u64 nsec;
169 s64 sec;
170 s32 seq;
171
172 do {
173 seq = vdso_read_begin(vd);
174 sec = vdso_ts->sec;
175 nsec = vdso_ts->nsec;
176 } while (unlikely(vdso_read_retry(vd, seq)));
177
178 /* Add the namespace offset */
179 sec += offs->sec;
180 nsec += offs->nsec;
181
182 /*
183 * Do this outside the loop: a race inside the loop could result
184 * in __iter_div_u64_rem() being extremely slow.
185 */
186 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
187 ts->tv_nsec = nsec;
188 return 0;
189 }
190 #else
do_coarse_timens(const struct vdso_data * vdns,clockid_t clk,struct __kernel_timespec * ts)191 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
192 struct __kernel_timespec *ts)
193 {
194 return -1;
195 }
196 #endif
197
do_coarse(const struct vdso_data * vd,clockid_t clk,struct __kernel_timespec * ts)198 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
199 struct __kernel_timespec *ts)
200 {
201 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
202 u32 seq;
203
204 do {
205 /*
206 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in
207 * do_hres().
208 */
209 while ((seq = READ_ONCE(vd->seq)) & 1) {
210 if (IS_ENABLED(CONFIG_TIME_NS) &&
211 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
212 return do_coarse_timens(vd, clk, ts);
213 cpu_relax();
214 }
215 smp_rmb();
216
217 ts->tv_sec = vdso_ts->sec;
218 ts->tv_nsec = vdso_ts->nsec;
219 } while (unlikely(vdso_read_retry(vd, seq)));
220
221 return 0;
222 }
223
224 static __always_inline int
__cvdso_clock_gettime_common(const struct vdso_data * vd,clockid_t clock,struct __kernel_timespec * ts)225 __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
226 struct __kernel_timespec *ts)
227 {
228 u32 msk;
229
230 /* Check for negative values or invalid clocks */
231 if (unlikely((u32) clock >= MAX_CLOCKS))
232 return -1;
233
234 /*
235 * Convert the clockid to a bitmask and use it to check which
236 * clocks are handled in the VDSO directly.
237 */
238 msk = 1U << clock;
239 if (likely(msk & VDSO_HRES))
240 vd = &vd[CS_HRES_COARSE];
241 else if (msk & VDSO_COARSE)
242 return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
243 else if (msk & VDSO_RAW)
244 vd = &vd[CS_RAW];
245 else
246 return -1;
247
248 return do_hres(vd, clock, ts);
249 }
250
251 static __maybe_unused int
__cvdso_clock_gettime_data(const struct vdso_data * vd,clockid_t clock,struct __kernel_timespec * ts)252 __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
253 struct __kernel_timespec *ts)
254 {
255 int ret = __cvdso_clock_gettime_common(vd, clock, ts);
256
257 if (unlikely(ret))
258 return clock_gettime_fallback(clock, ts);
259 return 0;
260 }
261
262 static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock,struct __kernel_timespec * ts)263 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
264 {
265 return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
266 }
267
268 #ifdef BUILD_VDSO32
269 static __maybe_unused int
__cvdso_clock_gettime32_data(const struct vdso_data * vd,clockid_t clock,struct old_timespec32 * res)270 __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
271 struct old_timespec32 *res)
272 {
273 struct __kernel_timespec ts;
274 int ret;
275
276 ret = __cvdso_clock_gettime_common(vd, clock, &ts);
277
278 if (unlikely(ret))
279 return clock_gettime32_fallback(clock, res);
280
281 /* For ret == 0 */
282 res->tv_sec = ts.tv_sec;
283 res->tv_nsec = ts.tv_nsec;
284
285 return ret;
286 }
287
288 static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock,struct old_timespec32 * res)289 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
290 {
291 return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
292 }
293 #endif /* BUILD_VDSO32 */
294
295 static __maybe_unused int
__cvdso_gettimeofday_data(const struct vdso_data * vd,struct __kernel_old_timeval * tv,struct timezone * tz)296 __cvdso_gettimeofday_data(const struct vdso_data *vd,
297 struct __kernel_old_timeval *tv, struct timezone *tz)
298 {
299
300 if (likely(tv != NULL)) {
301 struct __kernel_timespec ts;
302
303 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
304 return gettimeofday_fallback(tv, tz);
305
306 tv->tv_sec = ts.tv_sec;
307 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
308 }
309
310 if (unlikely(tz != NULL)) {
311 if (IS_ENABLED(CONFIG_TIME_NS) &&
312 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
313 vd = __arch_get_timens_vdso_data();
314
315 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
316 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
317 }
318
319 return 0;
320 }
321
322 static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval * tv,struct timezone * tz)323 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
324 {
325 return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
326 }
327
328 #ifdef VDSO_HAS_TIME
329 static __maybe_unused __kernel_old_time_t
__cvdso_time_data(const struct vdso_data * vd,__kernel_old_time_t * time)330 __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
331 {
332 __kernel_old_time_t t;
333
334 if (IS_ENABLED(CONFIG_TIME_NS) &&
335 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
336 vd = __arch_get_timens_vdso_data();
337
338 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
339
340 if (time)
341 *time = t;
342
343 return t;
344 }
345
__cvdso_time(__kernel_old_time_t * time)346 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
347 {
348 return __cvdso_time_data(__arch_get_vdso_data(), time);
349 }
350 #endif /* VDSO_HAS_TIME */
351
352 #ifdef VDSO_HAS_CLOCK_GETRES
353 static __maybe_unused
__cvdso_clock_getres_common(const struct vdso_data * vd,clockid_t clock,struct __kernel_timespec * res)354 int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
355 struct __kernel_timespec *res)
356 {
357 u32 msk;
358 u64 ns;
359
360 /* Check for negative values or invalid clocks */
361 if (unlikely((u32) clock >= MAX_CLOCKS))
362 return -1;
363
364 if (IS_ENABLED(CONFIG_TIME_NS) &&
365 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
366 vd = __arch_get_timens_vdso_data();
367
368 /*
369 * Convert the clockid to a bitmask and use it to check which
370 * clocks are handled in the VDSO directly.
371 */
372 msk = 1U << clock;
373 if (msk & (VDSO_HRES | VDSO_RAW)) {
374 /*
375 * Preserves the behaviour of posix_get_hrtimer_res().
376 */
377 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
378 } else if (msk & VDSO_COARSE) {
379 /*
380 * Preserves the behaviour of posix_get_coarse_res().
381 */
382 ns = LOW_RES_NSEC;
383 } else {
384 return -1;
385 }
386
387 if (likely(res)) {
388 res->tv_sec = 0;
389 res->tv_nsec = ns;
390 }
391 return 0;
392 }
393
394 static __maybe_unused
__cvdso_clock_getres_data(const struct vdso_data * vd,clockid_t clock,struct __kernel_timespec * res)395 int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
396 struct __kernel_timespec *res)
397 {
398 int ret = __cvdso_clock_getres_common(vd, clock, res);
399
400 if (unlikely(ret))
401 return clock_getres_fallback(clock, res);
402 return 0;
403 }
404
405 static __maybe_unused
__cvdso_clock_getres(clockid_t clock,struct __kernel_timespec * res)406 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
407 {
408 return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
409 }
410
411 #ifdef BUILD_VDSO32
412 static __maybe_unused int
__cvdso_clock_getres_time32_data(const struct vdso_data * vd,clockid_t clock,struct old_timespec32 * res)413 __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
414 struct old_timespec32 *res)
415 {
416 struct __kernel_timespec ts;
417 int ret;
418
419 ret = __cvdso_clock_getres_common(vd, clock, &ts);
420
421 if (unlikely(ret))
422 return clock_getres32_fallback(clock, res);
423
424 if (likely(res)) {
425 res->tv_sec = ts.tv_sec;
426 res->tv_nsec = ts.tv_nsec;
427 }
428 return ret;
429 }
430
431 static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock,struct old_timespec32 * res)432 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
433 {
434 return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
435 clock, res);
436 }
437 #endif /* BUILD_VDSO32 */
438 #endif /* VDSO_HAS_CLOCK_GETRES */
439
