1 /*
2 * Copyright (c) 2018 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/kernel.h>
8 #include <zephyr/spinlock.h>
9 #include <ksched.h>
10 #include <timeout_q.h>
11 #include <zephyr/internal/syscall_handler.h>
12 #include <zephyr/drivers/timer/system_timer.h>
13 #include <zephyr/sys_clock.h>
14
15 static uint64_t curr_tick;
16
17 static sys_dlist_t timeout_list = SYS_DLIST_STATIC_INIT(&timeout_list);
18
19 static struct k_spinlock timeout_lock;
20
21 #define MAX_WAIT (IS_ENABLED(CONFIG_SYSTEM_CLOCK_SLOPPY_IDLE) \
22 ? K_TICKS_FOREVER : INT_MAX)
23
24 /* Ticks left to process in the currently-executing sys_clock_announce() */
25 static int announce_remaining;
26
27 #if defined(CONFIG_TIMER_READS_ITS_FREQUENCY_AT_RUNTIME)
28 int z_clock_hw_cycles_per_sec = CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC;
29
30 #ifdef CONFIG_USERSPACE
z_vrfy_sys_clock_hw_cycles_per_sec_runtime_get(void)31 static inline int z_vrfy_sys_clock_hw_cycles_per_sec_runtime_get(void)
32 {
33 return z_impl_sys_clock_hw_cycles_per_sec_runtime_get();
34 }
35 #include <syscalls/sys_clock_hw_cycles_per_sec_runtime_get_mrsh.c>
36 #endif /* CONFIG_USERSPACE */
37 #endif /* CONFIG_TIMER_READS_ITS_FREQUENCY_AT_RUNTIME */
38
first(void)39 static struct _timeout *first(void)
40 {
41 sys_dnode_t *t = sys_dlist_peek_head(&timeout_list);
42
43 return t == NULL ? NULL : CONTAINER_OF(t, struct _timeout, node);
44 }
45
next(struct _timeout * t)46 static struct _timeout *next(struct _timeout *t)
47 {
48 sys_dnode_t *n = sys_dlist_peek_next(&timeout_list, &t->node);
49
50 return n == NULL ? NULL : CONTAINER_OF(n, struct _timeout, node);
51 }
52
remove_timeout(struct _timeout * t)53 static void remove_timeout(struct _timeout *t)
54 {
55 if (next(t) != NULL) {
56 next(t)->dticks += t->dticks;
57 }
58
59 sys_dlist_remove(&t->node);
60 }
61
elapsed(void)62 static int32_t elapsed(void)
63 {
64 /* While sys_clock_announce() is executing, new relative timeouts will be
65 * scheduled relatively to the currently firing timeout's original tick
66 * value (=curr_tick) rather than relative to the current
67 * sys_clock_elapsed().
68 *
69 * This means that timeouts being scheduled from within timeout callbacks
70 * will be scheduled at well-defined offsets from the currently firing
71 * timeout.
72 *
73 * As a side effect, the same will happen if an ISR with higher priority
74 * preempts a timeout callback and schedules a timeout.
75 *
76 * The distinction is implemented by looking at announce_remaining which
77 * will be non-zero while sys_clock_announce() is executing and zero
78 * otherwise.
79 */
80 return announce_remaining == 0 ? sys_clock_elapsed() : 0U;
81 }
82
next_timeout(void)83 static int32_t next_timeout(void)
84 {
85 struct _timeout *to = first();
86 int32_t ticks_elapsed = elapsed();
87 int32_t ret;
88
89 if ((to == NULL) ||
90 ((int64_t)(to->dticks - ticks_elapsed) > (int64_t)INT_MAX)) {
91 ret = MAX_WAIT;
92 } else {
93 ret = MAX(0, to->dticks - ticks_elapsed);
94 }
95
96 return ret;
97 }
98
z_add_timeout(struct _timeout * to,_timeout_func_t fn,k_timeout_t timeout)99 void z_add_timeout(struct _timeout *to, _timeout_func_t fn,
100 k_timeout_t timeout)
101 {
102 if (K_TIMEOUT_EQ(timeout, K_FOREVER)) {
103 return;
104 }
105
106 #ifdef CONFIG_KERNEL_COHERENCE
107 __ASSERT_NO_MSG(arch_mem_coherent(to));
108 #endif
109
110 __ASSERT(!sys_dnode_is_linked(&to->node), "");
111 to->fn = fn;
112
113 K_SPINLOCK(&timeout_lock) {
114 struct _timeout *t;
115
116 if (IS_ENABLED(CONFIG_TIMEOUT_64BIT) &&
117 Z_TICK_ABS(timeout.ticks) >= 0) {
118 k_ticks_t ticks = Z_TICK_ABS(timeout.ticks) - curr_tick;
119
120 to->dticks = MAX(1, ticks);
121 } else {
122 to->dticks = timeout.ticks + 1 + elapsed();
123 }
124
125 for (t = first(); t != NULL; t = next(t)) {
126 if (t->dticks > to->dticks) {
127 t->dticks -= to->dticks;
128 sys_dlist_insert(&t->node, &to->node);
129 break;
130 }
131 to->dticks -= t->dticks;
132 }
133
134 if (t == NULL) {
135 sys_dlist_append(&timeout_list, &to->node);
136 }
137
138 if (to == first()) {
139 sys_clock_set_timeout(next_timeout(), false);
140 }
141 }
142 }
143
z_abort_timeout(struct _timeout * to)144 int z_abort_timeout(struct _timeout *to)
145 {
146 int ret = -EINVAL;
147
148 K_SPINLOCK(&timeout_lock) {
149 if (sys_dnode_is_linked(&to->node)) {
150 remove_timeout(to);
151 ret = 0;
152 }
153 }
154
155 return ret;
156 }
157
158 /* must be locked */
timeout_rem(const struct _timeout * timeout)159 static k_ticks_t timeout_rem(const struct _timeout *timeout)
160 {
161 k_ticks_t ticks = 0;
162
163 if (z_is_inactive_timeout(timeout)) {
164 return 0;
165 }
166
167 for (struct _timeout *t = first(); t != NULL; t = next(t)) {
168 ticks += t->dticks;
169 if (timeout == t) {
170 break;
171 }
172 }
173
174 return ticks - elapsed();
175 }
176
z_timeout_remaining(const struct _timeout * timeout)177 k_ticks_t z_timeout_remaining(const struct _timeout *timeout)
178 {
179 k_ticks_t ticks = 0;
180
181 K_SPINLOCK(&timeout_lock) {
182 ticks = timeout_rem(timeout);
183 }
184
185 return ticks;
186 }
187
z_timeout_expires(const struct _timeout * timeout)188 k_ticks_t z_timeout_expires(const struct _timeout *timeout)
189 {
190 k_ticks_t ticks = 0;
191
192 K_SPINLOCK(&timeout_lock) {
193 ticks = curr_tick + timeout_rem(timeout) + elapsed();
194 }
195
196 return ticks;
197 }
198
z_get_next_timeout_expiry(void)199 int32_t z_get_next_timeout_expiry(void)
200 {
201 int32_t ret = (int32_t) K_TICKS_FOREVER;
202
203 K_SPINLOCK(&timeout_lock) {
204 ret = next_timeout();
205 }
206 return ret;
207 }
208
sys_clock_announce(int32_t ticks)209 void sys_clock_announce(int32_t ticks)
210 {
211 k_spinlock_key_t key = k_spin_lock(&timeout_lock);
212
213 /* We release the lock around the callbacks below, so on SMP
214 * systems someone might be already running the loop. Don't
215 * race (which will cause paralllel execution of "sequential"
216 * timeouts and confuse apps), just increment the tick count
217 * and return.
218 */
219 if (IS_ENABLED(CONFIG_SMP) && (announce_remaining != 0)) {
220 announce_remaining += ticks;
221 k_spin_unlock(&timeout_lock, key);
222 return;
223 }
224
225 announce_remaining = ticks;
226
227 struct _timeout *t;
228
229 for (t = first();
230 (t != NULL) && (t->dticks <= announce_remaining);
231 t = first()) {
232 int dt = t->dticks;
233
234 curr_tick += dt;
235 t->dticks = 0;
236 remove_timeout(t);
237
238 k_spin_unlock(&timeout_lock, key);
239 t->fn(t);
240 key = k_spin_lock(&timeout_lock);
241 announce_remaining -= dt;
242 }
243
244 if (t != NULL) {
245 t->dticks -= announce_remaining;
246 }
247
248 curr_tick += announce_remaining;
249 announce_remaining = 0;
250
251 sys_clock_set_timeout(next_timeout(), false);
252
253 k_spin_unlock(&timeout_lock, key);
254
255 #ifdef CONFIG_TIMESLICING
256 z_time_slice();
257 #endif
258 }
259
sys_clock_tick_get(void)260 int64_t sys_clock_tick_get(void)
261 {
262 uint64_t t = 0U;
263
264 K_SPINLOCK(&timeout_lock) {
265 t = curr_tick + elapsed();
266 }
267 return t;
268 }
269
sys_clock_tick_get_32(void)270 uint32_t sys_clock_tick_get_32(void)
271 {
272 #ifdef CONFIG_TICKLESS_KERNEL
273 return (uint32_t)sys_clock_tick_get();
274 #else
275 return (uint32_t)curr_tick;
276 #endif
277 }
278
z_impl_k_uptime_ticks(void)279 int64_t z_impl_k_uptime_ticks(void)
280 {
281 return sys_clock_tick_get();
282 }
283
284 #ifdef CONFIG_USERSPACE
z_vrfy_k_uptime_ticks(void)285 static inline int64_t z_vrfy_k_uptime_ticks(void)
286 {
287 return z_impl_k_uptime_ticks();
288 }
289 #include <syscalls/k_uptime_ticks_mrsh.c>
290 #endif
291
sys_timepoint_calc(k_timeout_t timeout)292 k_timepoint_t sys_timepoint_calc(k_timeout_t timeout)
293 {
294 k_timepoint_t timepoint;
295
296 if (K_TIMEOUT_EQ(timeout, K_FOREVER)) {
297 timepoint.tick = UINT64_MAX;
298 } else if (K_TIMEOUT_EQ(timeout, K_NO_WAIT)) {
299 timepoint.tick = 0;
300 } else {
301 k_ticks_t dt = timeout.ticks;
302
303 if (IS_ENABLED(CONFIG_TIMEOUT_64BIT) && Z_TICK_ABS(dt) >= 0) {
304 timepoint.tick = Z_TICK_ABS(dt);
305 } else {
306 timepoint.tick = sys_clock_tick_get() + MAX(1, dt);
307 }
308 }
309
310 return timepoint;
311 }
312
sys_timepoint_timeout(k_timepoint_t timepoint)313 k_timeout_t sys_timepoint_timeout(k_timepoint_t timepoint)
314 {
315 uint64_t now, remaining;
316
317 if (timepoint.tick == UINT64_MAX) {
318 return K_FOREVER;
319 }
320 if (timepoint.tick == 0) {
321 return K_NO_WAIT;
322 }
323
324 now = sys_clock_tick_get();
325 remaining = (timepoint.tick > now) ? (timepoint.tick - now) : 0;
326 return K_TICKS(remaining);
327 }
328
329 #ifdef CONFIG_ZTEST
z_impl_sys_clock_tick_set(uint64_t tick)330 void z_impl_sys_clock_tick_set(uint64_t tick)
331 {
332 curr_tick = tick;
333 }
334
z_vrfy_sys_clock_tick_set(uint64_t tick)335 void z_vrfy_sys_clock_tick_set(uint64_t tick)
336 {
337 z_impl_sys_clock_tick_set(tick);
338 }
339 #endif
340
