1 /*
2 * Copyright (c) 2018 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/kernel.h>
8 #include <zephyr/spinlock.h>
9 #include <ksched.h>
10 #include <timeout_q.h>
11 #include <zephyr/internal/syscall_handler.h>
12 #include <zephyr/drivers/timer/system_timer.h>
13 #include <zephyr/sys_clock.h>
14
15 static uint64_t curr_tick;
16
17 static sys_dlist_t timeout_list = SYS_DLIST_STATIC_INIT(&timeout_list);
18
19 /*
20 * The timeout code shall take no locks other than its own (timeout_lock), nor
21 * shall it call any other subsystem while holding this lock.
22 */
23 static struct k_spinlock timeout_lock;
24
25 #define MAX_WAIT (IS_ENABLED(CONFIG_SYSTEM_CLOCK_SLOPPY_IDLE) \
26 ? K_TICKS_FOREVER : INT_MAX)
27
28 /* Ticks left to process in the currently-executing sys_clock_announce() */
29 static int announce_remaining;
30
31 #if defined(CONFIG_TIMER_READS_ITS_FREQUENCY_AT_RUNTIME)
32 int z_clock_hw_cycles_per_sec = CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC;
33
34 #ifdef CONFIG_USERSPACE
z_vrfy_sys_clock_hw_cycles_per_sec_runtime_get(void)35 static inline int z_vrfy_sys_clock_hw_cycles_per_sec_runtime_get(void)
36 {
37 return z_impl_sys_clock_hw_cycles_per_sec_runtime_get();
38 }
39 #include <zephyr/syscalls/sys_clock_hw_cycles_per_sec_runtime_get_mrsh.c>
40 #endif /* CONFIG_USERSPACE */
41 #endif /* CONFIG_TIMER_READS_ITS_FREQUENCY_AT_RUNTIME */
42
first(void)43 static struct _timeout *first(void)
44 {
45 sys_dnode_t *t = sys_dlist_peek_head(&timeout_list);
46
47 return (t == NULL) ? NULL : CONTAINER_OF(t, struct _timeout, node);
48 }
49
next(struct _timeout * t)50 static struct _timeout *next(struct _timeout *t)
51 {
52 sys_dnode_t *n = sys_dlist_peek_next(&timeout_list, &t->node);
53
54 return (n == NULL) ? NULL : CONTAINER_OF(n, struct _timeout, node);
55 }
56
remove_timeout(struct _timeout * t)57 static void remove_timeout(struct _timeout *t)
58 {
59 if (next(t) != NULL) {
60 next(t)->dticks += t->dticks;
61 }
62
63 sys_dlist_remove(&t->node);
64 }
65
elapsed(void)66 static int32_t elapsed(void)
67 {
68 /* While sys_clock_announce() is executing, new relative timeouts will be
69 * scheduled relatively to the currently firing timeout's original tick
70 * value (=curr_tick) rather than relative to the current
71 * sys_clock_elapsed().
72 *
73 * This means that timeouts being scheduled from within timeout callbacks
74 * will be scheduled at well-defined offsets from the currently firing
75 * timeout.
76 *
77 * As a side effect, the same will happen if an ISR with higher priority
78 * preempts a timeout callback and schedules a timeout.
79 *
80 * The distinction is implemented by looking at announce_remaining which
81 * will be non-zero while sys_clock_announce() is executing and zero
82 * otherwise.
83 */
84 return announce_remaining == 0 ? sys_clock_elapsed() : 0U;
85 }
86
next_timeout(void)87 static int32_t next_timeout(void)
88 {
89 struct _timeout *to = first();
90 int32_t ticks_elapsed = elapsed();
91 int32_t ret;
92
93 if ((to == NULL) ||
94 ((int64_t)(to->dticks - ticks_elapsed) > (int64_t)INT_MAX)) {
95 ret = MAX_WAIT;
96 } else {
97 ret = MAX(0, to->dticks - ticks_elapsed);
98 }
99
100 return ret;
101 }
102
z_add_timeout(struct _timeout * to,_timeout_func_t fn,k_timeout_t timeout)103 void z_add_timeout(struct _timeout *to, _timeout_func_t fn,
104 k_timeout_t timeout)
105 {
106 if (K_TIMEOUT_EQ(timeout, K_FOREVER)) {
107 return;
108 }
109
110 #ifdef CONFIG_KERNEL_COHERENCE
111 __ASSERT_NO_MSG(arch_mem_coherent(to));
112 #endif /* CONFIG_KERNEL_COHERENCE */
113
114 __ASSERT(!sys_dnode_is_linked(&to->node), "");
115 to->fn = fn;
116
117 K_SPINLOCK(&timeout_lock) {
118 struct _timeout *t;
119
120 if (Z_IS_TIMEOUT_RELATIVE(timeout)) {
121 to->dticks = timeout.ticks + 1 + elapsed();
122 } else {
123 k_ticks_t ticks = Z_TICK_ABS(timeout.ticks) - curr_tick;
124
125 to->dticks = MAX(1, ticks);
126 }
127
128 for (t = first(); t != NULL; t = next(t)) {
129 if (t->dticks > to->dticks) {
130 t->dticks -= to->dticks;
131 sys_dlist_insert(&t->node, &to->node);
132 break;
133 }
134 to->dticks -= t->dticks;
135 }
136
137 if (t == NULL) {
138 sys_dlist_append(&timeout_list, &to->node);
139 }
140
141 if (to == first() && announce_remaining == 0) {
142 sys_clock_set_timeout(next_timeout(), false);
143 }
144 }
145 }
146
z_abort_timeout(struct _timeout * to)147 int z_abort_timeout(struct _timeout *to)
148 {
149 int ret = -EINVAL;
150
151 K_SPINLOCK(&timeout_lock) {
152 if (sys_dnode_is_linked(&to->node)) {
153 bool is_first = (to == first());
154
155 remove_timeout(to);
156 ret = 0;
157 if (is_first) {
158 sys_clock_set_timeout(next_timeout(), false);
159 }
160 }
161 }
162
163 return ret;
164 }
165
166 /* must be locked */
timeout_rem(const struct _timeout * timeout)167 static k_ticks_t timeout_rem(const struct _timeout *timeout)
168 {
169 k_ticks_t ticks = 0;
170
171 for (struct _timeout *t = first(); t != NULL; t = next(t)) {
172 ticks += t->dticks;
173 if (timeout == t) {
174 break;
175 }
176 }
177
178 return ticks;
179 }
180
z_timeout_remaining(const struct _timeout * timeout)181 k_ticks_t z_timeout_remaining(const struct _timeout *timeout)
182 {
183 k_ticks_t ticks = 0;
184
185 K_SPINLOCK(&timeout_lock) {
186 if (!z_is_inactive_timeout(timeout)) {
187 ticks = timeout_rem(timeout) - elapsed();
188 }
189 }
190
191 return ticks;
192 }
193
z_timeout_expires(const struct _timeout * timeout)194 k_ticks_t z_timeout_expires(const struct _timeout *timeout)
195 {
196 k_ticks_t ticks = 0;
197
198 K_SPINLOCK(&timeout_lock) {
199 ticks = curr_tick;
200 if (!z_is_inactive_timeout(timeout)) {
201 ticks += timeout_rem(timeout);
202 }
203 }
204
205 return ticks;
206 }
207
z_get_next_timeout_expiry(void)208 int32_t z_get_next_timeout_expiry(void)
209 {
210 int32_t ret = (int32_t) K_TICKS_FOREVER;
211
212 K_SPINLOCK(&timeout_lock) {
213 ret = next_timeout();
214 }
215 return ret;
216 }
217
sys_clock_announce(int32_t ticks)218 void sys_clock_announce(int32_t ticks)
219 {
220 k_spinlock_key_t key = k_spin_lock(&timeout_lock);
221
222 /* We release the lock around the callbacks below, so on SMP
223 * systems someone might be already running the loop. Don't
224 * race (which will cause parallel execution of "sequential"
225 * timeouts and confuse apps), just increment the tick count
226 * and return.
227 */
228 if (IS_ENABLED(CONFIG_SMP) && (announce_remaining != 0)) {
229 announce_remaining += ticks;
230 k_spin_unlock(&timeout_lock, key);
231 return;
232 }
233
234 announce_remaining = ticks;
235
236 struct _timeout *t;
237
238 for (t = first();
239 (t != NULL) && (t->dticks <= announce_remaining);
240 t = first()) {
241 int dt = t->dticks;
242
243 curr_tick += dt;
244 t->dticks = 0;
245 remove_timeout(t);
246
247 k_spin_unlock(&timeout_lock, key);
248 t->fn(t);
249 key = k_spin_lock(&timeout_lock);
250 announce_remaining -= dt;
251 }
252
253 if (t != NULL) {
254 t->dticks -= announce_remaining;
255 }
256
257 curr_tick += announce_remaining;
258 announce_remaining = 0;
259
260 sys_clock_set_timeout(next_timeout(), false);
261
262 k_spin_unlock(&timeout_lock, key);
263
264 #ifdef CONFIG_TIMESLICING
265 z_time_slice();
266 #endif /* CONFIG_TIMESLICING */
267 }
268
sys_clock_tick_get(void)269 int64_t sys_clock_tick_get(void)
270 {
271 uint64_t t = 0U;
272
273 K_SPINLOCK(&timeout_lock) {
274 t = curr_tick + elapsed();
275 }
276 return t;
277 }
278
sys_clock_tick_get_32(void)279 uint32_t sys_clock_tick_get_32(void)
280 {
281 #ifdef CONFIG_TICKLESS_KERNEL
282 return (uint32_t)sys_clock_tick_get();
283 #else
284 return (uint32_t)curr_tick;
285 #endif /* CONFIG_TICKLESS_KERNEL */
286 }
287
z_impl_k_uptime_ticks(void)288 int64_t z_impl_k_uptime_ticks(void)
289 {
290 return sys_clock_tick_get();
291 }
292
293 #ifdef CONFIG_USERSPACE
z_vrfy_k_uptime_ticks(void)294 static inline int64_t z_vrfy_k_uptime_ticks(void)
295 {
296 return z_impl_k_uptime_ticks();
297 }
298 #include <zephyr/syscalls/k_uptime_ticks_mrsh.c>
299 #endif /* CONFIG_USERSPACE */
300
sys_timepoint_calc(k_timeout_t timeout)301 k_timepoint_t sys_timepoint_calc(k_timeout_t timeout)
302 {
303 k_timepoint_t timepoint;
304
305 if (K_TIMEOUT_EQ(timeout, K_FOREVER)) {
306 timepoint.tick = UINT64_MAX;
307 } else if (K_TIMEOUT_EQ(timeout, K_NO_WAIT)) {
308 timepoint.tick = 0;
309 } else {
310 k_ticks_t dt = timeout.ticks;
311
312 if (Z_IS_TIMEOUT_RELATIVE(timeout)) {
313 timepoint.tick = sys_clock_tick_get() + MAX(1, dt);
314 } else {
315 timepoint.tick = Z_TICK_ABS(dt);
316 }
317 }
318
319 return timepoint;
320 }
321
sys_timepoint_timeout(k_timepoint_t timepoint)322 k_timeout_t sys_timepoint_timeout(k_timepoint_t timepoint)
323 {
324 uint64_t now, remaining;
325
326 if (timepoint.tick == UINT64_MAX) {
327 return K_FOREVER;
328 }
329 if (timepoint.tick == 0) {
330 return K_NO_WAIT;
331 }
332
333 now = sys_clock_tick_get();
334 remaining = (timepoint.tick > now) ? (timepoint.tick - now) : 0;
335 return K_TICKS(remaining);
336 }
337
338 #ifdef CONFIG_ZTEST
z_impl_sys_clock_tick_set(uint64_t tick)339 void z_impl_sys_clock_tick_set(uint64_t tick)
340 {
341 curr_tick = tick;
342 }
343
z_vrfy_sys_clock_tick_set(uint64_t tick)344 void z_vrfy_sys_clock_tick_set(uint64_t tick)
345 {
346 z_impl_sys_clock_tick_set(tick);
347 }
348 #endif /* CONFIG_ZTEST */
349
