1 /*
2 * Copyright (c) 2018 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/kernel.h>
8
9 #include <zephyr/timing/timing.h>
10 #include <ksched.h>
11 #include <zephyr/spinlock.h>
12 #include <zephyr/sys/check.h>
13
14 /* Need one of these for this to work */
15 #if !defined(CONFIG_USE_SWITCH) && !defined(CONFIG_INSTRUMENT_THREAD_SWITCHING)
16 #error "No data backend configured for CONFIG_SCHED_THREAD_USAGE"
17 #endif /* !CONFIG_USE_SWITCH && !CONFIG_INSTRUMENT_THREAD_SWITCHING */
18
19 static struct k_spinlock usage_lock;
20
usage_now(void)21 static uint32_t usage_now(void)
22 {
23 uint32_t now;
24
25 #ifdef CONFIG_THREAD_RUNTIME_STATS_USE_TIMING_FUNCTIONS
26 now = (uint32_t)timing_counter_get();
27 #else
28 now = k_cycle_get_32();
29 #endif /* CONFIG_THREAD_RUNTIME_STATS_USE_TIMING_FUNCTIONS */
30
31 /* Edge case: we use a zero as a null ("stop() already called") */
32 return (now == 0) ? 1 : now;
33 }
34
35 #ifdef CONFIG_SCHED_THREAD_USAGE_ALL
sched_cpu_update_usage(struct _cpu * cpu,uint32_t cycles)36 static void sched_cpu_update_usage(struct _cpu *cpu, uint32_t cycles)
37 {
38 if (!cpu->usage->track_usage) {
39 return;
40 }
41
42 if (cpu->current != cpu->idle_thread) {
43 cpu->usage->total += cycles;
44
45 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
46 cpu->usage->current += cycles;
47
48 if (cpu->usage->longest < cpu->usage->current) {
49 cpu->usage->longest = cpu->usage->current;
50 }
51 } else {
52 cpu->usage->current = 0;
53 cpu->usage->num_windows++;
54 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
55 }
56 }
57 #else
58 #define sched_cpu_update_usage(cpu, cycles) do { } while (0)
59 #endif /* CONFIG_SCHED_THREAD_USAGE_ALL */
60
sched_thread_update_usage(struct k_thread * thread,uint32_t cycles)61 static void sched_thread_update_usage(struct k_thread *thread, uint32_t cycles)
62 {
63 thread->base.usage.total += cycles;
64
65 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
66 thread->base.usage.current += cycles;
67
68 if (thread->base.usage.longest < thread->base.usage.current) {
69 thread->base.usage.longest = thread->base.usage.current;
70 }
71 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
72 }
73
z_sched_usage_start(struct k_thread * thread)74 void z_sched_usage_start(struct k_thread *thread)
75 {
76 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
77 k_spinlock_key_t key;
78
79 key = k_spin_lock(&usage_lock);
80
81 _current_cpu->usage0 = usage_now(); /* Always update */
82
83 if (thread->base.usage.track_usage) {
84 thread->base.usage.num_windows++;
85 thread->base.usage.current = 0;
86 }
87
88 k_spin_unlock(&usage_lock, key);
89 #else
90 /* One write through a volatile pointer doesn't require
91 * synchronization as long as _usage() treats it as volatile
92 * (we can't race with _stop() by design).
93 */
94
95 _current_cpu->usage0 = usage_now();
96 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
97 }
98
z_sched_usage_stop(void)99 void z_sched_usage_stop(void)
100 {
101 k_spinlock_key_t k = k_spin_lock(&usage_lock);
102
103 struct _cpu *cpu = _current_cpu;
104
105 uint32_t u0 = cpu->usage0;
106
107 if (u0 != 0) {
108 uint32_t cycles = usage_now() - u0;
109
110 if (cpu->current->base.usage.track_usage) {
111 sched_thread_update_usage(cpu->current, cycles);
112 }
113
114 sched_cpu_update_usage(cpu, cycles);
115 }
116
117 cpu->usage0 = 0;
118 k_spin_unlock(&usage_lock, k);
119 }
120
121 #ifdef CONFIG_SCHED_THREAD_USAGE_ALL
z_sched_cpu_usage(uint8_t cpu_id,struct k_thread_runtime_stats * stats)122 void z_sched_cpu_usage(uint8_t cpu_id, struct k_thread_runtime_stats *stats)
123 {
124 k_spinlock_key_t key;
125 struct _cpu *cpu;
126
127 key = k_spin_lock(&usage_lock);
128 cpu = _current_cpu;
129
130
131 if (&_kernel.cpus[cpu_id] == cpu) {
132 uint32_t now = usage_now();
133 uint32_t cycles = now - cpu->usage0;
134
135 /*
136 * Getting stats for the current CPU. Update both its
137 * current thread stats and the CPU stats as the CPU's
138 * [usage0] field will also get updated. This keeps all
139 * that information up-to-date.
140 */
141
142 if (cpu->current->base.usage.track_usage) {
143 sched_thread_update_usage(cpu->current, cycles);
144 }
145
146 sched_cpu_update_usage(cpu, cycles);
147
148 cpu->usage0 = now;
149 }
150
151 stats->total_cycles = cpu->usage->total;
152 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
153 stats->current_cycles = cpu->usage->current;
154 stats->peak_cycles = cpu->usage->longest;
155
156 if (cpu->usage->num_windows == 0) {
157 stats->average_cycles = 0;
158 } else {
159 stats->average_cycles = stats->total_cycles /
160 cpu->usage->num_windows;
161 }
162 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
163
164 stats->idle_cycles =
165 _kernel.cpus[cpu_id].idle_thread->base.usage.total;
166
167 stats->execution_cycles = stats->total_cycles + stats->idle_cycles;
168
169 k_spin_unlock(&usage_lock, key);
170 }
171 #endif /* CONFIG_SCHED_THREAD_USAGE_ALL */
172
z_sched_thread_usage(struct k_thread * thread,struct k_thread_runtime_stats * stats)173 void z_sched_thread_usage(struct k_thread *thread,
174 struct k_thread_runtime_stats *stats)
175 {
176 struct _cpu *cpu;
177 k_spinlock_key_t key;
178
179 key = k_spin_lock(&usage_lock);
180 cpu = _current_cpu;
181
182
183 if (thread == cpu->current) {
184 uint32_t now = usage_now();
185 uint32_t cycles = now - cpu->usage0;
186
187 /*
188 * Getting stats for the current thread. Update both the
189 * current thread stats and its CPU stats as the CPU's
190 * [usage0] field will also get updated. This keeps all
191 * that information up-to-date.
192 */
193
194 if (thread->base.usage.track_usage) {
195 sched_thread_update_usage(thread, cycles);
196 }
197
198 sched_cpu_update_usage(cpu, cycles);
199
200 cpu->usage0 = now;
201 }
202
203 stats->execution_cycles = thread->base.usage.total;
204 stats->total_cycles = thread->base.usage.total;
205
206 /* Copy-out the thread's usage stats */
207
208 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
209 stats->current_cycles = thread->base.usage.current;
210 stats->peak_cycles = thread->base.usage.longest;
211
212 if (thread->base.usage.num_windows == 0) {
213 stats->average_cycles = 0;
214 } else {
215 stats->average_cycles = stats->total_cycles /
216 thread->base.usage.num_windows;
217 }
218 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
219
220 #ifdef CONFIG_SCHED_THREAD_USAGE_ALL
221 stats->idle_cycles = 0;
222 #endif /* CONFIG_SCHED_THREAD_USAGE_ALL */
223
224 k_spin_unlock(&usage_lock, key);
225 }
226
227 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
k_thread_runtime_stats_enable(k_tid_t thread)228 int k_thread_runtime_stats_enable(k_tid_t thread)
229 {
230 k_spinlock_key_t key;
231
232 CHECKIF(thread == NULL) {
233 return -EINVAL;
234 }
235
236 key = k_spin_lock(&usage_lock);
237
238 if (!thread->base.usage.track_usage) {
239 thread->base.usage.track_usage = true;
240 thread->base.usage.num_windows++;
241 thread->base.usage.current = 0;
242 }
243
244 k_spin_unlock(&usage_lock, key);
245
246 return 0;
247 }
248
k_thread_runtime_stats_disable(k_tid_t thread)249 int k_thread_runtime_stats_disable(k_tid_t thread)
250 {
251 k_spinlock_key_t key;
252
253 CHECKIF(thread == NULL) {
254 return -EINVAL;
255 }
256
257 key = k_spin_lock(&usage_lock);
258 struct _cpu *cpu = _current_cpu;
259
260 if (thread->base.usage.track_usage) {
261 thread->base.usage.track_usage = false;
262
263 if (thread == cpu->current) {
264 uint32_t cycles = usage_now() - cpu->usage0;
265
266 sched_thread_update_usage(thread, cycles);
267 sched_cpu_update_usage(cpu, cycles);
268 }
269 }
270
271 k_spin_unlock(&usage_lock, key);
272
273 return 0;
274 }
275 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
276
277 #ifdef CONFIG_SCHED_THREAD_USAGE_ALL
k_sys_runtime_stats_enable(void)278 void k_sys_runtime_stats_enable(void)
279 {
280 k_spinlock_key_t key;
281
282 key = k_spin_lock(&usage_lock);
283
284 if (_current_cpu->usage->track_usage) {
285
286 /*
287 * Usage tracking is already enabled on the current CPU
288 * and thus on all other CPUs (if applicable). There is
289 * nothing left to do.
290 */
291
292 k_spin_unlock(&usage_lock, key);
293 return;
294 }
295
296 /* Enable gathering of runtime stats on each CPU */
297
298 unsigned int num_cpus = arch_num_cpus();
299
300 for (uint8_t i = 0; i < num_cpus; i++) {
301 _kernel.cpus[i].usage->track_usage = true;
302 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
303 _kernel.cpus[i].usage->num_windows++;
304 _kernel.cpus[i].usage->current = 0;
305 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
306 }
307
308 k_spin_unlock(&usage_lock, key);
309 }
310
k_sys_runtime_stats_disable(void)311 void k_sys_runtime_stats_disable(void)
312 {
313 struct _cpu *cpu;
314 k_spinlock_key_t key;
315
316 key = k_spin_lock(&usage_lock);
317
318 if (!_current_cpu->usage->track_usage) {
319
320 /*
321 * Usage tracking is already disabled on the current CPU
322 * and thus on all other CPUs (if applicable). There is
323 * nothing left to do.
324 */
325
326 k_spin_unlock(&usage_lock, key);
327 return;
328 }
329
330 uint32_t now = usage_now();
331
332 unsigned int num_cpus = arch_num_cpus();
333
334 for (uint8_t i = 0; i < num_cpus; i++) {
335 cpu = &_kernel.cpus[i];
336 if (cpu->usage0 != 0) {
337 sched_cpu_update_usage(cpu, now - cpu->usage0);
338 }
339 cpu->usage->track_usage = false;
340 }
341
342 k_spin_unlock(&usage_lock, key);
343 }
344 #endif /* CONFIG_SCHED_THREAD_USAGE_ALL */
345
346 #ifdef CONFIG_OBJ_CORE_STATS_THREAD
z_thread_stats_raw(struct k_obj_core * obj_core,void * stats)347 int z_thread_stats_raw(struct k_obj_core *obj_core, void *stats)
348 {
349 k_spinlock_key_t key;
350
351 key = k_spin_lock(&usage_lock);
352 memcpy(stats, obj_core->stats, sizeof(struct k_cycle_stats));
353 k_spin_unlock(&usage_lock, key);
354
355 return 0;
356 }
357
z_thread_stats_query(struct k_obj_core * obj_core,void * stats)358 int z_thread_stats_query(struct k_obj_core *obj_core, void *stats)
359 {
360 struct k_thread *thread;
361
362 thread = CONTAINER_OF(obj_core, struct k_thread, obj_core);
363
364 z_sched_thread_usage(thread, stats);
365
366 return 0;
367 }
368
z_thread_stats_reset(struct k_obj_core * obj_core)369 int z_thread_stats_reset(struct k_obj_core *obj_core)
370 {
371 k_spinlock_key_t key;
372 struct k_cycle_stats *stats;
373 struct k_thread *thread;
374
375 thread = CONTAINER_OF(obj_core, struct k_thread, obj_core);
376 key = k_spin_lock(&usage_lock);
377 stats = obj_core->stats;
378
379 stats->total = 0ULL;
380 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
381 stats->current = 0ULL;
382 stats->longest = 0ULL;
383 stats->num_windows = (thread->base.usage.track_usage) ? 1U : 0U;
384 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
385
386 if (thread != _current_cpu->current) {
387
388 /*
389 * If the thread is not running, there is nothing else to do.
390 * If the thread is running on another core, then it is not
391 * safe to do anything else but unlock and return (and pretend
392 * that its stats were reset at the start of its execution
393 * window.
394 */
395
396 k_spin_unlock(&usage_lock, key);
397
398 return 0;
399 }
400
401 /* Update the current CPU stats. */
402
403 uint32_t now = usage_now();
404 uint32_t cycles = now - _current_cpu->usage0;
405
406 sched_cpu_update_usage(_current_cpu, cycles);
407
408 _current_cpu->usage0 = now;
409
410 k_spin_unlock(&usage_lock, key);
411
412 return 0;
413 }
414
z_thread_stats_disable(struct k_obj_core * obj_core)415 int z_thread_stats_disable(struct k_obj_core *obj_core)
416 {
417 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
418 struct k_thread *thread;
419
420 thread = CONTAINER_OF(obj_core, struct k_thread, obj_core);
421
422 return k_thread_runtime_stats_disable(thread);
423 #else
424 return -ENOTSUP;
425 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
426 }
427
z_thread_stats_enable(struct k_obj_core * obj_core)428 int z_thread_stats_enable(struct k_obj_core *obj_core)
429 {
430 #ifdef CONFIG_SCHED_THREAD_USAGE_ANALYSIS
431 struct k_thread *thread;
432
433 thread = CONTAINER_OF(obj_core, struct k_thread, obj_core);
434
435 return k_thread_runtime_stats_enable(thread);
436 #else
437 return -ENOTSUP;
438 #endif /* CONFIG_SCHED_THREAD_USAGE_ANALYSIS */
439 }
440 #endif /* CONFIG_OBJ_CORE_STATS_THREAD */
441
442 #ifdef CONFIG_OBJ_CORE_STATS_SYSTEM
z_cpu_stats_raw(struct k_obj_core * obj_core,void * stats)443 int z_cpu_stats_raw(struct k_obj_core *obj_core, void *stats)
444 {
445 k_spinlock_key_t key;
446
447 key = k_spin_lock(&usage_lock);
448 memcpy(stats, obj_core->stats, sizeof(struct k_cycle_stats));
449 k_spin_unlock(&usage_lock, key);
450
451 return 0;
452 }
453
z_cpu_stats_query(struct k_obj_core * obj_core,void * stats)454 int z_cpu_stats_query(struct k_obj_core *obj_core, void *stats)
455 {
456 struct _cpu *cpu;
457
458 cpu = CONTAINER_OF(obj_core, struct _cpu, obj_core);
459
460 z_sched_cpu_usage(cpu->id, stats);
461
462 return 0;
463 }
464 #endif /* CONFIG_OBJ_CORE_STATS_SYSTEM */
465
466 #ifdef CONFIG_OBJ_CORE_STATS_SYSTEM
z_kernel_stats_raw(struct k_obj_core * obj_core,void * stats)467 int z_kernel_stats_raw(struct k_obj_core *obj_core, void *stats)
468 {
469 k_spinlock_key_t key;
470
471 key = k_spin_lock(&usage_lock);
472 memcpy(stats, obj_core->stats,
473 CONFIG_MP_MAX_NUM_CPUS * sizeof(struct k_cycle_stats));
474 k_spin_unlock(&usage_lock, key);
475
476 return 0;
477 }
478
z_kernel_stats_query(struct k_obj_core * obj_core,void * stats)479 int z_kernel_stats_query(struct k_obj_core *obj_core, void *stats)
480 {
481 ARG_UNUSED(obj_core);
482
483 return k_thread_runtime_stats_all_get(stats);
484 }
485 #endif /* CONFIG_OBJ_CORE_STATS_SYSTEM */
486
