Lines Matching refs:thread
39 static ALWAYS_INLINE void halt_thread(struct k_thread *thread, uint8_t new_state);
40 static void add_to_waitq_locked(struct k_thread *thread, _wait_q_t *wait_q);
48 static ALWAYS_INLINE void *thread_runq(struct k_thread *thread) in thread_runq() argument
51 int cpu, m = thread->base.cpu_mask; in thread_runq()
63 ARG_UNUSED(thread); in thread_runq()
77 static ALWAYS_INLINE void runq_add(struct k_thread *thread) in runq_add() argument
79 __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); in runq_add()
80 __ASSERT_NO_MSG(!is_thread_dummy(thread)); in runq_add()
82 _priq_run_add(thread_runq(thread), thread); in runq_add()
85 static ALWAYS_INLINE void runq_remove(struct k_thread *thread) in runq_remove() argument
87 __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); in runq_remove()
88 __ASSERT_NO_MSG(!is_thread_dummy(thread)); in runq_remove()
90 _priq_run_remove(thread_runq(thread), thread); in runq_remove()
106 static inline bool should_queue_thread(struct k_thread *thread) in should_queue_thread() argument
108 return !IS_ENABLED(CONFIG_SMP) || (thread != _current); in should_queue_thread()
111 static ALWAYS_INLINE void queue_thread(struct k_thread *thread) in queue_thread() argument
113 z_mark_thread_as_queued(thread); in queue_thread()
114 if (should_queue_thread(thread)) { in queue_thread()
115 runq_add(thread); in queue_thread()
118 if (thread == _current) { in queue_thread()
125 static ALWAYS_INLINE void dequeue_thread(struct k_thread *thread) in dequeue_thread() argument
127 z_mark_thread_as_not_queued(thread); in dequeue_thread()
128 if (should_queue_thread(thread)) { in dequeue_thread()
129 runq_remove(thread); in dequeue_thread()
139 void z_requeue_current(struct k_thread *thread) in z_requeue_current() argument
141 if (z_is_thread_queued(thread)) { in z_requeue_current()
142 runq_add(thread); in z_requeue_current()
148 static inline void clear_halting(struct k_thread *thread) in clear_halting() argument
152 thread->base.thread_state &= ~(_THREAD_ABORTING | _THREAD_SUSPENDING); in clear_halting()
160 static void update_metairq_preempt(struct k_thread *thread) in update_metairq_preempt() argument
163 if (thread_is_metairq(thread) && !thread_is_metairq(_current) && in update_metairq_preempt()
167 } else if (!thread_is_metairq(thread)) { in update_metairq_preempt()
172 ARG_UNUSED(thread); in update_metairq_preempt()
185 struct k_thread *thread = runq_best(); in next_up() local
195 if (mirqp != NULL && (thread == NULL || !thread_is_metairq(thread))) { in next_up()
197 thread = mirqp; in next_up()
211 return (thread != NULL) ? thread : _current_cpu->idle_thread; in next_up()
227 if (thread == NULL) { in next_up()
228 thread = _current_cpu->idle_thread; in next_up()
232 int32_t cmp = z_sched_prio_cmp(_current, thread); in next_up()
236 thread = _current; in next_up()
239 if (!should_preempt(thread, _current_cpu->swap_ok)) { in next_up()
240 thread = _current; in next_up()
244 if (thread != _current) { in next_up()
245 update_metairq_preempt(thread); in next_up()
263 if (z_is_thread_queued(thread)) { in next_up()
264 dequeue_thread(thread); in next_up()
268 return thread; in next_up()
282 struct k_thread *thread = next_up(); in update_cache() local
284 if (should_preempt(thread, preempt_ok)) { in update_cache()
286 if (thread != _current) { in update_cache()
287 z_reset_time_slice(thread); in update_cache()
290 update_metairq_preempt(thread); in update_cache()
291 _kernel.ready_q.cache = thread; in update_cache()
307 static struct _cpu *thread_active_elsewhere(struct k_thread *thread) in thread_active_elsewhere() argument
320 (_kernel.cpus[i].current == thread)) { in thread_active_elsewhere()
325 ARG_UNUSED(thread); in thread_active_elsewhere()
329 static void ready_thread(struct k_thread *thread) in ready_thread() argument
332 __ASSERT_NO_MSG(sys_cache_is_mem_coherent(thread)); in ready_thread()
338 if (!z_is_thread_queued(thread) && z_is_thread_ready(thread)) { in ready_thread()
339 SYS_PORT_TRACING_OBJ_FUNC(k_thread, sched_ready, thread); in ready_thread()
341 queue_thread(thread); in ready_thread()
344 flag_ipi(ipi_mask_create(thread)); in ready_thread()
348 void z_ready_thread(struct k_thread *thread) in z_ready_thread() argument
351 if (thread_active_elsewhere(thread) == NULL) { in z_ready_thread()
352 ready_thread(thread); in z_ready_thread()
371 static void thread_halt_spin(struct k_thread *thread, k_spinlock_key_t key) in thread_halt_spin() argument
378 while (z_is_thread_halting(thread)) { in thread_halt_spin()
390 static ALWAYS_INLINE void z_metairq_preempted_clear(struct k_thread *thread) in z_metairq_preempted_clear() argument
394 if (_kernel.cpus[i].metairq_preempted == thread) { in z_metairq_preempted_clear()
408 static ALWAYS_INLINE void z_thread_halt(struct k_thread *thread, k_spinlock_key_t key, in z_thread_halt() argument
411 _wait_q_t *wq = &thread->join_queue; in z_thread_halt()
413 wq = terminate ? wq : &thread->halt_queue; in z_thread_halt()
416 z_metairq_preempted_clear(thread); in z_thread_halt()
425 struct _cpu *cpu = thread_active_elsewhere(thread); in z_thread_halt()
428 thread->base.thread_state |= (terminate ? _THREAD_ABORTING in z_thread_halt()
438 thread_halt_spin(thread, key); in z_thread_halt()
444 halt_thread(thread, terminate ? _THREAD_DEAD : _THREAD_SUSPENDED); in z_thread_halt()
445 if ((thread == _current) && !arch_is_in_isr()) { in z_thread_halt()
446 if (z_is_thread_essential(thread)) { in z_thread_halt()
464 void z_impl_k_thread_suspend(k_tid_t thread) in z_impl_k_thread_suspend() argument
466 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_thread, suspend, thread); in z_impl_k_thread_suspend()
471 if (!IS_ENABLED(CONFIG_SMP) && (thread == _current) && !arch_is_in_isr()) { in z_impl_k_thread_suspend()
474 z_mark_thread_as_suspended(thread); in z_impl_k_thread_suspend()
475 z_metairq_preempted_clear(thread); in z_impl_k_thread_suspend()
476 dequeue_thread(thread); in z_impl_k_thread_suspend()
484 if (unlikely(z_is_thread_suspended(thread))) { in z_impl_k_thread_suspend()
492 z_thread_halt(thread, key, false); in z_impl_k_thread_suspend()
494 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_thread, suspend, thread); in z_impl_k_thread_suspend()
498 static inline void z_vrfy_k_thread_suspend(k_tid_t thread) in z_vrfy_k_thread_suspend() argument
500 K_OOPS(K_SYSCALL_OBJ(thread, K_OBJ_THREAD)); in z_vrfy_k_thread_suspend()
501 z_impl_k_thread_suspend(thread); in z_vrfy_k_thread_suspend()
506 void z_impl_k_thread_resume(k_tid_t thread) in z_impl_k_thread_resume() argument
508 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_thread, resume, thread); in z_impl_k_thread_resume()
513 if (unlikely(!z_is_thread_suspended(thread))) { in z_impl_k_thread_resume()
518 z_mark_thread_as_not_suspended(thread); in z_impl_k_thread_resume()
519 ready_thread(thread); in z_impl_k_thread_resume()
523 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_thread, resume, thread); in z_impl_k_thread_resume()
527 static inline void z_vrfy_k_thread_resume(k_tid_t thread) in z_vrfy_k_thread_resume() argument
529 K_OOPS(K_SYSCALL_OBJ(thread, K_OBJ_THREAD)); in z_vrfy_k_thread_resume()
530 z_impl_k_thread_resume(thread); in z_vrfy_k_thread_resume()
535 static void unready_thread(struct k_thread *thread) in unready_thread() argument
537 if (z_is_thread_queued(thread)) { in unready_thread()
538 dequeue_thread(thread); in unready_thread()
540 update_cache(thread == _current); in unready_thread()
544 static void add_to_waitq_locked(struct k_thread *thread, _wait_q_t *wait_q) in add_to_waitq_locked() argument
546 unready_thread(thread); in add_to_waitq_locked()
547 z_mark_thread_as_pending(thread); in add_to_waitq_locked()
549 SYS_PORT_TRACING_FUNC(k_thread, sched_pend, thread); in add_to_waitq_locked()
552 thread->base.pended_on = wait_q; in add_to_waitq_locked()
553 _priq_wait_add(&wait_q->waitq, thread); in add_to_waitq_locked()
557 static void add_thread_timeout(struct k_thread *thread, k_timeout_t timeout) in add_thread_timeout() argument
560 z_add_thread_timeout(thread, timeout); in add_thread_timeout()
564 static void pend_locked(struct k_thread *thread, _wait_q_t *wait_q, in pend_locked() argument
570 add_to_waitq_locked(thread, wait_q); in pend_locked()
571 add_thread_timeout(thread, timeout); in pend_locked()
574 void z_pend_thread(struct k_thread *thread, _wait_q_t *wait_q, in z_pend_thread() argument
577 __ASSERT_NO_MSG(thread == _current || is_thread_dummy(thread)); in z_pend_thread()
579 pend_locked(thread, wait_q, timeout); in z_pend_thread()
583 void z_unpend_thread_no_timeout(struct k_thread *thread) in z_unpend_thread_no_timeout() argument
586 if (thread->base.pended_on != NULL) { in z_unpend_thread_no_timeout()
587 unpend_thread_no_timeout(thread); in z_unpend_thread_no_timeout()
592 void z_sched_wake_thread(struct k_thread *thread, bool is_timeout) in z_sched_wake_thread() argument
595 bool killed = (thread->base.thread_state & in z_sched_wake_thread()
599 bool do_nothing = thread->no_wake_on_timeout && is_timeout; in z_sched_wake_thread()
601 thread->no_wake_on_timeout = false; in z_sched_wake_thread()
610 if (thread->base.pended_on != NULL) { in z_sched_wake_thread()
611 unpend_thread_no_timeout(thread); in z_sched_wake_thread()
613 z_mark_thread_as_not_sleeping(thread); in z_sched_wake_thread()
614 ready_thread(thread); in z_sched_wake_thread()
624 struct k_thread *thread = CONTAINER_OF(timeout, in z_thread_timeout() local
627 z_sched_wake_thread(thread, true); in z_thread_timeout()
655 struct k_thread *thread = NULL; in z_unpend1_no_timeout() local
658 thread = _priq_wait_best(&wait_q->waitq); in z_unpend1_no_timeout()
660 if (thread != NULL) { in z_unpend1_no_timeout()
661 unpend_thread_no_timeout(thread); in z_unpend1_no_timeout()
665 return thread; in z_unpend1_no_timeout()
668 void z_unpend_thread(struct k_thread *thread) in z_unpend_thread() argument
670 z_unpend_thread_no_timeout(thread); in z_unpend_thread()
671 z_abort_thread_timeout(thread); in z_unpend_thread()
677 bool z_thread_prio_set(struct k_thread *thread, int prio) in z_thread_prio_set() argument
680 int old_prio = thread->base.prio; in z_thread_prio_set()
683 need_sched = z_is_thread_ready(thread); in z_thread_prio_set()
686 if (!IS_ENABLED(CONFIG_SMP) || z_is_thread_queued(thread)) { in z_thread_prio_set()
687 dequeue_thread(thread); in z_thread_prio_set()
688 thread->base.prio = prio; in z_thread_prio_set()
689 queue_thread(thread); in z_thread_prio_set()
692 flag_ipi(ipi_mask_create(thread)); in z_thread_prio_set()
702 thread->base.prio = prio; in z_thread_prio_set()
706 cpu = thread_active_elsewhere(thread); in z_thread_prio_set()
713 } else if (z_is_thread_pending(thread)) { in z_thread_prio_set()
718 _wait_q_t *wait_q = pended_on_thread(thread); in z_thread_prio_set()
720 _priq_wait_remove(&wait_q->waitq, thread); in z_thread_prio_set()
721 thread->base.prio = prio; in z_thread_prio_set()
722 _priq_wait_add(&wait_q->waitq, thread); in z_thread_prio_set()
724 thread->base.prio = prio; in z_thread_prio_set()
728 SYS_PORT_TRACING_OBJ_FUNC(k_thread, sched_priority_set, thread, prio); in z_thread_prio_set()
953 struct k_thread *thread; in z_unpend_all() local
955 for (thread = z_waitq_head(wait_q); thread != NULL; thread = z_waitq_head(wait_q)) { in z_unpend_all()
956 z_unpend_thread(thread); in z_unpend_all()
957 z_ready_thread(thread); in z_unpend_all()
980 void z_impl_k_thread_priority_set(k_tid_t thread, int prio) in z_impl_k_thread_priority_set() argument
988 bool need_sched = z_thread_prio_set((struct k_thread *)thread, prio); in z_impl_k_thread_priority_set()
997 static inline void z_vrfy_k_thread_priority_set(k_tid_t thread, int prio) in z_vrfy_k_thread_priority_set() argument
999 K_OOPS(K_SYSCALL_OBJ(thread, K_OBJ_THREAD)); in z_vrfy_k_thread_priority_set()
1003 K_OOPS(K_SYSCALL_VERIFY_MSG((int8_t)prio >= thread->base.prio, in z_vrfy_k_thread_priority_set()
1005 prio, thread->base.prio)); in z_vrfy_k_thread_priority_set()
1007 z_impl_k_thread_priority_set(thread, prio); in z_vrfy_k_thread_priority_set()
1015 struct k_thread *thread = tid; in z_impl_k_thread_absolute_deadline_set() local
1024 if (z_is_thread_queued(thread)) { in z_impl_k_thread_absolute_deadline_set()
1025 dequeue_thread(thread); in z_impl_k_thread_absolute_deadline_set()
1026 thread->base.prio_deadline = deadline; in z_impl_k_thread_absolute_deadline_set()
1027 queue_thread(thread); in z_impl_k_thread_absolute_deadline_set()
1029 thread->base.prio_deadline = deadline; in z_impl_k_thread_absolute_deadline_set()
1047 struct k_thread *thread = tid; in z_vrfy_k_thread_absolute_deadline_set() local
1049 K_OOPS(K_SYSCALL_OBJ(thread, K_OBJ_THREAD)); in z_vrfy_k_thread_absolute_deadline_set()
1051 z_impl_k_thread_absolute_deadline_set((k_tid_t)thread, deadline); in z_vrfy_k_thread_absolute_deadline_set()
1057 struct k_thread *thread = tid; in z_vrfy_k_thread_deadline_set() local
1059 K_OOPS(K_SYSCALL_OBJ(thread, K_OBJ_THREAD)); in z_vrfy_k_thread_deadline_set()
1064 z_impl_k_thread_deadline_set((k_tid_t)thread, deadline); in z_vrfy_k_thread_deadline_set()
1209 void z_impl_k_wakeup(k_tid_t thread) in z_impl_k_wakeup() argument
1211 SYS_PORT_TRACING_OBJ_FUNC(k_thread, wakeup, thread); in z_impl_k_wakeup()
1215 if (z_is_thread_sleeping(thread)) { in z_impl_k_wakeup()
1216 z_abort_thread_timeout(thread); in z_impl_k_wakeup()
1217 z_mark_thread_as_not_sleeping(thread); in z_impl_k_wakeup()
1218 ready_thread(thread); in z_impl_k_wakeup()
1226 static inline void z_vrfy_k_wakeup(k_tid_t thread) in z_vrfy_k_wakeup() argument
1228 K_OOPS(K_SYSCALL_OBJ(thread, K_OBJ_THREAD)); in z_vrfy_k_wakeup()
1229 z_impl_k_wakeup(thread); in z_vrfy_k_wakeup()
1249 struct k_thread *thread; in unpend_all() local
1251 for (thread = z_waitq_head(wait_q); thread != NULL; thread = z_waitq_head(wait_q)) { in unpend_all()
1252 unpend_thread_no_timeout(thread); in unpend_all()
1253 z_abort_thread_timeout(thread); in unpend_all()
1254 arch_thread_return_value_set(thread, 0); in unpend_all()
1255 ready_thread(thread); in unpend_all()
1260 extern void thread_abort_hook(struct k_thread *thread);
1271 static ALWAYS_INLINE void halt_thread(struct k_thread *thread, uint8_t new_state) in halt_thread() argument
1278 if ((thread->base.thread_state & new_state) == 0U) { in halt_thread()
1279 thread->base.thread_state |= new_state; in halt_thread()
1280 if (z_is_thread_queued(thread)) { in halt_thread()
1281 dequeue_thread(thread); in halt_thread()
1285 if (thread->base.pended_on != NULL) { in halt_thread()
1286 unpend_thread_no_timeout(thread); in halt_thread()
1288 z_abort_thread_timeout(thread); in halt_thread()
1289 unpend_all(&thread->join_queue); in halt_thread()
1297 if (thread == _current && arch_is_in_isr()) { in halt_thread()
1302 unpend_all(&thread->halt_queue); in halt_thread()
1307 clear_halting(thread); in halt_thread()
1311 arch_coprocessors_disable(thread); in halt_thread()
1313 SYS_PORT_TRACING_FUNC(k_thread, sched_abort, thread); in halt_thread()
1315 z_thread_monitor_exit(thread); in halt_thread()
1317 thread_abort_hook(thread); in halt_thread()
1322 k_obj_core_stats_deregister(K_OBJ_CORE(thread)); in halt_thread()
1324 k_obj_core_unlink(K_OBJ_CORE(thread)); in halt_thread()
1328 z_mem_domain_exit_thread(thread); in halt_thread()
1329 k_thread_perms_all_clear(thread); in halt_thread()
1330 k_object_uninit(thread->stack_obj); in halt_thread()
1331 k_object_uninit(thread); in halt_thread()
1335 k_thread_abort_cleanup(thread); in halt_thread()
1360 clear_halting(thread); in halt_thread()
1364 void z_thread_abort(struct k_thread *thread) in z_thread_abort() argument
1366 bool essential = z_is_thread_essential(thread); in z_thread_abort()
1369 if (z_is_thread_dead(thread)) { in z_thread_abort()
1374 z_thread_halt(thread, key, true); in z_thread_abort()
1377 __ASSERT(!essential, "aborted essential thread %p", thread); in z_thread_abort()
1383 void z_impl_k_thread_abort(k_tid_t thread) in z_impl_k_thread_abort() argument
1385 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_thread, abort, thread); in z_impl_k_thread_abort()
1387 z_thread_abort(thread); in z_impl_k_thread_abort()
1389 __ASSERT_NO_MSG(z_is_thread_dead(thread)); in z_impl_k_thread_abort()
1391 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_thread, abort, thread); in z_impl_k_thread_abort()
1395 int z_impl_k_thread_join(struct k_thread *thread, k_timeout_t timeout) in z_impl_k_thread_join() argument
1400 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_thread, join, thread, timeout); in z_impl_k_thread_join()
1402 if (z_is_thread_dead(thread)) { in z_impl_k_thread_join()
1403 z_sched_switch_spin(thread); in z_impl_k_thread_join()
1407 } else if ((thread == _current) || in z_impl_k_thread_join()
1408 (thread->base.pended_on == &_current->join_queue)) { in z_impl_k_thread_join()
1412 add_to_waitq_locked(_current, &thread->join_queue); in z_impl_k_thread_join()
1415 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_thread, join, thread, timeout); in z_impl_k_thread_join()
1417 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_thread, join, thread, timeout, ret); in z_impl_k_thread_join()
1422 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_thread, join, thread, timeout, ret); in z_impl_k_thread_join()
1437 static bool thread_obj_validate(struct k_thread *thread) in thread_obj_validate() argument
1439 struct k_object *ko = k_object_find(thread); in thread_obj_validate()
1449 k_object_dump_error(ret, thread, ko, K_OBJ_THREAD); in thread_obj_validate()
1456 static inline int z_vrfy_k_thread_join(struct k_thread *thread, in z_vrfy_k_thread_join() argument
1459 if (thread_obj_validate(thread)) { in z_vrfy_k_thread_join()
1463 return z_impl_k_thread_join(thread, timeout); in z_vrfy_k_thread_join()
1467 static inline void z_vrfy_k_thread_abort(k_tid_t thread) in z_vrfy_k_thread_abort() argument
1469 if (thread_obj_validate(thread)) { in z_vrfy_k_thread_abort()
1473 K_OOPS(K_SYSCALL_VERIFY_MSG(!z_is_thread_essential(thread), in z_vrfy_k_thread_abort()
1474 "aborting essential thread %p", thread)); in z_vrfy_k_thread_abort()
1476 z_impl_k_thread_abort((struct k_thread *)thread); in z_vrfy_k_thread_abort()
1486 struct k_thread *thread; in z_sched_wake() local
1490 thread = _priq_wait_best(&wait_q->waitq); in z_sched_wake()
1492 if (thread != NULL) { in z_sched_wake()
1493 z_thread_return_value_set_with_data(thread, in z_sched_wake()
1496 unpend_thread_no_timeout(thread); in z_sched_wake()
1497 z_abort_thread_timeout(thread); in z_sched_wake()
1498 ready_thread(thread); in z_sched_wake()
1520 struct k_thread *thread; in z_sched_waitq_walk() local
1524 _WAIT_Q_FOR_EACH(wait_q, thread) { in z_sched_waitq_walk()
1532 status = func(thread, data); in z_sched_waitq_walk()
1545 void z_unready_thread(struct k_thread *thread) in z_unready_thread() argument
1548 unready_thread(thread); in z_unready_thread()