33 static inline int __ww_mutex_add_waiter(struct rt_mutex_waiter *waiter,  in __ww_mutex_add_waiter()  argument
51 					struct rt_mutex_waiter *waiter,  in __ww_mutex_check_kill()  argument
298 waiter_update_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)  in waiter_update_prio()  argument
300 	waiter->prio = __waiter_prio(task);  in waiter_update_prio()
301 	waiter->deadline = task->dl.deadline;  in waiter_update_prio()
346 static inline bool rt_mutex_steal(struct rt_mutex_waiter *waiter,  in rt_mutex_steal()  argument
349 	if (rt_mutex_waiter_less(waiter, top_waiter))  in rt_mutex_steal()
357 	if (rt_prio(waiter->prio) || dl_prio(waiter->prio))  in rt_mutex_steal()
360 	return rt_mutex_waiter_equal(waiter, top_waiter);  in rt_mutex_steal()
396 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
398 	rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);  in rt_mutex_enqueue()
402 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
404 	if (RB_EMPTY_NODE(&waiter->tree_entry))  in rt_mutex_dequeue()
407 	rb_erase_cached(&waiter->tree_entry, &lock->waiters);  in rt_mutex_dequeue()
408 	RB_CLEAR_NODE(&waiter->tree_entry);  in rt_mutex_dequeue()
421 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue_pi()  argument
423 	rb_add_cached(&waiter->pi_tree_entry, &task->pi_waiters, __pi_waiter_less);  in rt_mutex_enqueue_pi()
427 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue_pi()  argument
429 	if (RB_EMPTY_NODE(&waiter->pi_tree_entry))  in rt_mutex_dequeue_pi()
432 	rb_erase_cached(&waiter->pi_tree_entry, &task->pi_waiters);  in rt_mutex_dequeue_pi()
433 	RB_CLEAR_NODE(&waiter->pi_tree_entry);  in rt_mutex_dequeue_pi()
491 rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,  in rt_mutex_cond_detect_deadlock()  argument
495 		return waiter != NULL;  in rt_mutex_cond_detect_deadlock()
574 	struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;  in rt_mutex_adjust_prio_chain()  local
626 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_prio_chain()
637 	if (!waiter)  in rt_mutex_adjust_prio_chain()
656 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
681 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && waiter->ww_ctx && detect_deadlock)  in rt_mutex_adjust_prio_chain()
713 	if (rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {  in rt_mutex_adjust_prio_chain()
723 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
819 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
837 	waiter_update_prio(waiter, task);  in rt_mutex_adjust_prio_chain()
839 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
859 			wake_up_state(waiter->task, waiter->wake_state);  in rt_mutex_adjust_prio_chain()
869 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
877 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
880 	} else if (prerequeue_top_waiter == waiter) {  in rt_mutex_adjust_prio_chain()
891 		rt_mutex_dequeue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
892 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
893 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
938 	if (!detect_deadlock && waiter != top_waiter)  in rt_mutex_adjust_prio_chain()
963 		     struct rt_mutex_waiter *waiter)  in try_to_take_rt_mutex()  argument
997 	if (waiter) {  in try_to_take_rt_mutex()
1004 		if (waiter == top_waiter || rt_mutex_steal(waiter, top_waiter)) {  in try_to_take_rt_mutex()
1009 			rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
1079 					   struct rt_mutex_waiter *waiter,  in task_blocks_on_rt_mutex()  argument
1085 	struct rt_mutex_waiter *top_waiter = waiter;  in task_blocks_on_rt_mutex()
1104 	waiter->task = task;  in task_blocks_on_rt_mutex()
1105 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
1106 	waiter_update_prio(waiter, task);  in task_blocks_on_rt_mutex()
1111 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
1113 	task->pi_blocked_on = waiter;  in task_blocks_on_rt_mutex()
1122 		res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx);  in task_blocks_on_rt_mutex()
1125 			rt_mutex_dequeue(lock, waiter);  in task_blocks_on_rt_mutex()
1136 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
1138 		rt_mutex_enqueue_pi(owner, waiter);  in task_blocks_on_rt_mutex()
1143 	} else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {  in task_blocks_on_rt_mutex()
1169 					 next_lock, waiter, task);  in task_blocks_on_rt_mutex()
1185 	struct rt_mutex_waiter *waiter;  in mark_wakeup_next_waiter()  local
1189 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1198 	rt_mutex_dequeue_pi(current, waiter);  in mark_wakeup_next_waiter()
1222 	rt_mutex_wake_q_add(wqh, waiter);  in mark_wakeup_next_waiter()
1350 				  struct rt_mutex_waiter *waiter,  in rtmutex_spin_on_owner()  argument
1376 		    rt_mutex_waiter_is_top_waiter(lock, waiter) ||  in rtmutex_spin_on_owner()
1388 				  struct rt_mutex_waiter *waiter,  in rtmutex_spin_on_owner()  argument
1409 				  struct rt_mutex_waiter *waiter)  in remove_waiter()  argument
1411 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1418 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1431 	rt_mutex_dequeue_pi(owner, waiter);  in remove_waiter()
1476 					   struct rt_mutex_waiter *waiter)  in rt_mutex_slowlock_block()  argument
1484 		if (try_to_take_rt_mutex(lock, current, waiter))  in rt_mutex_slowlock_block()
1497 			ret = __ww_mutex_check_kill(rtm, waiter, ww_ctx);  in rt_mutex_slowlock_block()
1502 		if (waiter == rt_mutex_top_waiter(lock))  in rt_mutex_slowlock_block()
1508 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))  in rt_mutex_slowlock_block()
1554 				       struct rt_mutex_waiter *waiter)  in __rt_mutex_slowlock()  argument
1573 	ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);  in __rt_mutex_slowlock()
1575 		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);  in __rt_mutex_slowlock()
1586 		remove_waiter(lock, waiter);  in __rt_mutex_slowlock()
1587 		rt_mutex_handle_deadlock(ret, chwalk, waiter);  in __rt_mutex_slowlock()
1602 	struct rt_mutex_waiter waiter;  in __rt_mutex_slowlock_locked()  local
1605 	rt_mutex_init_waiter(&waiter);  in __rt_mutex_slowlock_locked()
1606 	waiter.ww_ctx = ww_ctx;  in __rt_mutex_slowlock_locked()
1609 				  &waiter);  in __rt_mutex_slowlock_locked()
1611 	debug_rt_mutex_free_waiter(&waiter);  in __rt_mutex_slowlock_locked()
1664 	struct rt_mutex_waiter waiter;  in rtlock_slowlock_locked()  local
1672 	rt_mutex_init_rtlock_waiter(&waiter);  in rtlock_slowlock_locked()
1677 	task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);  in rtlock_slowlock_locked()
1681 		if (try_to_take_rt_mutex(lock, current, &waiter))  in rtlock_slowlock_locked()
1684 		if (&waiter == rt_mutex_top_waiter(lock))  in rtlock_slowlock_locked()
1690 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))  in rtlock_slowlock_locked()
1705 	debug_rt_mutex_free_waiter(&waiter);  in rtlock_slowlock_locked()