272 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
282 		if (rt_mutex_waiter_less(waiter, entry)) {  in rt_mutex_enqueue()
290 	rb_link_node(&waiter->tree_entry, parent, link);  in rt_mutex_enqueue()
291 	rb_insert_color_cached(&waiter->tree_entry, &lock->waiters, leftmost);  in rt_mutex_enqueue()
295 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
297 	if (RB_EMPTY_NODE(&waiter->tree_entry))  in rt_mutex_dequeue()
300 	rb_erase_cached(&waiter->tree_entry, &lock->waiters);  in rt_mutex_dequeue()
301 	RB_CLEAR_NODE(&waiter->tree_entry);  in rt_mutex_dequeue()
305 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue_pi()  argument
315 		if (rt_mutex_waiter_less(waiter, entry)) {  in rt_mutex_enqueue_pi()
323 	rb_link_node(&waiter->pi_tree_entry, parent, link);  in rt_mutex_enqueue_pi()
324 	rb_insert_color_cached(&waiter->pi_tree_entry, &task->pi_waiters, leftmost);  in rt_mutex_enqueue_pi()
328 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue_pi()  argument
330 	if (RB_EMPTY_NODE(&waiter->pi_tree_entry))  in rt_mutex_dequeue_pi()
333 	rb_erase_cached(&waiter->pi_tree_entry, &task->pi_waiters);  in rt_mutex_dequeue_pi()
334 	RB_CLEAR_NODE(&waiter->pi_tree_entry);  in rt_mutex_dequeue_pi()
362 static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,  in rt_mutex_cond_detect_deadlock()  argument
372 	return debug_rt_mutex_detect_deadlock(waiter, chwalk);  in rt_mutex_cond_detect_deadlock()
455 	struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;  in rt_mutex_adjust_prio_chain()  local
507 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_prio_chain()
518 	if (!waiter)  in rt_mutex_adjust_prio_chain()
537 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
569 	if (rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {  in rt_mutex_adjust_prio_chain()
579 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
664 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
682 	waiter->prio = task->prio;  in rt_mutex_adjust_prio_chain()
683 	waiter->deadline = task->dl.deadline;  in rt_mutex_adjust_prio_chain()
685 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
716 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
724 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
727 	} else if (prerequeue_top_waiter == waiter) {  in rt_mutex_adjust_prio_chain()
738 		rt_mutex_dequeue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
739 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
740 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
785 	if (!detect_deadlock && waiter != top_waiter)  in rt_mutex_adjust_prio_chain()
809 				struct rt_mutex_waiter *waiter)  in try_to_take_rt_mutex()  argument
843 	if (waiter) {  in try_to_take_rt_mutex()
848 		if (waiter != rt_mutex_top_waiter(lock))  in try_to_take_rt_mutex()
855 		rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
930 				   struct rt_mutex_waiter *waiter,  in task_blocks_on_rt_mutex()  argument
935 	struct rt_mutex_waiter *top_waiter = waiter;  in task_blocks_on_rt_mutex()
954 	waiter->task = task;  in task_blocks_on_rt_mutex()
955 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
956 	waiter->prio = task->prio;  in task_blocks_on_rt_mutex()
957 	waiter->deadline = task->dl.deadline;  in task_blocks_on_rt_mutex()
962 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
964 	task->pi_blocked_on = waiter;  in task_blocks_on_rt_mutex()
972 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
974 		rt_mutex_enqueue_pi(owner, waiter);  in task_blocks_on_rt_mutex()
979 	} else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {  in task_blocks_on_rt_mutex()
1005 					 next_lock, waiter, task);  in task_blocks_on_rt_mutex()
1021 	struct rt_mutex_waiter *waiter;  in mark_wakeup_next_waiter()  local
1025 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1034 	rt_mutex_dequeue_pi(current, waiter);  in mark_wakeup_next_waiter()
1058 	wake_q_add(wake_q, waiter->task);  in mark_wakeup_next_waiter()
1069 			  struct rt_mutex_waiter *waiter)  in remove_waiter()  argument
1071 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1078 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1091 	rt_mutex_dequeue_pi(owner, waiter);  in remove_waiter()
1128 	struct rt_mutex_waiter *waiter;  in rt_mutex_adjust_pi()  local
1134 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_pi()
1135 	if (!waiter || rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {  in rt_mutex_adjust_pi()
1139 	next_lock = waiter->lock;  in rt_mutex_adjust_pi()
1149 void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)  in rt_mutex_init_waiter()  argument
1151 	debug_rt_mutex_init_waiter(waiter);  in rt_mutex_init_waiter()
1152 	RB_CLEAR_NODE(&waiter->pi_tree_entry);  in rt_mutex_init_waiter()
1153 	RB_CLEAR_NODE(&waiter->tree_entry);  in rt_mutex_init_waiter()
1154 	waiter->task = NULL;  in rt_mutex_init_waiter()
1170 		    struct rt_mutex_waiter *waiter)  in __rt_mutex_slowlock()  argument
1176 		if (try_to_take_rt_mutex(lock, current, waiter))  in __rt_mutex_slowlock()
1195 		debug_rt_mutex_print_deadlock(waiter);  in __rt_mutex_slowlock()
1235 	struct rt_mutex_waiter waiter;  in rt_mutex_slowlock()  local
1239 	rt_mutex_init_waiter(&waiter);  in rt_mutex_slowlock()
1263 	ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);  in rt_mutex_slowlock()
1267 		ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);  in rt_mutex_slowlock()
1271 		remove_waiter(lock, &waiter);  in rt_mutex_slowlock()
1272 		rt_mutex_handle_deadlock(ret, chwalk, &waiter);  in rt_mutex_slowlock()
1287 	debug_rt_mutex_free_waiter(&waiter);  in rt_mutex_slowlock()
1730 			      struct rt_mutex_waiter *waiter,  in __rt_mutex_start_proxy_lock()  argument
1739 	ret = task_blocks_on_rt_mutex(lock, waiter, task,  in __rt_mutex_start_proxy_lock()
1753 		remove_waiter(lock, waiter);  in __rt_mutex_start_proxy_lock()
1755 	debug_rt_mutex_print_deadlock(waiter);  in __rt_mutex_start_proxy_lock()
1774 			      struct rt_mutex_waiter *waiter,  in rt_mutex_start_proxy_lock()  argument
1780 	ret = __rt_mutex_start_proxy_lock(lock, waiter, task);  in rt_mutex_start_proxy_lock()
1825 			       struct rt_mutex_waiter *waiter)  in rt_mutex_wait_proxy_lock()  argument
1832 	ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);  in rt_mutex_wait_proxy_lock()
1863 				 struct rt_mutex_waiter *waiter)  in rt_mutex_cleanup_proxy_lock()  argument
1879 	try_to_take_rt_mutex(lock, current, waiter);  in rt_mutex_cleanup_proxy_lock()
1885 		remove_waiter(lock, waiter);  in rt_mutex_cleanup_proxy_lock()