306 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue_pi()  argument
308 	struct rb_node **link = &task->pi_waiters.rb_root.rb_node;  in rt_mutex_enqueue_pi()
325 	rb_insert_color_cached(&waiter->pi_tree_entry, &task->pi_waiters, leftmost);  in rt_mutex_enqueue_pi()
329 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue_pi()  argument
334 	rb_erase_cached(&waiter->pi_tree_entry, &task->pi_waiters);  in rt_mutex_dequeue_pi()
345 		pi_task = task_top_pi_waiter(p)->task;  in rt_mutex_adjust_prio()
449 static int rt_mutex_adjust_prio_chain(struct task_struct *task,  in rt_mutex_adjust_prio_chain()  argument
488 		put_task_struct(task);  in rt_mutex_adjust_prio_chain()
503 	raw_spin_lock_irq(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
508 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_prio_chain()
547 		if (!task_has_pi_waiters(task))  in rt_mutex_adjust_prio_chain()
555 		if (top_waiter != task_top_pi_waiter(task)) {  in rt_mutex_adjust_prio_chain()
570 	if (rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {  in rt_mutex_adjust_prio_chain()
587 		raw_spin_unlock_irq(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
618 		raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
619 		put_task_struct(task);  in rt_mutex_adjust_prio_chain()
631 		task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
632 		raw_spin_lock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
640 		next_lock = task_blocked_on_lock(task);  in rt_mutex_adjust_prio_chain()
647 		raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
682 	waiter->prio = task->prio;  in rt_mutex_adjust_prio_chain()
683 	waiter->deadline = task->dl.deadline;  in rt_mutex_adjust_prio_chain()
688 	raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
689 	put_task_struct(task);  in rt_mutex_adjust_prio_chain()
705 			wake_up_process(rt_mutex_top_waiter(lock)->task);  in rt_mutex_adjust_prio_chain()
711 	task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
712 	raw_spin_lock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
722 		rt_mutex_dequeue_pi(task, prerequeue_top_waiter);  in rt_mutex_adjust_prio_chain()
723 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
724 		rt_mutex_adjust_prio(task);  in rt_mutex_adjust_prio_chain()
737 		rt_mutex_dequeue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
739 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
740 		rt_mutex_adjust_prio(task);  in rt_mutex_adjust_prio_chain()
758 	next_lock = task_blocked_on_lock(task);  in rt_mutex_adjust_prio_chain()
766 	raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
790 	raw_spin_unlock_irq(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
792 	put_task_struct(task);  in rt_mutex_adjust_prio_chain()
807 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
871 			if (!rt_mutex_waiter_less(task_to_waiter(task),  in try_to_take_rt_mutex()
897 	raw_spin_lock(&task->pi_lock);  in try_to_take_rt_mutex()
898 	task->pi_blocked_on = NULL;  in try_to_take_rt_mutex()
905 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
906 	raw_spin_unlock(&task->pi_lock);  in try_to_take_rt_mutex()
916 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
930 				   struct task_struct *task,  in task_blocks_on_rt_mutex()  argument
949 	if (owner == task)  in task_blocks_on_rt_mutex()
952 	raw_spin_lock(&task->pi_lock);  in task_blocks_on_rt_mutex()
953 	waiter->task = task;  in task_blocks_on_rt_mutex()
955 	waiter->prio = task->prio;  in task_blocks_on_rt_mutex()
956 	waiter->deadline = task->dl.deadline;  in task_blocks_on_rt_mutex()
963 	task->pi_blocked_on = waiter;  in task_blocks_on_rt_mutex()
965 	raw_spin_unlock(&task->pi_lock);  in task_blocks_on_rt_mutex()
1004 					 next_lock, waiter, task);  in task_blocks_on_rt_mutex()
1057 	wake_q_add(wake_q, waiter->task);  in mark_wakeup_next_waiter()
1125 void rt_mutex_adjust_pi(struct task_struct *task)  in rt_mutex_adjust_pi()  argument
1131 	raw_spin_lock_irqsave(&task->pi_lock, flags);  in rt_mutex_adjust_pi()
1133 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_pi()
1134 	if (!waiter || rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {  in rt_mutex_adjust_pi()
1135 		raw_spin_unlock_irqrestore(&task->pi_lock, flags);  in rt_mutex_adjust_pi()
1139 	raw_spin_unlock_irqrestore(&task->pi_lock, flags);  in rt_mutex_adjust_pi()
1142 	get_task_struct(task);  in rt_mutex_adjust_pi()
1144 	rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,  in rt_mutex_adjust_pi()
1145 				   next_lock, NULL, task);  in rt_mutex_adjust_pi()
1153 	waiter->task = NULL;  in rt_mutex_init_waiter()
1186 			if (timeout && !timeout->task)  in __rt_mutex_slowlock()
1749 			      struct task_struct *task)  in __rt_mutex_start_proxy_lock()  argument
1755 	if (try_to_take_rt_mutex(lock, task, NULL))  in __rt_mutex_start_proxy_lock()
1759 	ret = task_blocks_on_rt_mutex(lock, waiter, task,  in __rt_mutex_start_proxy_lock()
1798 			      struct task_struct *task)  in rt_mutex_start_proxy_lock()  argument
1803 	ret = __rt_mutex_start_proxy_lock(lock, waiter, task);  in rt_mutex_start_proxy_lock()
1828 	return rt_mutex_top_waiter(lock)->task;  in rt_mutex_next_owner()