304 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue_pi()  argument
306 	struct rb_node **link = &task->pi_waiters.rb_root.rb_node;  in rt_mutex_enqueue_pi()
323 	rb_insert_color_cached(&waiter->pi_tree_entry, &task->pi_waiters, leftmost);  in rt_mutex_enqueue_pi()
327 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue_pi()  argument
332 	rb_erase_cached(&waiter->pi_tree_entry, &task->pi_waiters);  in rt_mutex_dequeue_pi()
343 		pi_task = task_top_pi_waiter(p)->task;  in rt_mutex_adjust_prio()
447 static int rt_mutex_adjust_prio_chain(struct task_struct *task,  in rt_mutex_adjust_prio_chain()  argument
486 		put_task_struct(task);  in rt_mutex_adjust_prio_chain()
501 	raw_spin_lock_irq(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
506 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_prio_chain()
545 		if (!task_has_pi_waiters(task))  in rt_mutex_adjust_prio_chain()
553 		if (top_waiter != task_top_pi_waiter(task)) {  in rt_mutex_adjust_prio_chain()
568 	if (rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {  in rt_mutex_adjust_prio_chain()
585 		raw_spin_unlock_irq(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
616 		raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
617 		put_task_struct(task);  in rt_mutex_adjust_prio_chain()
629 		task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
630 		raw_spin_lock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
638 		next_lock = task_blocked_on_lock(task);  in rt_mutex_adjust_prio_chain()
645 		raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
680 	waiter->prio = task->prio;  in rt_mutex_adjust_prio_chain()
681 	waiter->deadline = task->dl.deadline;  in rt_mutex_adjust_prio_chain()
686 	raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
687 	put_task_struct(task);  in rt_mutex_adjust_prio_chain()
703 			wake_up_process(rt_mutex_top_waiter(lock)->task);  in rt_mutex_adjust_prio_chain()
709 	task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
710 	raw_spin_lock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
720 		rt_mutex_dequeue_pi(task, prerequeue_top_waiter);  in rt_mutex_adjust_prio_chain()
721 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
722 		rt_mutex_adjust_prio(task);  in rt_mutex_adjust_prio_chain()
735 		rt_mutex_dequeue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
737 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
738 		rt_mutex_adjust_prio(task);  in rt_mutex_adjust_prio_chain()
756 	next_lock = task_blocked_on_lock(task);  in rt_mutex_adjust_prio_chain()
764 	raw_spin_unlock(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
788 	raw_spin_unlock_irq(&task->pi_lock);  in rt_mutex_adjust_prio_chain()
790 	put_task_struct(task);  in rt_mutex_adjust_prio_chain()
805 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
869 			if (!rt_mutex_waiter_less(task_to_waiter(task),  in try_to_take_rt_mutex()
895 	raw_spin_lock(&task->pi_lock);  in try_to_take_rt_mutex()
896 	task->pi_blocked_on = NULL;  in try_to_take_rt_mutex()
903 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
904 	raw_spin_unlock(&task->pi_lock);  in try_to_take_rt_mutex()
914 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
928 				   struct task_struct *task,  in task_blocks_on_rt_mutex()  argument
947 	if (owner == task)  in task_blocks_on_rt_mutex()
950 	raw_spin_lock(&task->pi_lock);  in task_blocks_on_rt_mutex()
951 	waiter->task = task;  in task_blocks_on_rt_mutex()
953 	waiter->prio = task->prio;  in task_blocks_on_rt_mutex()
954 	waiter->deadline = task->dl.deadline;  in task_blocks_on_rt_mutex()
961 	task->pi_blocked_on = waiter;  in task_blocks_on_rt_mutex()
963 	raw_spin_unlock(&task->pi_lock);  in task_blocks_on_rt_mutex()
1002 					 next_lock, waiter, task);  in task_blocks_on_rt_mutex()
1055 	wake_q_add(wake_q, waiter->task);  in mark_wakeup_next_waiter()
1123 void rt_mutex_adjust_pi(struct task_struct *task)  in rt_mutex_adjust_pi()  argument
1129 	raw_spin_lock_irqsave(&task->pi_lock, flags);  in rt_mutex_adjust_pi()
1131 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_pi()
1132 	if (!waiter || rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {  in rt_mutex_adjust_pi()
1133 		raw_spin_unlock_irqrestore(&task->pi_lock, flags);  in rt_mutex_adjust_pi()
1137 	raw_spin_unlock_irqrestore(&task->pi_lock, flags);  in rt_mutex_adjust_pi()
1140 	get_task_struct(task);  in rt_mutex_adjust_pi()
1142 	rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,  in rt_mutex_adjust_pi()
1143 				   next_lock, NULL, task);  in rt_mutex_adjust_pi()
1151 	waiter->task = NULL;  in rt_mutex_init_waiter()
1184 			if (timeout && !timeout->task)  in __rt_mutex_slowlock()
1747 			      struct task_struct *task)  in __rt_mutex_start_proxy_lock()  argument
1753 	if (try_to_take_rt_mutex(lock, task, NULL))  in __rt_mutex_start_proxy_lock()
1757 	ret = task_blocks_on_rt_mutex(lock, waiter, task,  in __rt_mutex_start_proxy_lock()
1796 			      struct task_struct *task)  in rt_mutex_start_proxy_lock()  argument
1801 	ret = __rt_mutex_start_proxy_lock(lock, waiter, task);  in rt_mutex_start_proxy_lock()
1826 	return rt_mutex_top_waiter(lock)->task;  in rt_mutex_next_owner()