220 	struct futex_pi_state *pi_state;  member
792 	struct futex_pi_state *pi_state;  in refill_pi_state_cache()  local
797 	pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);  in refill_pi_state_cache()
799 	if (!pi_state)  in refill_pi_state_cache()
802 	INIT_LIST_HEAD(&pi_state->list);  in refill_pi_state_cache()
804 	pi_state->owner = NULL;  in refill_pi_state_cache()
805 	refcount_set(&pi_state->refcount, 1);  in refill_pi_state_cache()
806 	pi_state->key = FUTEX_KEY_INIT;  in refill_pi_state_cache()
808 	current->pi_state_cache = pi_state;  in refill_pi_state_cache()
815 	struct futex_pi_state *pi_state = current->pi_state_cache;  in alloc_pi_state()  local
817 	WARN_ON(!pi_state);  in alloc_pi_state()
820 	return pi_state;  in alloc_pi_state()
823 static void pi_state_update_owner(struct futex_pi_state *pi_state,  in pi_state_update_owner()  argument
826 	struct task_struct *old_owner = pi_state->owner;  in pi_state_update_owner()
828 	lockdep_assert_held(&pi_state->pi_mutex.wait_lock);  in pi_state_update_owner()
832 		WARN_ON(list_empty(&pi_state->list));  in pi_state_update_owner()
833 		list_del_init(&pi_state->list);  in pi_state_update_owner()
839 		WARN_ON(!list_empty(&pi_state->list));  in pi_state_update_owner()
840 		list_add(&pi_state->list, &new_owner->pi_state_list);  in pi_state_update_owner()
841 		pi_state->owner = new_owner;  in pi_state_update_owner()
846 static void get_pi_state(struct futex_pi_state *pi_state)  in get_pi_state()  argument
848 	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));  in get_pi_state()
855 static void put_pi_state(struct futex_pi_state *pi_state)  in put_pi_state()  argument
857 	if (!pi_state)  in put_pi_state()
860 	if (!refcount_dec_and_test(&pi_state->refcount))  in put_pi_state()
867 	if (pi_state->owner) {  in put_pi_state()
870 		raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
871 		pi_state_update_owner(pi_state, NULL);  in put_pi_state()
872 		rt_mutex_proxy_unlock(&pi_state->pi_mutex);  in put_pi_state()
873 		raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
877 		kfree(pi_state);  in put_pi_state()
884 		pi_state->owner = NULL;  in put_pi_state()
885 		refcount_set(&pi_state->refcount, 1);  in put_pi_state()
886 		current->pi_state_cache = pi_state;  in put_pi_state()
900 	struct futex_pi_state *pi_state;  in exit_pi_state_list()  local
914 		pi_state = list_entry(next, struct futex_pi_state, list);  in exit_pi_state_list()
915 		key = pi_state->key;  in exit_pi_state_list()
928 		if (!refcount_inc_not_zero(&pi_state->refcount)) {  in exit_pi_state_list()
937 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
945 			raw_spin_unlock(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
947 			put_pi_state(pi_state);  in exit_pi_state_list()
951 		WARN_ON(pi_state->owner != curr);  in exit_pi_state_list()
952 		WARN_ON(list_empty(&pi_state->list));  in exit_pi_state_list()
953 		list_del_init(&pi_state->list);  in exit_pi_state_list()
954 		pi_state->owner = NULL;  in exit_pi_state_list()
957 		raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
960 		rt_mutex_futex_unlock(&pi_state->pi_mutex);  in exit_pi_state_list()
961 		put_pi_state(pi_state);  in exit_pi_state_list()
1062 			      struct futex_pi_state *pi_state,  in attach_to_pi_state()  argument
1072 	if (unlikely(!pi_state))  in attach_to_pi_state()
1087 	WARN_ON(!refcount_read(&pi_state->refcount));  in attach_to_pi_state()
1093 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1116 		if (!pi_state->owner) {  in attach_to_pi_state()
1144 		if (!pi_state->owner)  in attach_to_pi_state()
1153 	if (pid != task_pid_vnr(pi_state->owner))  in attach_to_pi_state()
1157 	get_pi_state(pi_state);  in attach_to_pi_state()
1158 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1159 	*ps = pi_state;  in attach_to_pi_state()
1175 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1275 	struct futex_pi_state *pi_state = alloc_pi_state();  in __attach_to_pi_owner()  local
1281 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);  in __attach_to_pi_owner()
1284 	pi_state->key = *key;  in __attach_to_pi_owner()
1286 	WARN_ON(!list_empty(&pi_state->list));  in __attach_to_pi_owner()
1287 	list_add(&pi_state->list, &p->pi_state_list);  in __attach_to_pi_owner()
1292 	pi_state->owner = p;  in __attach_to_pi_owner()
1294 	*ps = pi_state;  in __attach_to_pi_owner()
1440 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
1529 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in mark_wake_futex()
1553 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)  in wake_futex_pi()  argument
1562 	top_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);  in wake_futex_pi()
1610 		pi_state_update_owner(pi_state, new_owner);  in wake_futex_pi()
1611 		postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh);  in wake_futex_pi()
1615 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1676 			if (this->pi_state || this->rt_waiter) {  in futex_wake()
1797 			if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1811 				if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1856 					    struct futex_pi_state *pi_state)  in futex_requeue_pi_prepare()  argument
1884 	q->pi_state = pi_state;  in futex_requeue_pi_prepare()
2063 	if (!top_waiter->rt_waiter || top_waiter->pi_state)  in futex_proxy_trylock_atomic()
2133 	struct futex_pi_state *pi_state = NULL;  in futex_requeue()  local
2249 						 &key2, &pi_state,  in futex_requeue()
2350 		    this->pi_state) {  in futex_requeue()
2378 		get_pi_state(pi_state);  in futex_requeue()
2381 		if (!futex_requeue_pi_prepare(this, pi_state)) {  in futex_requeue()
2387 			put_pi_state(pi_state);  in futex_requeue()
2391 		ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,  in futex_requeue()
2419 			this->pi_state = NULL;  in futex_requeue()
2420 			put_pi_state(pi_state);  in futex_requeue()
2434 	put_pi_state(pi_state);  in futex_requeue()
2558 		BUG_ON(q->pi_state);  in unqueue_me()
2575 	BUG_ON(!q->pi_state);  in unqueue_me_pi()
2576 	put_pi_state(q->pi_state);  in unqueue_me_pi()
2577 	q->pi_state = NULL;  in unqueue_me_pi()
2583 	struct futex_pi_state *pi_state = q->pi_state;  in __fixup_pi_state_owner()  local
2588 	oldowner = pi_state->owner;  in __fixup_pi_state_owner()
2623 		if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {  in __fixup_pi_state_owner()
2632 		newowner = rt_mutex_owner(&pi_state->pi_mutex);  in __fixup_pi_state_owner()
2659 	if (!pi_state->owner)  in __fixup_pi_state_owner()
2682 	pi_state_update_owner(pi_state, newowner);  in __fixup_pi_state_owner()
2700 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in __fixup_pi_state_owner()
2719 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in __fixup_pi_state_owner()
2724 	if (pi_state->owner != oldowner)  in __fixup_pi_state_owner()
2747 	pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex));  in __fixup_pi_state_owner()
2755 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()  local
2760 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2762 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2794 		if (q->pi_state->owner != current)  in fixup_owner()
2807 	if (q->pi_state->owner == current)  in fixup_owner()
2814 	if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current))  in fixup_owner()
3044 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,  in futex_lock_pi()
3080 	WARN_ON(!q.pi_state);  in futex_lock_pi()
3088 		ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
3109 	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
3116 	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);  in futex_lock_pi()
3117 	raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
3128 	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);  in futex_lock_pi()
3141 	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))  in futex_lock_pi()
3223 		struct futex_pi_state *pi_state = top_waiter->pi_state;  in futex_unlock_pi()  local
3226 		if (!pi_state)  in futex_unlock_pi()
3233 		if (pi_state->owner != current)  in futex_unlock_pi()
3236 		get_pi_state(pi_state);  in futex_unlock_pi()
3247 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in futex_unlock_pi()
3251 		ret = wake_futex_pi(uaddr, uval, pi_state);  in futex_unlock_pi()
3253 		put_pi_state(pi_state);  in futex_unlock_pi()
3475 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
3482 			put_pi_state(q.pi_state);  in futex_wait_requeue_pi()
3494 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()