236 	struct futex_pi_state *pi_state;  member
806 	struct futex_pi_state *pi_state;  in refill_pi_state_cache()  local
811 	pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);  in refill_pi_state_cache()
813 	if (!pi_state)  in refill_pi_state_cache()
816 	INIT_LIST_HEAD(&pi_state->list);  in refill_pi_state_cache()
818 	pi_state->owner = NULL;  in refill_pi_state_cache()
819 	refcount_set(&pi_state->refcount, 1);  in refill_pi_state_cache()
820 	pi_state->key = FUTEX_KEY_INIT;  in refill_pi_state_cache()
822 	current->pi_state_cache = pi_state;  in refill_pi_state_cache()
829 	struct futex_pi_state *pi_state = current->pi_state_cache;  in alloc_pi_state()  local
831 	WARN_ON(!pi_state);  in alloc_pi_state()
834 	return pi_state;  in alloc_pi_state()
837 static void get_pi_state(struct futex_pi_state *pi_state)  in get_pi_state()  argument
839 	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));  in get_pi_state()
846 static void put_pi_state(struct futex_pi_state *pi_state)  in put_pi_state()  argument
848 	if (!pi_state)  in put_pi_state()
851 	if (!refcount_dec_and_test(&pi_state->refcount))  in put_pi_state()
858 	if (pi_state->owner) {  in put_pi_state()
861 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in put_pi_state()
862 		owner = pi_state->owner;  in put_pi_state()
865 			list_del_init(&pi_state->list);  in put_pi_state()
868 		rt_mutex_proxy_unlock(&pi_state->pi_mutex, owner);  in put_pi_state()
869 		raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in put_pi_state()
873 		kfree(pi_state);  in put_pi_state()
880 		pi_state->owner = NULL;  in put_pi_state()
881 		refcount_set(&pi_state->refcount, 1);  in put_pi_state()
882 		current->pi_state_cache = pi_state;  in put_pi_state()
896 	struct futex_pi_state *pi_state;  in exit_pi_state_list()  local
910 		pi_state = list_entry(next, struct futex_pi_state, list);  in exit_pi_state_list()
911 		key = pi_state->key;  in exit_pi_state_list()
924 		if (!refcount_inc_not_zero(&pi_state->refcount)) {  in exit_pi_state_list()
933 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
941 			raw_spin_unlock(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
943 			put_pi_state(pi_state);  in exit_pi_state_list()
947 		WARN_ON(pi_state->owner != curr);  in exit_pi_state_list()
948 		WARN_ON(list_empty(&pi_state->list));  in exit_pi_state_list()
949 		list_del_init(&pi_state->list);  in exit_pi_state_list()
950 		pi_state->owner = NULL;  in exit_pi_state_list()
953 		raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
956 		rt_mutex_futex_unlock(&pi_state->pi_mutex);  in exit_pi_state_list()
957 		put_pi_state(pi_state);  in exit_pi_state_list()
1055 			      struct futex_pi_state *pi_state,  in attach_to_pi_state()  argument
1065 	if (unlikely(!pi_state))  in attach_to_pi_state()
1080 	WARN_ON(!refcount_read(&pi_state->refcount));  in attach_to_pi_state()
1086 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1109 		if (!pi_state->owner) {  in attach_to_pi_state()
1137 		if (!pi_state->owner)  in attach_to_pi_state()
1146 	if (pid != task_pid_vnr(pi_state->owner))  in attach_to_pi_state()
1150 	get_pi_state(pi_state);  in attach_to_pi_state()
1151 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1152 	*ps = pi_state;  in attach_to_pi_state()
1168 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1234 	struct futex_pi_state *pi_state;  in attach_to_pi_owner()  local
1281 	pi_state = alloc_pi_state();  in attach_to_pi_owner()
1287 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);  in attach_to_pi_owner()
1290 	pi_state->key = *key;  in attach_to_pi_owner()
1292 	WARN_ON(!list_empty(&pi_state->list));  in attach_to_pi_owner()
1293 	list_add(&pi_state->list, &p->pi_state_list);  in attach_to_pi_owner()
1298 	pi_state->owner = p;  in attach_to_pi_owner()
1303 	*ps = pi_state;  in attach_to_pi_owner()
1319 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in lookup_pi_state()
1396 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
1467 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in mark_wake_futex()
1491 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)  in wake_futex_pi()  argument
1499 	new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);  in wake_futex_pi()
1545 	raw_spin_lock(&pi_state->owner->pi_lock);  in wake_futex_pi()
1546 	WARN_ON(list_empty(&pi_state->list));  in wake_futex_pi()
1547 	list_del_init(&pi_state->list);  in wake_futex_pi()
1548 	raw_spin_unlock(&pi_state->owner->pi_lock);  in wake_futex_pi()
1551 	WARN_ON(!list_empty(&pi_state->list));  in wake_futex_pi()
1552 	list_add(&pi_state->list, &new_owner->pi_state_list);  in wake_futex_pi()
1553 	pi_state->owner = new_owner;  in wake_futex_pi()
1556 	postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);  in wake_futex_pi()
1559 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1620 			if (this->pi_state || this->rt_waiter) {  in futex_wake()
1750 			if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1764 				if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1938 	struct futex_pi_state *pi_state = NULL;  in futex_requeue()  local
2042 						 &key2, &pi_state, nr_requeue);  in futex_requeue()
2054 			WARN_ON(pi_state);  in futex_requeue()
2069 			ret = lookup_pi_state(uaddr2, ret, hb2, &key2, &pi_state);  in futex_requeue()
2121 		    this->pi_state) {  in futex_requeue()
2152 			get_pi_state(pi_state);  in futex_requeue()
2153 			this->pi_state = pi_state;  in futex_requeue()
2154 			ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,  in futex_requeue()
2178 				this->pi_state = NULL;  in futex_requeue()
2179 				put_pi_state(pi_state);  in futex_requeue()
2196 	put_pi_state(pi_state);  in futex_requeue()
2335 		BUG_ON(q->pi_state);  in unqueue_me()
2355 	BUG_ON(!q->pi_state);  in unqueue_me_pi()
2356 	put_pi_state(q->pi_state);  in unqueue_me_pi()
2357 	q->pi_state = NULL;  in unqueue_me_pi()
2365 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()  local
2373 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2375 	oldowner = pi_state->owner;  in fixup_pi_state_owner()
2411 		if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {  in fixup_pi_state_owner()
2420 		newowner = rt_mutex_owner(&pi_state->pi_mutex);  in fixup_pi_state_owner()
2437 	if (!pi_state->owner)  in fixup_pi_state_owner()
2460 	if (pi_state->owner != NULL) {  in fixup_pi_state_owner()
2461 		raw_spin_lock(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
2462 		WARN_ON(list_empty(&pi_state->list));  in fixup_pi_state_owner()
2463 		list_del_init(&pi_state->list);  in fixup_pi_state_owner()
2464 		raw_spin_unlock(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
2467 	pi_state->owner = newowner;  in fixup_pi_state_owner()
2470 	WARN_ON(!list_empty(&pi_state->list));  in fixup_pi_state_owner()
2471 	list_add(&pi_state->list, &newowner->pi_state_list);  in fixup_pi_state_owner()
2473 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2491 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2511 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2516 	if (pi_state->owner != oldowner) {  in fixup_pi_state_owner()
2527 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2561 		if (q->pi_state->owner != current)  in fixup_owner()
2574 	if (q->pi_state->owner == current) {  in fixup_owner()
2583 	if (rt_mutex_owner(&q->pi_state->pi_mutex) == current) {  in fixup_owner()
2586 				q->pi_state->pi_mutex.owner,  in fixup_owner()
2587 				q->pi_state->owner);  in fixup_owner()
2803 	struct futex_pi_state *pi_state = NULL;  in futex_lock_pi()  local
2825 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);  in futex_lock_pi()
2854 	WARN_ON(!q.pi_state);  in futex_lock_pi()
2862 		ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2883 	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
2890 	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);  in futex_lock_pi()
2891 	raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
2902 	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);  in futex_lock_pi()
2915 	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))  in futex_lock_pi()
2935 	if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) {  in futex_lock_pi()
2936 		pi_state = q.pi_state;  in futex_lock_pi()
2937 		get_pi_state(pi_state);  in futex_lock_pi()
2943 	if (pi_state) {  in futex_lock_pi()
2944 		rt_mutex_futex_unlock(&pi_state->pi_mutex);  in futex_lock_pi()
2945 		put_pi_state(pi_state);  in futex_lock_pi()
3015 		struct futex_pi_state *pi_state = top_waiter->pi_state;  in futex_unlock_pi()  local
3018 		if (!pi_state)  in futex_unlock_pi()
3025 		if (pi_state->owner != current)  in futex_unlock_pi()
3028 		get_pi_state(pi_state);  in futex_unlock_pi()
3039 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in futex_unlock_pi()
3043 		ret = wake_futex_pi(uaddr, uval, pi_state);  in futex_unlock_pi()
3045 		put_pi_state(pi_state);  in futex_unlock_pi()
3213 	struct futex_pi_state *pi_state = NULL;  in futex_wait_requeue_pi()  local
3288 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
3291 			if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {  in futex_wait_requeue_pi()
3292 				pi_state = q.pi_state;  in futex_wait_requeue_pi()
3293 				get_pi_state(pi_state);  in futex_wait_requeue_pi()
3299 			put_pi_state(q.pi_state);  in futex_wait_requeue_pi()
3310 		WARN_ON(!q.pi_state);  in futex_wait_requeue_pi()
3311 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()
3336 		if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {  in futex_wait_requeue_pi()
3337 			pi_state = q.pi_state;  in futex_wait_requeue_pi()
3338 			get_pi_state(pi_state);  in futex_wait_requeue_pi()
3345 	if (pi_state) {  in futex_wait_requeue_pi()
3346 		rt_mutex_futex_unlock(&pi_state->pi_mutex);  in futex_wait_requeue_pi()
3347 		put_pi_state(pi_state);  in futex_wait_requeue_pi()