219 	struct futex_pi_state *pi_state;  member
737 	struct futex_pi_state *pi_state;  in refill_pi_state_cache()  local
742 	pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);  in refill_pi_state_cache()
744 	if (!pi_state)  in refill_pi_state_cache()
747 	INIT_LIST_HEAD(&pi_state->list);  in refill_pi_state_cache()
749 	pi_state->owner = NULL;  in refill_pi_state_cache()
750 	refcount_set(&pi_state->refcount, 1);  in refill_pi_state_cache()
751 	pi_state->key = FUTEX_KEY_INIT;  in refill_pi_state_cache()
753 	current->pi_state_cache = pi_state;  in refill_pi_state_cache()
760 	struct futex_pi_state *pi_state = current->pi_state_cache;  in alloc_pi_state()  local
762 	WARN_ON(!pi_state);  in alloc_pi_state()
765 	return pi_state;  in alloc_pi_state()
768 static void get_pi_state(struct futex_pi_state *pi_state)  in get_pi_state()  argument
770 	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));  in get_pi_state()
777 static void put_pi_state(struct futex_pi_state *pi_state)  in put_pi_state()  argument
779 	if (!pi_state)  in put_pi_state()
782 	if (!refcount_dec_and_test(&pi_state->refcount))  in put_pi_state()
789 	if (pi_state->owner) {  in put_pi_state()
793 		raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
794 		owner = pi_state->owner;  in put_pi_state()
797 			list_del_init(&pi_state->list);  in put_pi_state()
800 		rt_mutex_proxy_unlock(&pi_state->pi_mutex, owner);  in put_pi_state()
801 		raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
805 		kfree(pi_state);  in put_pi_state()
812 		pi_state->owner = NULL;  in put_pi_state()
813 		refcount_set(&pi_state->refcount, 1);  in put_pi_state()
814 		current->pi_state_cache = pi_state;  in put_pi_state()
828 	struct futex_pi_state *pi_state;  in exit_pi_state_list()  local
842 		pi_state = list_entry(next, struct futex_pi_state, list);  in exit_pi_state_list()
843 		key = pi_state->key;  in exit_pi_state_list()
856 		if (!refcount_inc_not_zero(&pi_state->refcount)) {  in exit_pi_state_list()
865 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
873 			raw_spin_unlock(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
875 			put_pi_state(pi_state);  in exit_pi_state_list()
879 		WARN_ON(pi_state->owner != curr);  in exit_pi_state_list()
880 		WARN_ON(list_empty(&pi_state->list));  in exit_pi_state_list()
881 		list_del_init(&pi_state->list);  in exit_pi_state_list()
882 		pi_state->owner = NULL;  in exit_pi_state_list()
885 		raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in exit_pi_state_list()
888 		rt_mutex_futex_unlock(&pi_state->pi_mutex);  in exit_pi_state_list()
889 		put_pi_state(pi_state);  in exit_pi_state_list()
988 			      struct futex_pi_state *pi_state,  in attach_to_pi_state()  argument
998 	if (unlikely(!pi_state))  in attach_to_pi_state()
1013 	WARN_ON(!refcount_read(&pi_state->refcount));  in attach_to_pi_state()
1019 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1042 		if (!pi_state->owner) {  in attach_to_pi_state()
1070 		if (!pi_state->owner)  in attach_to_pi_state()
1079 	if (pid != task_pid_vnr(pi_state->owner))  in attach_to_pi_state()
1083 	get_pi_state(pi_state);  in attach_to_pi_state()
1084 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1085 	*ps = pi_state;  in attach_to_pi_state()
1101 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
1201 	struct futex_pi_state *pi_state;  in attach_to_pi_owner()  local
1259 	pi_state = alloc_pi_state();  in attach_to_pi_owner()
1265 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);  in attach_to_pi_owner()
1268 	pi_state->key = *key;  in attach_to_pi_owner()
1270 	WARN_ON(!list_empty(&pi_state->list));  in attach_to_pi_owner()
1271 	list_add(&pi_state->list, &p->pi_state_list);  in attach_to_pi_owner()
1276 	pi_state->owner = p;  in attach_to_pi_owner()
1281 	*ps = pi_state;  in attach_to_pi_owner()
1298 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in lookup_pi_state()
1383 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
1454 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in mark_wake_futex()
1478 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)  in wake_futex_pi()  argument
1486 	new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);  in wake_futex_pi()
1534 	raw_spin_lock(&pi_state->owner->pi_lock);  in wake_futex_pi()
1535 	WARN_ON(list_empty(&pi_state->list));  in wake_futex_pi()
1536 	list_del_init(&pi_state->list);  in wake_futex_pi()
1537 	raw_spin_unlock(&pi_state->owner->pi_lock);  in wake_futex_pi()
1540 	WARN_ON(!list_empty(&pi_state->list));  in wake_futex_pi()
1541 	list_add(&pi_state->list, &new_owner->pi_state_list);  in wake_futex_pi()
1542 	pi_state->owner = new_owner;  in wake_futex_pi()
1545 	postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);  in wake_futex_pi()
1548 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1609 			if (this->pi_state || this->rt_waiter) {  in futex_wake()
1733 			if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1747 				if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1920 	struct futex_pi_state *pi_state = NULL;  in futex_requeue()  local
2022 						 &key2, &pi_state,  in futex_requeue()
2035 			WARN_ON(pi_state);  in futex_requeue()
2050 					      &pi_state, &exiting);  in futex_requeue()
2105 		    this->pi_state) {  in futex_requeue()
2136 			get_pi_state(pi_state);  in futex_requeue()
2137 			this->pi_state = pi_state;  in futex_requeue()
2138 			ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,  in futex_requeue()
2161 				this->pi_state = NULL;  in futex_requeue()
2162 				put_pi_state(pi_state);  in futex_requeue()
2178 	put_pi_state(pi_state);  in futex_requeue()
2302 		BUG_ON(q->pi_state);  in unqueue_me()
2321 	BUG_ON(!q->pi_state);  in unqueue_me_pi()
2322 	put_pi_state(q->pi_state);  in unqueue_me_pi()
2323 	q->pi_state = NULL;  in unqueue_me_pi()
2331 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()  local
2339 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2341 	oldowner = pi_state->owner;  in fixup_pi_state_owner()
2377 		if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {  in fixup_pi_state_owner()
2387 		newowner = rt_mutex_owner(&pi_state->pi_mutex);  in fixup_pi_state_owner()
2415 	if (!pi_state->owner)  in fixup_pi_state_owner()
2438 	if (pi_state->owner != NULL) {  in fixup_pi_state_owner()
2439 		raw_spin_lock(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
2440 		WARN_ON(list_empty(&pi_state->list));  in fixup_pi_state_owner()
2441 		list_del_init(&pi_state->list);  in fixup_pi_state_owner()
2442 		raw_spin_unlock(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
2445 	pi_state->owner = newowner;  in fixup_pi_state_owner()
2448 	WARN_ON(!list_empty(&pi_state->list));  in fixup_pi_state_owner()
2449 	list_add(&pi_state->list, &newowner->pi_state_list);  in fixup_pi_state_owner()
2451 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2469 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2489 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2494 	if (pi_state->owner != oldowner) {  in fixup_pi_state_owner()
2505 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
2539 		if (q->pi_state->owner != current)  in fixup_owner()
2552 	if (q->pi_state->owner == current) {  in fixup_owner()
2561 	if (rt_mutex_owner(&q->pi_state->pi_mutex) == current) {  in fixup_owner()
2564 				q->pi_state->pi_mutex.owner,  in fixup_owner()
2565 				q->pi_state->owner);  in fixup_owner()
2776 	struct futex_pi_state *pi_state = NULL;  in futex_lock_pi()  local
2799 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,  in futex_lock_pi()
2835 	WARN_ON(!q.pi_state);  in futex_lock_pi()
2843 		ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2864 	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
2871 	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);  in futex_lock_pi()
2872 	raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
2883 	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);  in futex_lock_pi()
2896 	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))  in futex_lock_pi()
2916 	if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) {  in futex_lock_pi()
2917 		pi_state = q.pi_state;  in futex_lock_pi()
2918 		get_pi_state(pi_state);  in futex_lock_pi()
2924 	if (pi_state) {  in futex_lock_pi()
2925 		rt_mutex_futex_unlock(&pi_state->pi_mutex);  in futex_lock_pi()
2926 		put_pi_state(pi_state);  in futex_lock_pi()
2993 		struct futex_pi_state *pi_state = top_waiter->pi_state;  in futex_unlock_pi()  local
2996 		if (!pi_state)  in futex_unlock_pi()
3003 		if (pi_state->owner != current)  in futex_unlock_pi()
3006 		get_pi_state(pi_state);  in futex_unlock_pi()
3017 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in futex_unlock_pi()
3021 		ret = wake_futex_pi(uaddr, uval, pi_state);  in futex_unlock_pi()
3023 		put_pi_state(pi_state);  in futex_unlock_pi()
3188 	struct futex_pi_state *pi_state = NULL;  in futex_wait_requeue_pi()  local
3263 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
3266 			if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {  in futex_wait_requeue_pi()
3267 				pi_state = q.pi_state;  in futex_wait_requeue_pi()
3268 				get_pi_state(pi_state);  in futex_wait_requeue_pi()
3274 			put_pi_state(q.pi_state);  in futex_wait_requeue_pi()
3285 		WARN_ON(!q.pi_state);  in futex_wait_requeue_pi()
3286 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()
3311 		if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {  in futex_wait_requeue_pi()
3312 			pi_state = q.pi_state;  in futex_wait_requeue_pi()
3313 			get_pi_state(pi_state);  in futex_wait_requeue_pi()
3320 	if (pi_state) {  in futex_wait_requeue_pi()
3321 		rt_mutex_futex_unlock(&pi_state->pi_mutex);  in futex_wait_requeue_pi()
3322 		put_pi_state(pi_state);  in futex_wait_requeue_pi()