375 static inline void hb_waiters_inc(struct futex_hash_bucket *hb)  in hb_waiters_inc()  argument
378 	atomic_inc(&hb->waiters);  in hb_waiters_inc()
390 static inline void hb_waiters_dec(struct futex_hash_bucket *hb)  in hb_waiters_dec()  argument
393 	atomic_dec(&hb->waiters);  in hb_waiters_dec()
397 static inline int hb_waiters_pending(struct futex_hash_bucket *hb)  in hb_waiters_pending()  argument
404 	return atomic_read(&hb->waiters);  in hb_waiters_pending()
751 static struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb,  in futex_top_waiter()  argument
756 	plist_for_each_entry(this, &hb->chain, list) {  in futex_top_waiter()
901 	struct futex_hash_bucket *hb;  in exit_pi_state_list()  local
916 		hb = hash_futex(&key);  in exit_pi_state_list()
936 		spin_lock(&hb->lock);  in exit_pi_state_list()
946 			spin_unlock(&hb->lock);  in exit_pi_state_list()
958 		spin_unlock(&hb->lock);  in exit_pi_state_list()
1404 static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,  in futex_lock_pi_atomic()  argument
1438 	top_waiter = futex_top_waiter(hb, key);  in futex_lock_pi_atomic()
1508 	struct futex_hash_bucket *hb;  in __unqueue_futex()  local
1514 	hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);  in __unqueue_futex()
1515 	plist_del(&q->list, &hb->chain);  in __unqueue_futex()
1516 	hb_waiters_dec(hb);  in __unqueue_futex()
1653 	struct futex_hash_bucket *hb;  in futex_wake()  local
1666 	hb = hash_futex(&key);  in futex_wake()
1669 	if (!hb_waiters_pending(hb))  in futex_wake()
1672 	spin_lock(&hb->lock);  in futex_wake()
1674 	plist_for_each_entry_safe(this, next, &hb->chain, list) {  in futex_wake()
1691 	spin_unlock(&hb->lock);  in futex_wake()
1987 			   struct futex_hash_bucket *hb)  in requeue_pi_wake_futex()  argument
1996 	q->lock_ptr = &hb->lock;  in requeue_pi_wake_futex()
2445 	__acquires(&hb->lock)  in queue_lock()
2447 	struct futex_hash_bucket *hb;  in queue_lock()  local
2449 	hb = hash_futex(&q->key);  in queue_lock()
2459 	hb_waiters_inc(hb); /* implies smp_mb(); (A) */  in queue_lock()
2461 	q->lock_ptr = &hb->lock;  in queue_lock()
2463 	spin_lock(&hb->lock);  in queue_lock()
2464 	return hb;  in queue_lock()
2468 queue_unlock(struct futex_hash_bucket *hb)  in queue_unlock()  argument
2469 	__releases(&hb->lock)  in queue_unlock()
2471 	spin_unlock(&hb->lock);  in queue_unlock()
2472 	hb_waiters_dec(hb);  in queue_unlock()
2475 static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb)  in __queue_me()  argument
2490 	plist_add(&q->list, &hb->chain);  in __queue_me()
2506 static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)  in queue_me()  argument
2507 	__releases(&hb->lock)  in queue_me()
2509 	__queue_me(q, hb);  in queue_me()
2510 	spin_unlock(&hb->lock);  in queue_me()
2826 static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,  in futex_wait_queue_me()  argument
2836 	queue_me(q, hb);  in futex_wait_queue_me()
2875 			   struct futex_q *q, struct futex_hash_bucket **hb)  in futex_wait_setup()  argument
2904 	*hb = queue_lock(q);  in futex_wait_setup()
2909 		queue_unlock(*hb);  in futex_wait_setup()
2922 		queue_unlock(*hb);  in futex_wait_setup()
2934 	struct futex_hash_bucket *hb;  in futex_wait()  local
2949 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait()
2954 	futex_wait_queue_me(hb, &q, to);  in futex_wait()
3024 	struct futex_hash_bucket *hb;  in futex_lock_pi()  local
3042 	hb = queue_lock(&q);  in futex_lock_pi()
3044 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,  in futex_lock_pi()
3066 			queue_unlock(hb);  in futex_lock_pi()
3085 	__queue_me(&q, hb);  in futex_lock_pi()
3162 	queue_unlock(hb);  in futex_lock_pi()
3172 	queue_unlock(hb);  in futex_lock_pi()
3193 	struct futex_hash_bucket *hb;  in futex_unlock_pi()  local
3213 	hb = hash_futex(&key);  in futex_unlock_pi()
3214 	spin_lock(&hb->lock);  in futex_unlock_pi()
3221 	top_waiter = futex_top_waiter(hb, &key);  in futex_unlock_pi()
3248 		spin_unlock(&hb->lock);  in futex_unlock_pi()
3287 		spin_unlock(&hb->lock);  in futex_unlock_pi()
3307 	spin_unlock(&hb->lock);  in futex_unlock_pi()
3335 int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,  in handle_early_requeue_pi_wakeup()  argument
3348 	WARN_ON_ONCE(&hb->lock != q->lock_ptr);  in handle_early_requeue_pi_wakeup()
3354 	plist_del(&q->list, &hb->chain);  in handle_early_requeue_pi_wakeup()
3355 	hb_waiters_dec(hb);  in handle_early_requeue_pi_wakeup()
3412 	struct futex_hash_bucket *hb;  in futex_wait_requeue_pi()  local
3448 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait_requeue_pi()
3457 		queue_unlock(hb);  in futex_wait_requeue_pi()
3463 	futex_wait_queue_me(hb, &q, to);  in futex_wait_requeue_pi()
3468 		spin_lock(&hb->lock);  in futex_wait_requeue_pi()
3469 		ret = handle_early_requeue_pi_wakeup(hb, &q, to);  in futex_wait_requeue_pi()
3470 		spin_unlock(&hb->lock);  in futex_wait_requeue_pi()