1 // SPDX-License-Identifier: GPL-2.0-or-later
46  *     unlock(hash_bucket(futex));
63  *   smp_mb(); (A) <-- paired with -.
72  *                                  `--------> smp_mb(); (B)
75  *     unlock(hash_bucket(futex));
79  *     waiters--; (b)                        unlock(hash_bucket(futex));
103  * acquiring the lock. It then decrements them again  after releasing it -
117 	struct task_struct *p = q->task;  in futex_wake_mark()
119 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in futex_wake_mark()
125 	 * The waiting task can free the futex_q as soon as q->lock_ptr = NULL  in futex_wake_mark()
131 	smp_store_release(&q->lock_ptr, NULL);  in futex_wake_mark()
135 	 * the hb->lock.  in futex_wake_mark()
152 		return -EINVAL;  in futex_wake()
164 	spin_lock(&hb->lock);  in futex_wake()
166 	plist_for_each_entry_safe(this, next, &hb->chain, list) {  in futex_wake()
167 		if (futex_match (&this->key, &key)) {  in futex_wake()
168 			if (this->pi_state || this->rt_waiter) {  in futex_wake()
169 				ret = -EINVAL;  in futex_wake()
174 			if (!(this->bitset & bitset))  in futex_wake()
183 	spin_unlock(&hb->lock);  in futex_wake()
198 			char comm[sizeof(current->comm)];  in futex_atomic_op_inuser()
200 			 * kill this print and return -EINVAL when userspace  in futex_atomic_op_inuser()
230 		return -ENOSYS;  in futex_atomic_op_inuser()
265 		    unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {  in futex_wake_op()
274 		if (op_ret == -EFAULT) {  in futex_wake_op()
286 	plist_for_each_entry_safe(this, next, &hb1->chain, list) {  in futex_wake_op()
287 		if (futex_match (&this->key, &key1)) {  in futex_wake_op()
288 			if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
289 				ret = -EINVAL;  in futex_wake_op()
300 		plist_for_each_entry_safe(this, next, &hb2->chain, list) {  in futex_wake_op()
301 			if (futex_match (&this->key, &key2)) {  in futex_wake_op()
302 				if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
303 					ret = -EINVAL;  in futex_wake_op()
323  * futex_wait_queue() - futex_queue() and wait for wakeup, timeout, or signal
348 	if (likely(!plist_node_empty(&q->list))) {  in futex_wait_queue()
354 		if (!timeout || timeout->task)  in futex_wait_queue()
361  * unqueue_multiple - Remove various futexes from their hash bucket
368  *  - >=0 - Index of the last futex that was awoken;
369  *  - -1  - No futex was awoken
373 	int ret = -1, i;  in unqueue_multiple()
384  * futex_wait_multiple_setup - Prepare to wait and enqueue multiple futexes
395  *  -  1 - One of the futexes was woken by another thread
396  *  -  0 - Success
397  *  - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL
410 	 * make sure that current->state is TASK_INTERRUPTIBLE, so we don't  in futex_wait_multiple_setup()
413 	 * Thus, we fetch the list of futexes keys in two steps, by first  in futex_wait_multiple_setup()
414 	 * pinning all the memory keys in the futex key, and only then we read  in futex_wait_multiple_setup()
474 				return -EFAULT;  in futex_wait_multiple_setup()
481 			return -EWOULDBLOCK;  in futex_wait_multiple_setup()
488  * futex_sleep_multiple - Check sleeping conditions and sleep
499 	if (to && !to->task)  in futex_sleep_multiple()
502 	for (; count; count--, vs++) {  in futex_sleep_multiple()
503 		if (!READ_ONCE(vs->q.lock_ptr))  in futex_sleep_multiple()
511  * futex_wait_multiple - Prepare to wait on and enqueue several futexes
521  *  - >=0 - Hint to the futex that was awoken
522  *  - <0  - On error
550 		if (to && !to->task)  in futex_wait_multiple()
551 			return -ETIMEDOUT;  in futex_wait_multiple()
553 			return -ERESTARTSYS;  in futex_wait_multiple()
562  * futex_wait_setup() - Prepare to wait on a futex
574  *  -  0 - uaddr contains val and hb has been locked;
575  *  - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
584 	 * Access the page AFTER the hash-bucket is locked.  in futex_wait_setup()
592 	 * any cond.  If we locked the hash-bucket after testing *uaddr, that  in futex_wait_setup()
596 	 * On the other hand, we insert q and release the hash-bucket only  in futex_wait_setup()
602 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);  in futex_wait_setup()
626 		ret = -EWOULDBLOCK;  in futex_wait_setup()
641 		return -EINVAL;  in futex_wait()
645 			       current->timer_slack_ns);  in futex_wait()
648 	 * Prepare to wait on uaddr. On success, it holds hb->lock and q  in futex_wait()
662 	ret = -ETIMEDOUT;  in futex_wait()
663 	if (to && !to->task)  in futex_wait()
673 	ret = -ERESTARTSYS;  in futex_wait()
677 	restart = &current->restart_block;  in futex_wait()
678 	restart->futex.uaddr = uaddr;  in futex_wait()
679 	restart->futex.val = val;  in futex_wait()
680 	restart->futex.time = *abs_time;  in futex_wait()
681 	restart->futex.bitset = bitset;  in futex_wait()
682 	restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;  in futex_wait()
688 		hrtimer_cancel(&to->timer);  in futex_wait()
689 		destroy_hrtimer_on_stack(&to->timer);  in futex_wait()
696 	u32 __user *uaddr = restart->futex.uaddr;  in futex_wait_restart()
699 	if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {  in futex_wait_restart()
700 		t = restart->futex.time;  in futex_wait_restart()
703 	restart->fn = do_no_restart_syscall;  in futex_wait_restart()
705 	return (long)futex_wait(uaddr, restart->futex.flags,  in futex_wait_restart()
706 				restart->futex.val, tp, restart->futex.bitset);  in futex_wait_restart()