1 // SPDX-License-Identifier: GPL-2.0-only
10 	spin_lock_init(&wq_head->lock);  in __init_waitqueue_head()
11 	lockdep_set_class_and_name(&wq_head->lock, key, name);  in __init_waitqueue_head()
12 	INIT_LIST_HEAD(&wq_head->head);  in __init_waitqueue_head()
21 	wq_entry->flags &= ~WQ_FLAG_EXCLUSIVE;  in add_wait_queue()
22 	spin_lock_irqsave(&wq_head->lock, flags);  in add_wait_queue()
24 	spin_unlock_irqrestore(&wq_head->lock, flags);  in add_wait_queue()
32 	wq_entry->flags |= WQ_FLAG_EXCLUSIVE;  in add_wait_queue_exclusive()
33 	spin_lock_irqsave(&wq_head->lock, flags);  in add_wait_queue_exclusive()
35 	spin_unlock_irqrestore(&wq_head->lock, flags);  in add_wait_queue_exclusive()
43 	wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY;  in add_wait_queue_priority()
44 	spin_lock_irqsave(&wq_head->lock, flags);  in add_wait_queue_priority()
46 	spin_unlock_irqrestore(&wq_head->lock, flags);  in add_wait_queue_priority()
54 	spin_lock_irqsave(&wq_head->lock, flags);  in remove_wait_queue()
56 	spin_unlock_irqrestore(&wq_head->lock, flags);  in remove_wait_queue()
61  * Scan threshold to break wait queue walk.
63  * wait queue lock during the wait queue walk.
68  * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
71  * the non-exclusive tasks. Normally, exclusive tasks will be at the end of
72  * the list and any non-exclusive tasks will be woken first. A priority task
77  * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
87 	lockdep_assert_held(&wq_head->lock);  in __wake_up_common()
89 	if (bookmark && (bookmark->flags & WQ_FLAG_BOOKMARK)) {  in __wake_up_common()
92 		list_del(&bookmark->entry);  in __wake_up_common()
93 		bookmark->flags = 0;  in __wake_up_common()
95 		curr = list_first_entry(&wq_head->head, wait_queue_entry_t, entry);  in __wake_up_common()
97 	if (&curr->entry == &wq_head->head)  in __wake_up_common()
100 	list_for_each_entry_safe_from(curr, next, &wq_head->head, entry) {  in __wake_up_common()
101 		unsigned flags = curr->flags;  in __wake_up_common()
107 		ret = curr->func(curr, mode, wake_flags, key);  in __wake_up_common()
110 		if (ret && (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)  in __wake_up_common()
114 				(&next->entry != &wq_head->head)) {  in __wake_up_common()
115 			bookmark->flags = WQ_FLAG_BOOKMARK;  in __wake_up_common()
116 			list_add_tail(&bookmark->entry, &next->entry);  in __wake_up_common()
136 		spin_lock_irqsave(&wq_head->lock, flags);  in __wake_up_common_lock()
139 		spin_unlock_irqrestore(&wq_head->lock, flags);  in __wake_up_common_lock()
144  * __wake_up - wake up threads blocked on a waitqueue.
147  * @nr_exclusive: how many wake-one or wake-many threads to wake up
151  * accessing the task state.
183  * __wake_up_sync_key - wake up threads blocked on a waitqueue.
190  * be migrated to another CPU - ie. the two threads are 'synchronized'
196  * accessing the task state.
209  * __wake_up_locked_sync_key - wake up a thread blocked on a locked waitqueue.
216  * be migrated to another CPU - ie. the two threads are 'synchronized'
222  * accessing the task state.
232  * __wake_up_sync - see __wake_up_sync_key()
248  * Note: we use "set_current_state()" _after_ the wait-queue add,
250  * wake-function that tests for the wait-queue being active
254  * The spin_unlock() itself is semi-permeable and only protects
256  * stops them from bleeding out - it would still allow subsequent
260 prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)  in prepare_to_wait()  argument
264 	wq_entry->flags &= ~WQ_FLAG_EXCLUSIVE;  in prepare_to_wait()
265 	spin_lock_irqsave(&wq_head->lock, flags);  in prepare_to_wait()
266 	if (list_empty(&wq_entry->entry))  in prepare_to_wait()
268 	set_current_state(state);  in prepare_to_wait()
269 	spin_unlock_irqrestore(&wq_head->lock, flags);  in prepare_to_wait()
275 …re_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)  in prepare_to_wait_exclusive()  argument
280 	wq_entry->flags |= WQ_FLAG_EXCLUSIVE;  in prepare_to_wait_exclusive()
281 	spin_lock_irqsave(&wq_head->lock, flags);  in prepare_to_wait_exclusive()
282 	if (list_empty(&wq_entry->entry)) {  in prepare_to_wait_exclusive()
283 		was_empty = list_empty(&wq_head->head);  in prepare_to_wait_exclusive()
286 	set_current_state(state);  in prepare_to_wait_exclusive()
287 	spin_unlock_irqrestore(&wq_head->lock, flags);  in prepare_to_wait_exclusive()
294 	wq_entry->flags = flags;  in init_wait_entry()
295 	wq_entry->private = current;  in init_wait_entry()
296 	wq_entry->func = autoremove_wake_function;  in init_wait_entry()
297 	INIT_LIST_HEAD(&wq_entry->entry);  in init_wait_entry()
301 …repare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)  in prepare_to_wait_event()  argument
306 	spin_lock_irqsave(&wq_head->lock, flags);  in prepare_to_wait_event()
307 	if (signal_pending_state(state, current)) {  in prepare_to_wait_event()
314 		 * wakeup locks/unlocks the same wq_head->lock.  in prepare_to_wait_event()
316 		 * But we need to ensure that set-condition + wakeup after that  in prepare_to_wait_event()
320 		list_del_init(&wq_entry->entry);  in prepare_to_wait_event()
321 		ret = -ERESTARTSYS;  in prepare_to_wait_event()
323 		if (list_empty(&wq_entry->entry)) {  in prepare_to_wait_event()
324 			if (wq_entry->flags & WQ_FLAG_EXCLUSIVE)  in prepare_to_wait_event()
329 		set_current_state(state);  in prepare_to_wait_event()
331 	spin_unlock_irqrestore(&wq_head->lock, flags);  in prepare_to_wait_event()
338  * Note! These two wait functions are entered with the
339  * wait-queue lock held (and interrupts off in the _irq
341  * condition in the caller before they add the wait
344 int do_wait_intr(wait_queue_head_t *wq, wait_queue_entry_t *wait)  in do_wait_intr()  argument
346 	if (likely(list_empty(&wait->entry)))  in do_wait_intr()
347 		__add_wait_queue_entry_tail(wq, wait);  in do_wait_intr()
351 		return -ERESTARTSYS;  in do_wait_intr()
353 	spin_unlock(&wq->lock);  in do_wait_intr()
355 	spin_lock(&wq->lock);  in do_wait_intr()
361 int do_wait_intr_irq(wait_queue_head_t *wq, wait_queue_entry_t *wait)  in do_wait_intr_irq()  argument
363 	if (likely(list_empty(&wait->entry)))  in do_wait_intr_irq()
364 		__add_wait_queue_entry_tail(wq, wait);  in do_wait_intr_irq()
368 		return -ERESTARTSYS;  in do_wait_intr_irq()
370 	spin_unlock_irq(&wq->lock);  in do_wait_intr_irq()
372 	spin_lock_irq(&wq->lock);  in do_wait_intr_irq()
379  * finish_wait - clean up after waiting in a queue
381  * @wq_entry: wait descriptor
383  * Sets current thread back to running state and removes
384  * the wait descriptor from the given waitqueue if still
395 	 *  - we use the "careful" check that verifies both  in finish_wait()
397 	 *    be any half-pending updates in progress on other  in finish_wait()
401 	 *  - all other users take the lock (ie we can only  in finish_wait()
405 	if (!list_empty_careful(&wq_entry->entry)) {  in finish_wait()
406 		spin_lock_irqsave(&wq_head->lock, flags);  in finish_wait()
407 		list_del_init(&wq_entry->entry);  in finish_wait()
408 		spin_unlock_irqrestore(&wq_head->lock, flags);  in finish_wait()
418 		list_del_init_careful(&wq_entry->entry);  in autoremove_wake_function()
426 	return (current->flags & PF_KTHREAD) && kthread_should_stop();  in is_kthread_should_stop()
430  * DEFINE_WAIT_FUNC(wait, woken_wake_func);
432  * add_wait_queue(&wq_head, &wait);
439  *     p->state = mode;				wq_entry->flags |= WQ_FLAG_WOKEN;
441  *     if (!(wq_entry->flags & WQ_FLAG_WOKEN))	   <full barrier>
442  *         schedule()				   if (p->state & mode)
443  *     p->state = TASK_RUNNING;			      p->state = TASK_RUNNING;
444  *     wq_entry->flags &= ~WQ_FLAG_WOKEN;	~~~~~~~~~~~~~~~~~~
447  * remove_wait_queue(&wq_head, &wait);		wq_entry->flags |= WQ_FLAG_WOKEN;
454 	 * either we see the store to wq_entry->flags in woken_wake_function()  in wait_woken()
455 	 * or woken_wake_function() sees our store to current->state.  in wait_woken()
458 	if (!(wq_entry->flags & WQ_FLAG_WOKEN) && !is_kthread_should_stop())  in wait_woken()
464 	 * in woken_wake_function() such that either we see the wait condition  in wait_woken()
465 	 * being true or the store to wq_entry->flags in woken_wake_function()  in wait_woken()
468 	smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN); /* B */  in wait_woken()
478 	wq_entry->flags |= WQ_FLAG_WOKEN;  in woken_wake_function()