1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_WAIT_H
3 #define _LINUX_WAIT_H
4 /*
5 * Linux wait queue related types and methods
6 */
7 #include <linux/list.h>
8 #include <linux/stddef.h>
9 #include <linux/spinlock.h>
10
11 #include <asm/current.h>
12 #include <uapi/linux/wait.h>
13
14 typedef struct wait_queue_entry wait_queue_entry_t;
15
16 typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
17 int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
18
19 /* wait_queue_entry::flags */
20 #define WQ_FLAG_EXCLUSIVE 0x01
21 #define WQ_FLAG_WOKEN 0x02
22 #define WQ_FLAG_BOOKMARK 0x04
23 #define WQ_FLAG_CUSTOM 0x08
24 #define WQ_FLAG_DONE 0x10
25
26 /*
27 * A single wait-queue entry structure:
28 */
29 struct wait_queue_entry {
30 unsigned int flags;
31 void *private;
32 wait_queue_func_t func;
33 struct list_head entry;
34 };
35
36 struct wait_queue_head {
37 spinlock_t lock;
38 struct list_head head;
39 };
40 typedef struct wait_queue_head wait_queue_head_t;
41
42 struct task_struct;
43
44 /*
45 * Macros for declaration and initialisaton of the datatypes
46 */
47
48 #define __WAITQUEUE_INITIALIZER(name, tsk) { \
49 .private = tsk, \
50 .func = default_wake_function, \
51 .entry = { NULL, NULL } }
52
53 #define DECLARE_WAITQUEUE(name, tsk) \
54 struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
55
56 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
57 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
58 .head = { &(name).head, &(name).head } }
59
60 #define DECLARE_WAIT_QUEUE_HEAD(name) \
61 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
62
63 extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
64
65 #define init_waitqueue_head(wq_head) \
66 do { \
67 static struct lock_class_key __key; \
68 \
69 __init_waitqueue_head((wq_head), #wq_head, &__key); \
70 } while (0)
71
72 #ifdef CONFIG_LOCKDEP
73 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
74 ({ init_waitqueue_head(&name); name; })
75 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
76 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
77 #else
78 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
79 #endif
80
init_waitqueue_entry(struct wait_queue_entry * wq_entry,struct task_struct * p)81 static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
82 {
83 wq_entry->flags = 0;
84 wq_entry->private = p;
85 wq_entry->func = default_wake_function;
86 }
87
88 static inline void
init_waitqueue_func_entry(struct wait_queue_entry * wq_entry,wait_queue_func_t func)89 init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
90 {
91 wq_entry->flags = 0;
92 wq_entry->private = NULL;
93 wq_entry->func = func;
94 }
95
96 /**
97 * waitqueue_active -- locklessly test for waiters on the queue
98 * @wq_head: the waitqueue to test for waiters
99 *
100 * returns true if the wait list is not empty
101 *
102 * NOTE: this function is lockless and requires care, incorrect usage _will_
103 * lead to sporadic and non-obvious failure.
104 *
105 * Use either while holding wait_queue_head::lock or when used for wakeups
106 * with an extra smp_mb() like::
107 *
108 * CPU0 - waker CPU1 - waiter
109 *
110 * for (;;) {
111 * @cond = true; prepare_to_wait(&wq_head, &wait, state);
112 * smp_mb(); // smp_mb() from set_current_state()
113 * if (waitqueue_active(wq_head)) if (@cond)
114 * wake_up(wq_head); break;
115 * schedule();
116 * }
117 * finish_wait(&wq_head, &wait);
118 *
119 * Because without the explicit smp_mb() it's possible for the
120 * waitqueue_active() load to get hoisted over the @cond store such that we'll
121 * observe an empty wait list while the waiter might not observe @cond.
122 *
123 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
124 * which (when the lock is uncontended) are of roughly equal cost.
125 */
waitqueue_active(struct wait_queue_head * wq_head)126 static inline int waitqueue_active(struct wait_queue_head *wq_head)
127 {
128 return !list_empty(&wq_head->head);
129 }
130
131 /**
132 * wq_has_single_sleeper - check if there is only one sleeper
133 * @wq_head: wait queue head
134 *
135 * Returns true of wq_head has only one sleeper on the list.
136 *
137 * Please refer to the comment for waitqueue_active.
138 */
wq_has_single_sleeper(struct wait_queue_head * wq_head)139 static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
140 {
141 return list_is_singular(&wq_head->head);
142 }
143
144 /**
145 * wq_has_sleeper - check if there are any waiting processes
146 * @wq_head: wait queue head
147 *
148 * Returns true if wq_head has waiting processes
149 *
150 * Please refer to the comment for waitqueue_active.
151 */
wq_has_sleeper(struct wait_queue_head * wq_head)152 static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
153 {
154 /*
155 * We need to be sure we are in sync with the
156 * add_wait_queue modifications to the wait queue.
157 *
158 * This memory barrier should be paired with one on the
159 * waiting side.
160 */
161 smp_mb();
162 return waitqueue_active(wq_head);
163 }
164
165 extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
166 extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
167 extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
168
__add_wait_queue(struct wait_queue_head * wq_head,struct wait_queue_entry * wq_entry)169 static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
170 {
171 list_add(&wq_entry->entry, &wq_head->head);
172 }
173
174 /*
175 * Used for wake-one threads:
176 */
177 static inline void
__add_wait_queue_exclusive(struct wait_queue_head * wq_head,struct wait_queue_entry * wq_entry)178 __add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
179 {
180 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
181 __add_wait_queue(wq_head, wq_entry);
182 }
183
__add_wait_queue_entry_tail(struct wait_queue_head * wq_head,struct wait_queue_entry * wq_entry)184 static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
185 {
186 list_add_tail(&wq_entry->entry, &wq_head->head);
187 }
188
189 static inline void
__add_wait_queue_entry_tail_exclusive(struct wait_queue_head * wq_head,struct wait_queue_entry * wq_entry)190 __add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
191 {
192 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
193 __add_wait_queue_entry_tail(wq_head, wq_entry);
194 }
195
196 static inline void
__remove_wait_queue(struct wait_queue_head * wq_head,struct wait_queue_entry * wq_entry)197 __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
198 {
199 list_del(&wq_entry->entry);
200 }
201
202 void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
203 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
204 void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
205 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
206 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
207 void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
208 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
209 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
210
211 #define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
212 #define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
213 #define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
214 #define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
215 #define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
216
217 #define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
218 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
219 #define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
220 #define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
221
222 /*
223 * Wakeup macros to be used to report events to the targets.
224 */
225 #define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
226 #define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
227 #define wake_up_poll(x, m) \
228 __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
229 #define wake_up_locked_poll(x, m) \
230 __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
231 #define wake_up_interruptible_poll(x, m) \
232 __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
233 #define wake_up_interruptible_sync_poll(x, m) \
234 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
235 #define wake_up_interruptible_sync_poll_locked(x, m) \
236 __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
237
238 #define ___wait_cond_timeout(condition) \
239 ({ \
240 bool __cond = (condition); \
241 if (__cond && !__ret) \
242 __ret = 1; \
243 __cond || !__ret; \
244 })
245
246 #define ___wait_is_interruptible(state) \
247 (!__builtin_constant_p(state) || \
248 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
249
250 extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
251
252 /*
253 * The below macro ___wait_event() has an explicit shadow of the __ret
254 * variable when used from the wait_event_*() macros.
255 *
256 * This is so that both can use the ___wait_cond_timeout() construct
257 * to wrap the condition.
258 *
259 * The type inconsistency of the wait_event_*() __ret variable is also
260 * on purpose; we use long where we can return timeout values and int
261 * otherwise.
262 */
263
264 #define ___wait_event(wq_head, condition, state, exclusive, ret, cmd) \
265 ({ \
266 __label__ __out; \
267 struct wait_queue_entry __wq_entry; \
268 long __ret = ret; /* explicit shadow */ \
269 \
270 init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \
271 for (;;) { \
272 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
273 \
274 if (condition) \
275 break; \
276 \
277 if (___wait_is_interruptible(state) && __int) { \
278 __ret = __int; \
279 goto __out; \
280 } \
281 \
282 cmd; \
283 } \
284 finish_wait(&wq_head, &__wq_entry); \
285 __out: __ret; \
286 })
287
288 #define __wait_event(wq_head, condition) \
289 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
290 schedule())
291
292 /**
293 * wait_event - sleep until a condition gets true
294 * @wq_head: the waitqueue to wait on
295 * @condition: a C expression for the event to wait for
296 *
297 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
298 * @condition evaluates to true. The @condition is checked each time
299 * the waitqueue @wq_head is woken up.
300 *
301 * wake_up() has to be called after changing any variable that could
302 * change the result of the wait condition.
303 */
304 #define wait_event(wq_head, condition) \
305 do { \
306 might_sleep(); \
307 if (condition) \
308 break; \
309 __wait_event(wq_head, condition); \
310 } while (0)
311
312 #define __io_wait_event(wq_head, condition) \
313 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
314 io_schedule())
315
316 /*
317 * io_wait_event() -- like wait_event() but with io_schedule()
318 */
319 #define io_wait_event(wq_head, condition) \
320 do { \
321 might_sleep(); \
322 if (condition) \
323 break; \
324 __io_wait_event(wq_head, condition); \
325 } while (0)
326
327 #define __wait_event_freezable(wq_head, condition) \
328 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
329 freezable_schedule())
330
331 /**
332 * wait_event_freezable - sleep (or freeze) until a condition gets true
333 * @wq_head: the waitqueue to wait on
334 * @condition: a C expression for the event to wait for
335 *
336 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
337 * to system load) until the @condition evaluates to true. The
338 * @condition is checked each time the waitqueue @wq_head is woken up.
339 *
340 * wake_up() has to be called after changing any variable that could
341 * change the result of the wait condition.
342 */
343 #define wait_event_freezable(wq_head, condition) \
344 ({ \
345 int __ret = 0; \
346 might_sleep(); \
347 if (!(condition)) \
348 __ret = __wait_event_freezable(wq_head, condition); \
349 __ret; \
350 })
351
352 #define __wait_event_timeout(wq_head, condition, timeout) \
353 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
354 TASK_UNINTERRUPTIBLE, 0, timeout, \
355 __ret = schedule_timeout(__ret))
356
357 /**
358 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
359 * @wq_head: the waitqueue to wait on
360 * @condition: a C expression for the event to wait for
361 * @timeout: timeout, in jiffies
362 *
363 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
364 * @condition evaluates to true. The @condition is checked each time
365 * the waitqueue @wq_head is woken up.
366 *
367 * wake_up() has to be called after changing any variable that could
368 * change the result of the wait condition.
369 *
370 * Returns:
371 * 0 if the @condition evaluated to %false after the @timeout elapsed,
372 * 1 if the @condition evaluated to %true after the @timeout elapsed,
373 * or the remaining jiffies (at least 1) if the @condition evaluated
374 * to %true before the @timeout elapsed.
375 */
376 #define wait_event_timeout(wq_head, condition, timeout) \
377 ({ \
378 long __ret = timeout; \
379 might_sleep(); \
380 if (!___wait_cond_timeout(condition)) \
381 __ret = __wait_event_timeout(wq_head, condition, timeout); \
382 __ret; \
383 })
384
385 #define __wait_event_freezable_timeout(wq_head, condition, timeout) \
386 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
387 TASK_INTERRUPTIBLE, 0, timeout, \
388 __ret = freezable_schedule_timeout(__ret))
389
390 /*
391 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
392 * increasing load and is freezable.
393 */
394 #define wait_event_freezable_timeout(wq_head, condition, timeout) \
395 ({ \
396 long __ret = timeout; \
397 might_sleep(); \
398 if (!___wait_cond_timeout(condition)) \
399 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
400 __ret; \
401 })
402
403 #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
404 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
405 cmd1; schedule(); cmd2)
406 /*
407 * Just like wait_event_cmd(), except it sets exclusive flag
408 */
409 #define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
410 do { \
411 if (condition) \
412 break; \
413 __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2); \
414 } while (0)
415
416 #define __wait_event_cmd(wq_head, condition, cmd1, cmd2) \
417 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
418 cmd1; schedule(); cmd2)
419
420 /**
421 * wait_event_cmd - sleep until a condition gets true
422 * @wq_head: the waitqueue to wait on
423 * @condition: a C expression for the event to wait for
424 * @cmd1: the command will be executed before sleep
425 * @cmd2: the command will be executed after sleep
426 *
427 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
428 * @condition evaluates to true. The @condition is checked each time
429 * the waitqueue @wq_head is woken up.
430 *
431 * wake_up() has to be called after changing any variable that could
432 * change the result of the wait condition.
433 */
434 #define wait_event_cmd(wq_head, condition, cmd1, cmd2) \
435 do { \
436 if (condition) \
437 break; \
438 __wait_event_cmd(wq_head, condition, cmd1, cmd2); \
439 } while (0)
440
441 #define __wait_event_interruptible(wq_head, condition) \
442 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
443 schedule())
444
445 /**
446 * wait_event_interruptible - sleep until a condition gets true
447 * @wq_head: the waitqueue to wait on
448 * @condition: a C expression for the event to wait for
449 *
450 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
451 * @condition evaluates to true or a signal is received.
452 * The @condition is checked each time the waitqueue @wq_head is woken up.
453 *
454 * wake_up() has to be called after changing any variable that could
455 * change the result of the wait condition.
456 *
457 * The function will return -ERESTARTSYS if it was interrupted by a
458 * signal and 0 if @condition evaluated to true.
459 */
460 #define wait_event_interruptible(wq_head, condition) \
461 ({ \
462 int __ret = 0; \
463 might_sleep(); \
464 if (!(condition)) \
465 __ret = __wait_event_interruptible(wq_head, condition); \
466 __ret; \
467 })
468
469 #define __wait_event_interruptible_timeout(wq_head, condition, timeout) \
470 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
471 TASK_INTERRUPTIBLE, 0, timeout, \
472 __ret = schedule_timeout(__ret))
473
474 /**
475 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
476 * @wq_head: the waitqueue to wait on
477 * @condition: a C expression for the event to wait for
478 * @timeout: timeout, in jiffies
479 *
480 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
481 * @condition evaluates to true or a signal is received.
482 * The @condition is checked each time the waitqueue @wq_head is woken up.
483 *
484 * wake_up() has to be called after changing any variable that could
485 * change the result of the wait condition.
486 *
487 * Returns:
488 * 0 if the @condition evaluated to %false after the @timeout elapsed,
489 * 1 if the @condition evaluated to %true after the @timeout elapsed,
490 * the remaining jiffies (at least 1) if the @condition evaluated
491 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
492 * interrupted by a signal.
493 */
494 #define wait_event_interruptible_timeout(wq_head, condition, timeout) \
495 ({ \
496 long __ret = timeout; \
497 might_sleep(); \
498 if (!___wait_cond_timeout(condition)) \
499 __ret = __wait_event_interruptible_timeout(wq_head, \
500 condition, timeout); \
501 __ret; \
502 })
503
504 #define __wait_event_hrtimeout(wq_head, condition, timeout, state) \
505 ({ \
506 int __ret = 0; \
507 struct hrtimer_sleeper __t; \
508 \
509 hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
510 HRTIMER_MODE_REL); \
511 if ((timeout) != KTIME_MAX) \
512 hrtimer_start_range_ns(&__t.timer, timeout, \
513 current->timer_slack_ns, \
514 HRTIMER_MODE_REL); \
515 \
516 __ret = ___wait_event(wq_head, condition, state, 0, 0, \
517 if (!__t.task) { \
518 __ret = -ETIME; \
519 break; \
520 } \
521 schedule()); \
522 \
523 hrtimer_cancel(&__t.timer); \
524 destroy_hrtimer_on_stack(&__t.timer); \
525 __ret; \
526 })
527
528 /**
529 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
530 * @wq_head: the waitqueue to wait on
531 * @condition: a C expression for the event to wait for
532 * @timeout: timeout, as a ktime_t
533 *
534 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
535 * @condition evaluates to true or a signal is received.
536 * The @condition is checked each time the waitqueue @wq_head is woken up.
537 *
538 * wake_up() has to be called after changing any variable that could
539 * change the result of the wait condition.
540 *
541 * The function returns 0 if @condition became true, or -ETIME if the timeout
542 * elapsed.
543 */
544 #define wait_event_hrtimeout(wq_head, condition, timeout) \
545 ({ \
546 int __ret = 0; \
547 might_sleep(); \
548 if (!(condition)) \
549 __ret = __wait_event_hrtimeout(wq_head, condition, timeout, \
550 TASK_UNINTERRUPTIBLE); \
551 __ret; \
552 })
553
554 /**
555 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
556 * @wq: the waitqueue to wait on
557 * @condition: a C expression for the event to wait for
558 * @timeout: timeout, as a ktime_t
559 *
560 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
561 * @condition evaluates to true or a signal is received.
562 * The @condition is checked each time the waitqueue @wq is woken up.
563 *
564 * wake_up() has to be called after changing any variable that could
565 * change the result of the wait condition.
566 *
567 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
568 * interrupted by a signal, or -ETIME if the timeout elapsed.
569 */
570 #define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
571 ({ \
572 long __ret = 0; \
573 might_sleep(); \
574 if (!(condition)) \
575 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
576 TASK_INTERRUPTIBLE); \
577 __ret; \
578 })
579
580 #define __wait_event_interruptible_exclusive(wq, condition) \
581 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
582 schedule())
583
584 #define wait_event_interruptible_exclusive(wq, condition) \
585 ({ \
586 int __ret = 0; \
587 might_sleep(); \
588 if (!(condition)) \
589 __ret = __wait_event_interruptible_exclusive(wq, condition); \
590 __ret; \
591 })
592
593 #define __wait_event_killable_exclusive(wq, condition) \
594 ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
595 schedule())
596
597 #define wait_event_killable_exclusive(wq, condition) \
598 ({ \
599 int __ret = 0; \
600 might_sleep(); \
601 if (!(condition)) \
602 __ret = __wait_event_killable_exclusive(wq, condition); \
603 __ret; \
604 })
605
606
607 #define __wait_event_freezable_exclusive(wq, condition) \
608 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
609 freezable_schedule())
610
611 #define wait_event_freezable_exclusive(wq, condition) \
612 ({ \
613 int __ret = 0; \
614 might_sleep(); \
615 if (!(condition)) \
616 __ret = __wait_event_freezable_exclusive(wq, condition); \
617 __ret; \
618 })
619
620 /**
621 * wait_event_idle - wait for a condition without contributing to system load
622 * @wq_head: the waitqueue to wait on
623 * @condition: a C expression for the event to wait for
624 *
625 * The process is put to sleep (TASK_IDLE) until the
626 * @condition evaluates to true.
627 * The @condition is checked each time the waitqueue @wq_head is woken up.
628 *
629 * wake_up() has to be called after changing any variable that could
630 * change the result of the wait condition.
631 *
632 */
633 #define wait_event_idle(wq_head, condition) \
634 do { \
635 might_sleep(); \
636 if (!(condition)) \
637 ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule()); \
638 } while (0)
639
640 /**
641 * wait_event_idle_exclusive - wait for a condition with contributing to system load
642 * @wq_head: the waitqueue to wait on
643 * @condition: a C expression for the event to wait for
644 *
645 * The process is put to sleep (TASK_IDLE) until the
646 * @condition evaluates to true.
647 * The @condition is checked each time the waitqueue @wq_head is woken up.
648 *
649 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
650 * set thus if other processes wait on the same list, when this
651 * process is woken further processes are not considered.
652 *
653 * wake_up() has to be called after changing any variable that could
654 * change the result of the wait condition.
655 *
656 */
657 #define wait_event_idle_exclusive(wq_head, condition) \
658 do { \
659 might_sleep(); \
660 if (!(condition)) \
661 ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \
662 } while (0)
663
664 #define __wait_event_idle_timeout(wq_head, condition, timeout) \
665 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
666 TASK_IDLE, 0, timeout, \
667 __ret = schedule_timeout(__ret))
668
669 /**
670 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
671 * @wq_head: the waitqueue to wait on
672 * @condition: a C expression for the event to wait for
673 * @timeout: timeout, in jiffies
674 *
675 * The process is put to sleep (TASK_IDLE) until the
676 * @condition evaluates to true. The @condition is checked each time
677 * the waitqueue @wq_head is woken up.
678 *
679 * wake_up() has to be called after changing any variable that could
680 * change the result of the wait condition.
681 *
682 * Returns:
683 * 0 if the @condition evaluated to %false after the @timeout elapsed,
684 * 1 if the @condition evaluated to %true after the @timeout elapsed,
685 * or the remaining jiffies (at least 1) if the @condition evaluated
686 * to %true before the @timeout elapsed.
687 */
688 #define wait_event_idle_timeout(wq_head, condition, timeout) \
689 ({ \
690 long __ret = timeout; \
691 might_sleep(); \
692 if (!___wait_cond_timeout(condition)) \
693 __ret = __wait_event_idle_timeout(wq_head, condition, timeout); \
694 __ret; \
695 })
696
697 #define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
698 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
699 TASK_IDLE, 1, timeout, \
700 __ret = schedule_timeout(__ret))
701
702 /**
703 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
704 * @wq_head: the waitqueue to wait on
705 * @condition: a C expression for the event to wait for
706 * @timeout: timeout, in jiffies
707 *
708 * The process is put to sleep (TASK_IDLE) until the
709 * @condition evaluates to true. The @condition is checked each time
710 * the waitqueue @wq_head is woken up.
711 *
712 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
713 * set thus if other processes wait on the same list, when this
714 * process is woken further processes are not considered.
715 *
716 * wake_up() has to be called after changing any variable that could
717 * change the result of the wait condition.
718 *
719 * Returns:
720 * 0 if the @condition evaluated to %false after the @timeout elapsed,
721 * 1 if the @condition evaluated to %true after the @timeout elapsed,
722 * or the remaining jiffies (at least 1) if the @condition evaluated
723 * to %true before the @timeout elapsed.
724 */
725 #define wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
726 ({ \
727 long __ret = timeout; \
728 might_sleep(); \
729 if (!___wait_cond_timeout(condition)) \
730 __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
731 __ret; \
732 })
733
734 extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
735 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
736
737 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
738 ({ \
739 int __ret; \
740 DEFINE_WAIT(__wait); \
741 if (exclusive) \
742 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
743 do { \
744 __ret = fn(&(wq), &__wait); \
745 if (__ret) \
746 break; \
747 } while (!(condition)); \
748 __remove_wait_queue(&(wq), &__wait); \
749 __set_current_state(TASK_RUNNING); \
750 __ret; \
751 })
752
753
754 /**
755 * wait_event_interruptible_locked - sleep until a condition gets true
756 * @wq: the waitqueue to wait on
757 * @condition: a C expression for the event to wait for
758 *
759 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
760 * @condition evaluates to true or a signal is received.
761 * The @condition is checked each time the waitqueue @wq is woken up.
762 *
763 * It must be called with wq.lock being held. This spinlock is
764 * unlocked while sleeping but @condition testing is done while lock
765 * is held and when this macro exits the lock is held.
766 *
767 * The lock is locked/unlocked using spin_lock()/spin_unlock()
768 * functions which must match the way they are locked/unlocked outside
769 * of this macro.
770 *
771 * wake_up_locked() has to be called after changing any variable that could
772 * change the result of the wait condition.
773 *
774 * The function will return -ERESTARTSYS if it was interrupted by a
775 * signal and 0 if @condition evaluated to true.
776 */
777 #define wait_event_interruptible_locked(wq, condition) \
778 ((condition) \
779 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
780
781 /**
782 * wait_event_interruptible_locked_irq - sleep until a condition gets true
783 * @wq: the waitqueue to wait on
784 * @condition: a C expression for the event to wait for
785 *
786 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
787 * @condition evaluates to true or a signal is received.
788 * The @condition is checked each time the waitqueue @wq is woken up.
789 *
790 * It must be called with wq.lock being held. This spinlock is
791 * unlocked while sleeping but @condition testing is done while lock
792 * is held and when this macro exits the lock is held.
793 *
794 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
795 * functions which must match the way they are locked/unlocked outside
796 * of this macro.
797 *
798 * wake_up_locked() has to be called after changing any variable that could
799 * change the result of the wait condition.
800 *
801 * The function will return -ERESTARTSYS if it was interrupted by a
802 * signal and 0 if @condition evaluated to true.
803 */
804 #define wait_event_interruptible_locked_irq(wq, condition) \
805 ((condition) \
806 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
807
808 /**
809 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
810 * @wq: the waitqueue to wait on
811 * @condition: a C expression for the event to wait for
812 *
813 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
814 * @condition evaluates to true or a signal is received.
815 * The @condition is checked each time the waitqueue @wq is woken up.
816 *
817 * It must be called with wq.lock being held. This spinlock is
818 * unlocked while sleeping but @condition testing is done while lock
819 * is held and when this macro exits the lock is held.
820 *
821 * The lock is locked/unlocked using spin_lock()/spin_unlock()
822 * functions which must match the way they are locked/unlocked outside
823 * of this macro.
824 *
825 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
826 * set thus when other process waits process on the list if this
827 * process is awaken further processes are not considered.
828 *
829 * wake_up_locked() has to be called after changing any variable that could
830 * change the result of the wait condition.
831 *
832 * The function will return -ERESTARTSYS if it was interrupted by a
833 * signal and 0 if @condition evaluated to true.
834 */
835 #define wait_event_interruptible_exclusive_locked(wq, condition) \
836 ((condition) \
837 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
838
839 /**
840 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
841 * @wq: the waitqueue to wait on
842 * @condition: a C expression for the event to wait for
843 *
844 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
845 * @condition evaluates to true or a signal is received.
846 * The @condition is checked each time the waitqueue @wq is woken up.
847 *
848 * It must be called with wq.lock being held. This spinlock is
849 * unlocked while sleeping but @condition testing is done while lock
850 * is held and when this macro exits the lock is held.
851 *
852 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
853 * functions which must match the way they are locked/unlocked outside
854 * of this macro.
855 *
856 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
857 * set thus when other process waits process on the list if this
858 * process is awaken further processes are not considered.
859 *
860 * wake_up_locked() has to be called after changing any variable that could
861 * change the result of the wait condition.
862 *
863 * The function will return -ERESTARTSYS if it was interrupted by a
864 * signal and 0 if @condition evaluated to true.
865 */
866 #define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
867 ((condition) \
868 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
869
870
871 #define __wait_event_killable(wq, condition) \
872 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
873
874 /**
875 * wait_event_killable - sleep until a condition gets true
876 * @wq_head: the waitqueue to wait on
877 * @condition: a C expression for the event to wait for
878 *
879 * The process is put to sleep (TASK_KILLABLE) until the
880 * @condition evaluates to true or a signal is received.
881 * The @condition is checked each time the waitqueue @wq_head is woken up.
882 *
883 * wake_up() has to be called after changing any variable that could
884 * change the result of the wait condition.
885 *
886 * The function will return -ERESTARTSYS if it was interrupted by a
887 * signal and 0 if @condition evaluated to true.
888 */
889 #define wait_event_killable(wq_head, condition) \
890 ({ \
891 int __ret = 0; \
892 might_sleep(); \
893 if (!(condition)) \
894 __ret = __wait_event_killable(wq_head, condition); \
895 __ret; \
896 })
897
898 #define __wait_event_killable_timeout(wq_head, condition, timeout) \
899 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
900 TASK_KILLABLE, 0, timeout, \
901 __ret = schedule_timeout(__ret))
902
903 /**
904 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
905 * @wq_head: the waitqueue to wait on
906 * @condition: a C expression for the event to wait for
907 * @timeout: timeout, in jiffies
908 *
909 * The process is put to sleep (TASK_KILLABLE) until the
910 * @condition evaluates to true or a kill signal is received.
911 * The @condition is checked each time the waitqueue @wq_head is woken up.
912 *
913 * wake_up() has to be called after changing any variable that could
914 * change the result of the wait condition.
915 *
916 * Returns:
917 * 0 if the @condition evaluated to %false after the @timeout elapsed,
918 * 1 if the @condition evaluated to %true after the @timeout elapsed,
919 * the remaining jiffies (at least 1) if the @condition evaluated
920 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
921 * interrupted by a kill signal.
922 *
923 * Only kill signals interrupt this process.
924 */
925 #define wait_event_killable_timeout(wq_head, condition, timeout) \
926 ({ \
927 long __ret = timeout; \
928 might_sleep(); \
929 if (!___wait_cond_timeout(condition)) \
930 __ret = __wait_event_killable_timeout(wq_head, \
931 condition, timeout); \
932 __ret; \
933 })
934
935
936 #define __wait_event_lock_irq(wq_head, condition, lock, cmd) \
937 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
938 spin_unlock_irq(&lock); \
939 cmd; \
940 schedule(); \
941 spin_lock_irq(&lock))
942
943 /**
944 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
945 * condition is checked under the lock. This
946 * is expected to be called with the lock
947 * taken.
948 * @wq_head: the waitqueue to wait on
949 * @condition: a C expression for the event to wait for
950 * @lock: a locked spinlock_t, which will be released before cmd
951 * and schedule() and reacquired afterwards.
952 * @cmd: a command which is invoked outside the critical section before
953 * sleep
954 *
955 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
956 * @condition evaluates to true. The @condition is checked each time
957 * the waitqueue @wq_head is woken up.
958 *
959 * wake_up() has to be called after changing any variable that could
960 * change the result of the wait condition.
961 *
962 * This is supposed to be called while holding the lock. The lock is
963 * dropped before invoking the cmd and going to sleep and is reacquired
964 * afterwards.
965 */
966 #define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd) \
967 do { \
968 if (condition) \
969 break; \
970 __wait_event_lock_irq(wq_head, condition, lock, cmd); \
971 } while (0)
972
973 /**
974 * wait_event_lock_irq - sleep until a condition gets true. The
975 * condition is checked under the lock. This
976 * is expected to be called with the lock
977 * taken.
978 * @wq_head: the waitqueue to wait on
979 * @condition: a C expression for the event to wait for
980 * @lock: a locked spinlock_t, which will be released before schedule()
981 * and reacquired afterwards.
982 *
983 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
984 * @condition evaluates to true. The @condition is checked each time
985 * the waitqueue @wq_head is woken up.
986 *
987 * wake_up() has to be called after changing any variable that could
988 * change the result of the wait condition.
989 *
990 * This is supposed to be called while holding the lock. The lock is
991 * dropped before going to sleep and is reacquired afterwards.
992 */
993 #define wait_event_lock_irq(wq_head, condition, lock) \
994 do { \
995 if (condition) \
996 break; \
997 __wait_event_lock_irq(wq_head, condition, lock, ); \
998 } while (0)
999
1000
1001 #define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd) \
1002 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
1003 spin_unlock_irq(&lock); \
1004 cmd; \
1005 schedule(); \
1006 spin_lock_irq(&lock))
1007
1008 /**
1009 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
1010 * The condition is checked under the lock. This is expected to
1011 * be called with the lock taken.
1012 * @wq_head: the waitqueue to wait on
1013 * @condition: a C expression for the event to wait for
1014 * @lock: a locked spinlock_t, which will be released before cmd and
1015 * schedule() and reacquired afterwards.
1016 * @cmd: a command which is invoked outside the critical section before
1017 * sleep
1018 *
1019 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1020 * @condition evaluates to true or a signal is received. The @condition is
1021 * checked each time the waitqueue @wq_head is woken up.
1022 *
1023 * wake_up() has to be called after changing any variable that could
1024 * change the result of the wait condition.
1025 *
1026 * This is supposed to be called while holding the lock. The lock is
1027 * dropped before invoking the cmd and going to sleep and is reacquired
1028 * afterwards.
1029 *
1030 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1031 * and 0 if @condition evaluated to true.
1032 */
1033 #define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd) \
1034 ({ \
1035 int __ret = 0; \
1036 if (!(condition)) \
1037 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1038 condition, lock, cmd); \
1039 __ret; \
1040 })
1041
1042 /**
1043 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
1044 * The condition is checked under the lock. This is expected
1045 * to be called with the lock taken.
1046 * @wq_head: the waitqueue to wait on
1047 * @condition: a C expression for the event to wait for
1048 * @lock: a locked spinlock_t, which will be released before schedule()
1049 * and reacquired afterwards.
1050 *
1051 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1052 * @condition evaluates to true or signal is received. The @condition is
1053 * checked each time the waitqueue @wq_head is woken up.
1054 *
1055 * wake_up() has to be called after changing any variable that could
1056 * change the result of the wait condition.
1057 *
1058 * This is supposed to be called while holding the lock. The lock is
1059 * dropped before going to sleep and is reacquired afterwards.
1060 *
1061 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1062 * and 0 if @condition evaluated to true.
1063 */
1064 #define wait_event_interruptible_lock_irq(wq_head, condition, lock) \
1065 ({ \
1066 int __ret = 0; \
1067 if (!(condition)) \
1068 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1069 condition, lock,); \
1070 __ret; \
1071 })
1072
1073 #define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state) \
1074 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
1075 state, 0, timeout, \
1076 spin_unlock_irq(&lock); \
1077 __ret = schedule_timeout(__ret); \
1078 spin_lock_irq(&lock));
1079
1080 /**
1081 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
1082 * true or a timeout elapses. The condition is checked under
1083 * the lock. This is expected to be called with the lock taken.
1084 * @wq_head: the waitqueue to wait on
1085 * @condition: a C expression for the event to wait for
1086 * @lock: a locked spinlock_t, which will be released before schedule()
1087 * and reacquired afterwards.
1088 * @timeout: timeout, in jiffies
1089 *
1090 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1091 * @condition evaluates to true or signal is received. The @condition is
1092 * checked each time the waitqueue @wq_head is woken up.
1093 *
1094 * wake_up() has to be called after changing any variable that could
1095 * change the result of the wait condition.
1096 *
1097 * This is supposed to be called while holding the lock. The lock is
1098 * dropped before going to sleep and is reacquired afterwards.
1099 *
1100 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
1101 * was interrupted by a signal, and the remaining jiffies otherwise
1102 * if the condition evaluated to true before the timeout elapsed.
1103 */
1104 #define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock, \
1105 timeout) \
1106 ({ \
1107 long __ret = timeout; \
1108 if (!___wait_cond_timeout(condition)) \
1109 __ret = __wait_event_lock_irq_timeout( \
1110 wq_head, condition, lock, timeout, \
1111 TASK_INTERRUPTIBLE); \
1112 __ret; \
1113 })
1114
1115 #define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout) \
1116 ({ \
1117 long __ret = timeout; \
1118 if (!___wait_cond_timeout(condition)) \
1119 __ret = __wait_event_lock_irq_timeout( \
1120 wq_head, condition, lock, timeout, \
1121 TASK_UNINTERRUPTIBLE); \
1122 __ret; \
1123 })
1124
1125 /*
1126 * Waitqueues which are removed from the waitqueue_head at wakeup time
1127 */
1128 void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1129 void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1130 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1131 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
1132 long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
1133 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1134 int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1135
1136 #define DEFINE_WAIT_FUNC(name, function) \
1137 struct wait_queue_entry name = { \
1138 .private = current, \
1139 .func = function, \
1140 .entry = LIST_HEAD_INIT((name).entry), \
1141 }
1142
1143 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1144
1145 #define init_wait(wait) \
1146 do { \
1147 (wait)->private = current; \
1148 (wait)->func = autoremove_wake_function; \
1149 INIT_LIST_HEAD(&(wait)->entry); \
1150 (wait)->flags = 0; \
1151 } while (0)
1152
1153 bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg);
1154
1155 #endif /* _LINUX_WAIT_H */
1156