4 * Linux wait queue related types and methods
6 #include <linux/list.h>
7 #include <linux/stddef.h>
8 #include <linux/spinlock.h>
9 #include <asm/current.h>
10 #include <uapi/linux/wait.h>
12 typedef struct __wait_queue wait_queue_t;
13 typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
14 int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key);
16 /* __wait_queue::flags */
17 #define WQ_FLAG_EXCLUSIVE 0x01
18 #define WQ_FLAG_WOKEN 0x02
23 wait_queue_func_t func;
24 struct list_head task_list;
30 #define WAIT_ATOMIC_T_BIT_NR -1
31 unsigned long timeout;
34 struct wait_bit_queue {
35 struct wait_bit_key key;
39 struct __wait_queue_head {
41 struct list_head task_list;
43 typedef struct __wait_queue_head wait_queue_head_t;
48 * Macros for declaration and initialisaton of the datatypes
51 #define __WAITQUEUE_INITIALIZER(name, tsk) { \
53 .func = default_wake_function, \
54 .task_list = { NULL, NULL } }
56 #define DECLARE_WAITQUEUE(name, tsk) \
57 wait_queue_t name = __WAITQUEUE_INITIALIZER(name, tsk)
59 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
60 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
61 .task_list = { &(name).task_list, &(name).task_list } }
63 #define DECLARE_WAIT_QUEUE_HEAD(name) \
64 wait_queue_head_t name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
66 #define __WAIT_BIT_KEY_INITIALIZER(word, bit) \
67 { .flags = word, .bit_nr = bit, }
69 #define __WAIT_ATOMIC_T_KEY_INITIALIZER(p) \
70 { .flags = p, .bit_nr = WAIT_ATOMIC_T_BIT_NR, }
72 extern void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *);
74 #define init_waitqueue_head(q) \
76 static struct lock_class_key __key; \
78 __init_waitqueue_head((q), #q, &__key); \
82 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
83 ({ init_waitqueue_head(&name); name; })
84 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
85 wait_queue_head_t name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
87 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
90 static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p)
94 q->func = default_wake_function;
98 init_waitqueue_func_entry(wait_queue_t *q, wait_queue_func_t func)
106 * waitqueue_active -- locklessly test for waiters on the queue
107 * @q: the waitqueue to test for waiters
109 * returns true if the wait list is not empty
111 * NOTE: this function is lockless and requires care, incorrect usage _will_
112 * lead to sporadic and non-obvious failure.
114 * Use either while holding wait_queue_head_t::lock or when used for wakeups
115 * with an extra smp_mb() like:
117 * CPU0 - waker CPU1 - waiter
120 * @cond = true; prepare_to_wait(&wq, &wait, state);
121 * smp_mb(); // smp_mb() from set_current_state()
122 * if (waitqueue_active(wq)) if (@cond)
123 * wake_up(wq); break;
126 * finish_wait(&wq, &wait);
128 * Because without the explicit smp_mb() it's possible for the
129 * waitqueue_active() load to get hoisted over the @cond store such that we'll
130 * observe an empty wait list while the waiter might not observe @cond.
132 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
133 * which (when the lock is uncontended) are of roughly equal cost.
135 static inline int waitqueue_active(wait_queue_head_t *q)
137 return !list_empty(&q->task_list);
141 * wq_has_sleeper - check if there are any waiting processes
142 * @wq: wait queue head
144 * Returns true if wq has waiting processes
146 * Please refer to the comment for waitqueue_active.
148 static inline bool wq_has_sleeper(wait_queue_head_t *wq)
151 * We need to be sure we are in sync with the
152 * add_wait_queue modifications to the wait queue.
154 * This memory barrier should be paired with one on the
158 return waitqueue_active(wq);
161 extern void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
162 extern void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait);
163 extern void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
165 static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
167 list_add(&new->task_list, &head->task_list);
171 * Used for wake-one threads:
174 __add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
176 wait->flags |= WQ_FLAG_EXCLUSIVE;
177 __add_wait_queue(q, wait);
180 static inline void __add_wait_queue_tail(wait_queue_head_t *head,
183 list_add_tail(&new->task_list, &head->task_list);
187 __add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
189 wait->flags |= WQ_FLAG_EXCLUSIVE;
190 __add_wait_queue_tail(q, wait);
194 __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)
196 list_del(&old->task_list);
199 typedef int wait_bit_action_f(struct wait_bit_key *, int mode);
200 void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
201 void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
202 void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
203 void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
204 void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
205 void __wake_up_bit(wait_queue_head_t *, void *, int);
206 int __wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
207 int __wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
208 void wake_up_bit(void *, int);
209 void wake_up_atomic_t(atomic_t *);
210 int out_of_line_wait_on_bit(void *, int, wait_bit_action_f *, unsigned);
211 int out_of_line_wait_on_bit_timeout(void *, int, wait_bit_action_f *, unsigned, unsigned long);
212 int out_of_line_wait_on_bit_lock(void *, int, wait_bit_action_f *, unsigned);
213 int out_of_line_wait_on_atomic_t(atomic_t *, int (*)(atomic_t *), unsigned);
214 wait_queue_head_t *bit_waitqueue(void *, int);
216 #define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
217 #define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
218 #define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
219 #define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
220 #define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
222 #define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
223 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
224 #define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
225 #define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
228 * Wakeup macros to be used to report events to the targets.
230 #define wake_up_poll(x, m) \
231 __wake_up(x, TASK_NORMAL, 1, (void *) (m))
232 #define wake_up_locked_poll(x, m) \
233 __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
234 #define wake_up_interruptible_poll(x, m) \
235 __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
236 #define wake_up_interruptible_sync_poll(x, m) \
237 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
239 #define ___wait_cond_timeout(condition) \
241 bool __cond = (condition); \
242 if (__cond && !__ret) \
247 #define ___wait_is_interruptible(state) \
248 (!__builtin_constant_p(state) || \
249 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
252 * The below macro ___wait_event() has an explicit shadow of the __ret
253 * variable when used from the wait_event_*() macros.
255 * This is so that both can use the ___wait_cond_timeout() construct
256 * to wrap the condition.
258 * The type inconsistency of the wait_event_*() __ret variable is also
259 * on purpose; we use long where we can return timeout values and int
263 #define ___wait_event(wq, condition, state, exclusive, ret, cmd) \
266 wait_queue_t __wait; \
267 long __ret = ret; /* explicit shadow */ \
269 INIT_LIST_HEAD(&__wait.task_list); \
271 __wait.flags = WQ_FLAG_EXCLUSIVE; \
276 long __int = prepare_to_wait_event(&wq, &__wait, state);\
281 if (___wait_is_interruptible(state) && __int) { \
288 finish_wait(&wq, &__wait); \
292 #define __wait_event(wq, condition) \
293 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
297 * wait_event - sleep until a condition gets true
298 * @wq: the waitqueue to wait on
299 * @condition: a C expression for the event to wait for
301 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
302 * @condition evaluates to true. The @condition is checked each time
303 * the waitqueue @wq is woken up.
305 * wake_up() has to be called after changing any variable that could
306 * change the result of the wait condition.
308 #define wait_event(wq, condition) \
313 __wait_event(wq, condition); \
316 #define __io_wait_event(wq, condition) \
317 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
321 * io_wait_event() -- like wait_event() but with io_schedule()
323 #define io_wait_event(wq, condition) \
328 __io_wait_event(wq, condition); \
331 #define __wait_event_freezable(wq, condition) \
332 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
333 schedule(); try_to_freeze())
336 * wait_event_freezable - sleep (or freeze) until a condition gets true
337 * @wq: the waitqueue to wait on
338 * @condition: a C expression for the event to wait for
340 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
341 * to system load) until the @condition evaluates to true. The
342 * @condition is checked each time the waitqueue @wq is woken up.
344 * wake_up() has to be called after changing any variable that could
345 * change the result of the wait condition.
347 #define wait_event_freezable(wq, condition) \
352 __ret = __wait_event_freezable(wq, condition); \
356 #define __wait_event_timeout(wq, condition, timeout) \
357 ___wait_event(wq, ___wait_cond_timeout(condition), \
358 TASK_UNINTERRUPTIBLE, 0, timeout, \
359 __ret = schedule_timeout(__ret))
362 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
363 * @wq: the waitqueue to wait on
364 * @condition: a C expression for the event to wait for
365 * @timeout: timeout, in jiffies
367 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
368 * @condition evaluates to true. The @condition is checked each time
369 * the waitqueue @wq is woken up.
371 * wake_up() has to be called after changing any variable that could
372 * change the result of the wait condition.
375 * 0 if the @condition evaluated to %false after the @timeout elapsed,
376 * 1 if the @condition evaluated to %true after the @timeout elapsed,
377 * or the remaining jiffies (at least 1) if the @condition evaluated
378 * to %true before the @timeout elapsed.
380 #define wait_event_timeout(wq, condition, timeout) \
382 long __ret = timeout; \
384 if (!___wait_cond_timeout(condition)) \
385 __ret = __wait_event_timeout(wq, condition, timeout); \
389 #define __wait_event_freezable_timeout(wq, condition, timeout) \
390 ___wait_event(wq, ___wait_cond_timeout(condition), \
391 TASK_INTERRUPTIBLE, 0, timeout, \
392 __ret = schedule_timeout(__ret); try_to_freeze())
395 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
396 * increasing load and is freezable.
398 #define wait_event_freezable_timeout(wq, condition, timeout) \
400 long __ret = timeout; \
402 if (!___wait_cond_timeout(condition)) \
403 __ret = __wait_event_freezable_timeout(wq, condition, timeout); \
407 #define __wait_event_exclusive_cmd(wq, condition, cmd1, cmd2) \
408 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
409 cmd1; schedule(); cmd2)
411 * Just like wait_event_cmd(), except it sets exclusive flag
413 #define wait_event_exclusive_cmd(wq, condition, cmd1, cmd2) \
417 __wait_event_exclusive_cmd(wq, condition, cmd1, cmd2); \
420 #define __wait_event_cmd(wq, condition, cmd1, cmd2) \
421 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
422 cmd1; schedule(); cmd2)
425 * wait_event_cmd - sleep until a condition gets true
426 * @wq: the waitqueue to wait on
427 * @condition: a C expression for the event to wait for
428 * @cmd1: the command will be executed before sleep
429 * @cmd2: the command will be executed after sleep
431 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
432 * @condition evaluates to true. The @condition is checked each time
433 * the waitqueue @wq is woken up.
435 * wake_up() has to be called after changing any variable that could
436 * change the result of the wait condition.
438 #define wait_event_cmd(wq, condition, cmd1, cmd2) \
442 __wait_event_cmd(wq, condition, cmd1, cmd2); \
445 #define __wait_event_interruptible(wq, condition) \
446 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
450 * wait_event_interruptible - sleep until a condition gets true
451 * @wq: the waitqueue to wait on
452 * @condition: a C expression for the event to wait for
454 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
455 * @condition evaluates to true or a signal is received.
456 * The @condition is checked each time the waitqueue @wq is woken up.
458 * wake_up() has to be called after changing any variable that could
459 * change the result of the wait condition.
461 * The function will return -ERESTARTSYS if it was interrupted by a
462 * signal and 0 if @condition evaluated to true.
464 #define wait_event_interruptible(wq, condition) \
469 __ret = __wait_event_interruptible(wq, condition); \
473 #define __wait_event_interruptible_timeout(wq, condition, timeout) \
474 ___wait_event(wq, ___wait_cond_timeout(condition), \
475 TASK_INTERRUPTIBLE, 0, timeout, \
476 __ret = schedule_timeout(__ret))
479 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
480 * @wq: the waitqueue to wait on
481 * @condition: a C expression for the event to wait for
482 * @timeout: timeout, in jiffies
484 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
485 * @condition evaluates to true or a signal is received.
486 * The @condition is checked each time the waitqueue @wq is woken up.
488 * wake_up() has to be called after changing any variable that could
489 * change the result of the wait condition.
492 * 0 if the @condition evaluated to %false after the @timeout elapsed,
493 * 1 if the @condition evaluated to %true after the @timeout elapsed,
494 * the remaining jiffies (at least 1) if the @condition evaluated
495 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
496 * interrupted by a signal.
498 #define wait_event_interruptible_timeout(wq, condition, timeout) \
500 long __ret = timeout; \
502 if (!___wait_cond_timeout(condition)) \
503 __ret = __wait_event_interruptible_timeout(wq, \
504 condition, timeout); \
508 #define __wait_event_hrtimeout(wq, condition, timeout, state) \
511 struct hrtimer_sleeper __t; \
513 hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, \
515 hrtimer_init_sleeper(&__t, current); \
516 if ((timeout).tv64 != KTIME_MAX) \
517 hrtimer_start_range_ns(&__t.timer, timeout, \
518 current->timer_slack_ns, \
521 __ret = ___wait_event(wq, condition, state, 0, 0, \
528 hrtimer_cancel(&__t.timer); \
529 destroy_hrtimer_on_stack(&__t.timer); \
534 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
535 * @wq: the waitqueue to wait on
536 * @condition: a C expression for the event to wait for
537 * @timeout: timeout, as a ktime_t
539 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
540 * @condition evaluates to true or a signal is received.
541 * The @condition is checked each time the waitqueue @wq is woken up.
543 * wake_up() has to be called after changing any variable that could
544 * change the result of the wait condition.
546 * The function returns 0 if @condition became true, or -ETIME if the timeout
549 #define wait_event_hrtimeout(wq, condition, timeout) \
554 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
555 TASK_UNINTERRUPTIBLE); \
560 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
561 * @wq: the waitqueue to wait on
562 * @condition: a C expression for the event to wait for
563 * @timeout: timeout, as a ktime_t
565 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
566 * @condition evaluates to true or a signal is received.
567 * The @condition is checked each time the waitqueue @wq is woken up.
569 * wake_up() has to be called after changing any variable that could
570 * change the result of the wait condition.
572 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
573 * interrupted by a signal, or -ETIME if the timeout elapsed.
575 #define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
580 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
581 TASK_INTERRUPTIBLE); \
585 #define __wait_event_interruptible_exclusive(wq, condition) \
586 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
589 #define wait_event_interruptible_exclusive(wq, condition) \
594 __ret = __wait_event_interruptible_exclusive(wq, condition);\
598 #define __wait_event_killable_exclusive(wq, condition) \
599 ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
602 #define wait_event_killable_exclusive(wq, condition) \
607 __ret = __wait_event_killable_exclusive(wq, condition); \
612 #define __wait_event_freezable_exclusive(wq, condition) \
613 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
614 schedule(); try_to_freeze())
616 #define wait_event_freezable_exclusive(wq, condition) \
621 __ret = __wait_event_freezable_exclusive(wq, condition);\
626 #define __wait_event_interruptible_locked(wq, condition, exclusive, irq) \
629 DEFINE_WAIT(__wait); \
631 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
633 if (likely(list_empty(&__wait.task_list))) \
634 __add_wait_queue_tail(&(wq), &__wait); \
635 set_current_state(TASK_INTERRUPTIBLE); \
636 if (signal_pending(current)) { \
637 __ret = -ERESTARTSYS; \
641 spin_unlock_irq(&(wq).lock); \
643 spin_unlock(&(wq).lock); \
646 spin_lock_irq(&(wq).lock); \
648 spin_lock(&(wq).lock); \
649 } while (!(condition)); \
650 __remove_wait_queue(&(wq), &__wait); \
651 __set_current_state(TASK_RUNNING); \
657 * wait_event_interruptible_locked - sleep until a condition gets true
658 * @wq: the waitqueue to wait on
659 * @condition: a C expression for the event to wait for
661 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
662 * @condition evaluates to true or a signal is received.
663 * The @condition is checked each time the waitqueue @wq is woken up.
665 * It must be called with wq.lock being held. This spinlock is
666 * unlocked while sleeping but @condition testing is done while lock
667 * is held and when this macro exits the lock is held.
669 * The lock is locked/unlocked using spin_lock()/spin_unlock()
670 * functions which must match the way they are locked/unlocked outside
673 * wake_up_locked() has to be called after changing any variable that could
674 * change the result of the wait condition.
676 * The function will return -ERESTARTSYS if it was interrupted by a
677 * signal and 0 if @condition evaluated to true.
679 #define wait_event_interruptible_locked(wq, condition) \
681 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 0))
684 * wait_event_interruptible_locked_irq - sleep until a condition gets true
685 * @wq: the waitqueue to wait on
686 * @condition: a C expression for the event to wait for
688 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
689 * @condition evaluates to true or a signal is received.
690 * The @condition is checked each time the waitqueue @wq is woken up.
692 * It must be called with wq.lock being held. This spinlock is
693 * unlocked while sleeping but @condition testing is done while lock
694 * is held and when this macro exits the lock is held.
696 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
697 * functions which must match the way they are locked/unlocked outside
700 * wake_up_locked() has to be called after changing any variable that could
701 * change the result of the wait condition.
703 * The function will return -ERESTARTSYS if it was interrupted by a
704 * signal and 0 if @condition evaluated to true.
706 #define wait_event_interruptible_locked_irq(wq, condition) \
708 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 1))
711 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
712 * @wq: the waitqueue to wait on
713 * @condition: a C expression for the event to wait for
715 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
716 * @condition evaluates to true or a signal is received.
717 * The @condition is checked each time the waitqueue @wq is woken up.
719 * It must be called with wq.lock being held. This spinlock is
720 * unlocked while sleeping but @condition testing is done while lock
721 * is held and when this macro exits the lock is held.
723 * The lock is locked/unlocked using spin_lock()/spin_unlock()
724 * functions which must match the way they are locked/unlocked outside
727 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
728 * set thus when other process waits process on the list if this
729 * process is awaken further processes are not considered.
731 * wake_up_locked() has to be called after changing any variable that could
732 * change the result of the wait condition.
734 * The function will return -ERESTARTSYS if it was interrupted by a
735 * signal and 0 if @condition evaluated to true.
737 #define wait_event_interruptible_exclusive_locked(wq, condition) \
739 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 0))
742 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
743 * @wq: the waitqueue to wait on
744 * @condition: a C expression for the event to wait for
746 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
747 * @condition evaluates to true or a signal is received.
748 * The @condition is checked each time the waitqueue @wq is woken up.
750 * It must be called with wq.lock being held. This spinlock is
751 * unlocked while sleeping but @condition testing is done while lock
752 * is held and when this macro exits the lock is held.
754 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
755 * functions which must match the way they are locked/unlocked outside
758 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
759 * set thus when other process waits process on the list if this
760 * process is awaken further processes are not considered.
762 * wake_up_locked() has to be called after changing any variable that could
763 * change the result of the wait condition.
765 * The function will return -ERESTARTSYS if it was interrupted by a
766 * signal and 0 if @condition evaluated to true.
768 #define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
770 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1))
773 #define __wait_event_killable(wq, condition) \
774 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
777 * wait_event_killable - sleep until a condition gets true
778 * @wq: the waitqueue to wait on
779 * @condition: a C expression for the event to wait for
781 * The process is put to sleep (TASK_KILLABLE) until the
782 * @condition evaluates to true or a signal is received.
783 * The @condition is checked each time the waitqueue @wq is woken up.
785 * wake_up() has to be called after changing any variable that could
786 * change the result of the wait condition.
788 * The function will return -ERESTARTSYS if it was interrupted by a
789 * signal and 0 if @condition evaluated to true.
791 #define wait_event_killable(wq, condition) \
796 __ret = __wait_event_killable(wq, condition); \
801 #define __wait_event_lock_irq(wq, condition, lock, cmd) \
802 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
803 spin_unlock_irq(&lock); \
806 spin_lock_irq(&lock))
809 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
810 * condition is checked under the lock. This
811 * is expected to be called with the lock
813 * @wq: the waitqueue to wait on
814 * @condition: a C expression for the event to wait for
815 * @lock: a locked spinlock_t, which will be released before cmd
816 * and schedule() and reacquired afterwards.
817 * @cmd: a command which is invoked outside the critical section before
820 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
821 * @condition evaluates to true. The @condition is checked each time
822 * the waitqueue @wq is woken up.
824 * wake_up() has to be called after changing any variable that could
825 * change the result of the wait condition.
827 * This is supposed to be called while holding the lock. The lock is
828 * dropped before invoking the cmd and going to sleep and is reacquired
831 #define wait_event_lock_irq_cmd(wq, condition, lock, cmd) \
835 __wait_event_lock_irq(wq, condition, lock, cmd); \
839 * wait_event_lock_irq - sleep until a condition gets true. The
840 * condition is checked under the lock. This
841 * is expected to be called with the lock
843 * @wq: the waitqueue to wait on
844 * @condition: a C expression for the event to wait for
845 * @lock: a locked spinlock_t, which will be released before schedule()
846 * and reacquired afterwards.
848 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
849 * @condition evaluates to true. The @condition is checked each time
850 * the waitqueue @wq is woken up.
852 * wake_up() has to be called after changing any variable that could
853 * change the result of the wait condition.
855 * This is supposed to be called while holding the lock. The lock is
856 * dropped before going to sleep and is reacquired afterwards.
858 #define wait_event_lock_irq(wq, condition, lock) \
862 __wait_event_lock_irq(wq, condition, lock, ); \
866 #define __wait_event_interruptible_lock_irq(wq, condition, lock, cmd) \
867 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
868 spin_unlock_irq(&lock); \
871 spin_lock_irq(&lock))
874 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
875 * The condition is checked under the lock. This is expected to
876 * be called with the lock taken.
877 * @wq: the waitqueue to wait on
878 * @condition: a C expression for the event to wait for
879 * @lock: a locked spinlock_t, which will be released before cmd and
880 * schedule() and reacquired afterwards.
881 * @cmd: a command which is invoked outside the critical section before
884 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
885 * @condition evaluates to true or a signal is received. The @condition is
886 * checked each time the waitqueue @wq is woken up.
888 * wake_up() has to be called after changing any variable that could
889 * change the result of the wait condition.
891 * This is supposed to be called while holding the lock. The lock is
892 * dropped before invoking the cmd and going to sleep and is reacquired
895 * The macro will return -ERESTARTSYS if it was interrupted by a signal
896 * and 0 if @condition evaluated to true.
898 #define wait_event_interruptible_lock_irq_cmd(wq, condition, lock, cmd) \
902 __ret = __wait_event_interruptible_lock_irq(wq, \
903 condition, lock, cmd); \
908 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
909 * The condition is checked under the lock. This is expected
910 * to be called with the lock taken.
911 * @wq: the waitqueue to wait on
912 * @condition: a C expression for the event to wait for
913 * @lock: a locked spinlock_t, which will be released before schedule()
914 * and reacquired afterwards.
916 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
917 * @condition evaluates to true or signal is received. The @condition is
918 * checked each time the waitqueue @wq is woken up.
920 * wake_up() has to be called after changing any variable that could
921 * change the result of the wait condition.
923 * This is supposed to be called while holding the lock. The lock is
924 * dropped before going to sleep and is reacquired afterwards.
926 * The macro will return -ERESTARTSYS if it was interrupted by a signal
927 * and 0 if @condition evaluated to true.
929 #define wait_event_interruptible_lock_irq(wq, condition, lock) \
933 __ret = __wait_event_interruptible_lock_irq(wq, \
938 #define __wait_event_interruptible_lock_irq_timeout(wq, condition, \
940 ___wait_event(wq, ___wait_cond_timeout(condition), \
941 TASK_INTERRUPTIBLE, 0, timeout, \
942 spin_unlock_irq(&lock); \
943 __ret = schedule_timeout(__ret); \
944 spin_lock_irq(&lock));
947 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
948 * true or a timeout elapses. The condition is checked under
949 * the lock. This is expected to be called with the lock taken.
950 * @wq: the waitqueue to wait on
951 * @condition: a C expression for the event to wait for
952 * @lock: a locked spinlock_t, which will be released before schedule()
953 * and reacquired afterwards.
954 * @timeout: timeout, in jiffies
956 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
957 * @condition evaluates to true or signal is received. The @condition is
958 * checked each time the waitqueue @wq is woken up.
960 * wake_up() has to be called after changing any variable that could
961 * change the result of the wait condition.
963 * This is supposed to be called while holding the lock. The lock is
964 * dropped before going to sleep and is reacquired afterwards.
966 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
967 * was interrupted by a signal, and the remaining jiffies otherwise
968 * if the condition evaluated to true before the timeout elapsed.
970 #define wait_event_interruptible_lock_irq_timeout(wq, condition, lock, \
973 long __ret = timeout; \
974 if (!___wait_cond_timeout(condition)) \
975 __ret = __wait_event_interruptible_lock_irq_timeout( \
976 wq, condition, lock, timeout); \
981 * Waitqueues which are removed from the waitqueue_head at wakeup time
983 void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state);
984 void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state);
985 long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state);
986 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait);
987 long wait_woken(wait_queue_t *wait, unsigned mode, long timeout);
988 int woken_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
989 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
990 int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
992 #define DEFINE_WAIT_FUNC(name, function) \
993 wait_queue_t name = { \
994 .private = current, \
996 .task_list = LIST_HEAD_INIT((name).task_list), \
999 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1001 #define DEFINE_WAIT_BIT(name, word, bit) \
1002 struct wait_bit_queue name = { \
1003 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit), \
1005 .private = current, \
1006 .func = wake_bit_function, \
1008 LIST_HEAD_INIT((name).wait.task_list), \
1012 #define init_wait(wait) \
1014 (wait)->private = current; \
1015 (wait)->func = autoremove_wake_function; \
1016 INIT_LIST_HEAD(&(wait)->task_list); \
1017 (wait)->flags = 0; \
1021 extern int bit_wait(struct wait_bit_key *, int);
1022 extern int bit_wait_io(struct wait_bit_key *, int);
1023 extern int bit_wait_timeout(struct wait_bit_key *, int);
1024 extern int bit_wait_io_timeout(struct wait_bit_key *, int);
1027 * wait_on_bit - wait for a bit to be cleared
1028 * @word: the word being waited on, a kernel virtual address
1029 * @bit: the bit of the word being waited on
1030 * @mode: the task state to sleep in
1032 * There is a standard hashed waitqueue table for generic use. This
1033 * is the part of the hashtable's accessor API that waits on a bit.
1034 * For instance, if one were to have waiters on a bitflag, one would
1035 * call wait_on_bit() in threads waiting for the bit to clear.
1036 * One uses wait_on_bit() where one is waiting for the bit to clear,
1037 * but has no intention of setting it.
1038 * Returned value will be zero if the bit was cleared, or non-zero
1039 * if the process received a signal and the mode permitted wakeup
1043 wait_on_bit(unsigned long *word, int bit, unsigned mode)
1046 if (!test_bit(bit, word))
1048 return out_of_line_wait_on_bit(word, bit,
1054 * wait_on_bit_io - wait for a bit to be cleared
1055 * @word: the word being waited on, a kernel virtual address
1056 * @bit: the bit of the word being waited on
1057 * @mode: the task state to sleep in
1059 * Use the standard hashed waitqueue table to wait for a bit
1060 * to be cleared. This is similar to wait_on_bit(), but calls
1061 * io_schedule() instead of schedule() for the actual waiting.
1063 * Returned value will be zero if the bit was cleared, or non-zero
1064 * if the process received a signal and the mode permitted wakeup
1068 wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
1071 if (!test_bit(bit, word))
1073 return out_of_line_wait_on_bit(word, bit,
1079 * wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses
1080 * @word: the word being waited on, a kernel virtual address
1081 * @bit: the bit of the word being waited on
1082 * @mode: the task state to sleep in
1083 * @timeout: timeout, in jiffies
1085 * Use the standard hashed waitqueue table to wait for a bit
1086 * to be cleared. This is similar to wait_on_bit(), except also takes a
1087 * timeout parameter.
1089 * Returned value will be zero if the bit was cleared before the
1090 * @timeout elapsed, or non-zero if the @timeout elapsed or process
1091 * received a signal and the mode permitted wakeup on that signal.
1094 wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode,
1095 unsigned long timeout)
1098 if (!test_bit(bit, word))
1100 return out_of_line_wait_on_bit_timeout(word, bit,
1106 * wait_on_bit_action - wait for a bit to be cleared
1107 * @word: the word being waited on, a kernel virtual address
1108 * @bit: the bit of the word being waited on
1109 * @action: the function used to sleep, which may take special actions
1110 * @mode: the task state to sleep in
1112 * Use the standard hashed waitqueue table to wait for a bit
1113 * to be cleared, and allow the waiting action to be specified.
1114 * This is like wait_on_bit() but allows fine control of how the waiting
1117 * Returned value will be zero if the bit was cleared, or non-zero
1118 * if the process received a signal and the mode permitted wakeup
1122 wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action,
1126 if (!test_bit(bit, word))
1128 return out_of_line_wait_on_bit(word, bit, action, mode);
1132 * wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
1133 * @word: the word being waited on, a kernel virtual address
1134 * @bit: the bit of the word being waited on
1135 * @mode: the task state to sleep in
1137 * There is a standard hashed waitqueue table for generic use. This
1138 * is the part of the hashtable's accessor API that waits on a bit
1139 * when one intends to set it, for instance, trying to lock bitflags.
1140 * For instance, if one were to have waiters trying to set bitflag
1141 * and waiting for it to clear before setting it, one would call
1142 * wait_on_bit() in threads waiting to be able to set the bit.
1143 * One uses wait_on_bit_lock() where one is waiting for the bit to
1144 * clear with the intention of setting it, and when done, clearing it.
1146 * Returns zero if the bit was (eventually) found to be clear and was
1147 * set. Returns non-zero if a signal was delivered to the process and
1148 * the @mode allows that signal to wake the process.
1151 wait_on_bit_lock(unsigned long *word, int bit, unsigned mode)
1154 if (!test_and_set_bit(bit, word))
1156 return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
1160 * wait_on_bit_lock_io - wait for a bit to be cleared, when wanting to set it
1161 * @word: the word being waited on, a kernel virtual address
1162 * @bit: the bit of the word being waited on
1163 * @mode: the task state to sleep in
1165 * Use the standard hashed waitqueue table to wait for a bit
1166 * to be cleared and then to atomically set it. This is similar
1167 * to wait_on_bit(), but calls io_schedule() instead of schedule()
1168 * for the actual waiting.
1170 * Returns zero if the bit was (eventually) found to be clear and was
1171 * set. Returns non-zero if a signal was delivered to the process and
1172 * the @mode allows that signal to wake the process.
1175 wait_on_bit_lock_io(unsigned long *word, int bit, unsigned mode)
1178 if (!test_and_set_bit(bit, word))
1180 return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
1184 * wait_on_bit_lock_action - wait for a bit to be cleared, when wanting to set it
1185 * @word: the word being waited on, a kernel virtual address
1186 * @bit: the bit of the word being waited on
1187 * @action: the function used to sleep, which may take special actions
1188 * @mode: the task state to sleep in
1190 * Use the standard hashed waitqueue table to wait for a bit
1191 * to be cleared and then to set it, and allow the waiting action
1193 * This is like wait_on_bit() but allows fine control of how the waiting
1196 * Returns zero if the bit was (eventually) found to be clear and was
1197 * set. Returns non-zero if a signal was delivered to the process and
1198 * the @mode allows that signal to wake the process.
1201 wait_on_bit_lock_action(unsigned long *word, int bit, wait_bit_action_f *action,
1205 if (!test_and_set_bit(bit, word))
1207 return out_of_line_wait_on_bit_lock(word, bit, action, mode);
1211 * wait_on_atomic_t - Wait for an atomic_t to become 0
1212 * @val: The atomic value being waited on, a kernel virtual address
1213 * @action: the function used to sleep, which may take special actions
1214 * @mode: the task state to sleep in
1216 * Wait for an atomic_t to become 0. We abuse the bit-wait waitqueue table for
1217 * the purpose of getting a waitqueue, but we set the key to a bit number
1218 * outside of the target 'word'.
1221 int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
1224 if (atomic_read(val) == 0)
1226 return out_of_line_wait_on_atomic_t(val, action, mode);
1229 #endif /* _LINUX_WAIT_H */