Lines Matching full:poll
8 #include <linux/poll.h>
20 #include "poll.h"
38 /* output value, set only if arm poll returns >0 */
90 * arming poll and wakeups.
106 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ in io_poll_get_double()
116 return &req->apoll->poll; in io_poll_get_single()
170 static void io_init_poll_iocb(struct io_poll *poll, __poll_t events) in io_init_poll_iocb() argument
172 poll->head = NULL; in io_init_poll_iocb()
175 poll->events = events | IO_POLL_UNMASK; in io_init_poll_iocb()
176 INIT_LIST_HEAD(&poll->wait.entry); in io_init_poll_iocb()
177 init_waitqueue_func_entry(&poll->wait, io_poll_wake); in io_init_poll_iocb()
180 static inline void io_poll_remove_entry(struct io_poll *poll) in io_poll_remove_entry() argument
182 struct wait_queue_head *head = smp_load_acquire(&poll->head); in io_poll_remove_entry()
186 list_del_init(&poll->wait.entry); in io_poll_remove_entry()
187 poll->head = NULL; in io_poll_remove_entry()
196 * into the poll/apoll/double cachelines if we can. in io_poll_remove_entries()
232 * All poll tw should go through this. Checks for poll events, manages
239 * poll and that the result is stored in req->cqe.
341 struct io_poll *poll; in io_poll_task_func() local
343 poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_task_func()
344 req->cqe.res = mangle_poll(req->cqe.res & poll->events); in io_poll_task_func()
391 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) in io_pollfree_wake() argument
404 list_del_init(&poll->wait.entry); in io_pollfree_wake()
412 smp_store_release(&poll->head, NULL); in io_pollfree_wake()
420 struct io_poll *poll = container_of(wait, struct io_poll, wait); in io_poll_wake() local
424 return io_pollfree_wake(req, poll); in io_poll_wake()
427 if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON))) in io_poll_wake()
432 * If we trigger a multishot poll off our own wakeup path, in io_poll_wake()
437 poll->events |= EPOLLONESHOT; in io_poll_wake()
440 if (mask && poll->events & EPOLLONESHOT) { in io_poll_wake()
441 list_del_init(&poll->wait.entry); in io_poll_wake()
442 poll->head = NULL; in io_poll_wake()
457 struct io_poll *poll = io_poll_get_single(req); in io_poll_double_prepare() local
461 head = smp_load_acquire(&poll->head); in io_poll_double_prepare()
463 * poll arm might not hold ownership and so race for req->flags with in io_poll_double_prepare()
464 * io_poll_wake(). There is only one poll entry queued, serialise with in io_poll_double_prepare()
479 static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt, in __io_queue_proc() argument
487 * The file being polled uses multiple waitqueues for poll handling in __io_queue_proc()
492 struct io_poll *first = poll; in __io_queue_proc()
505 poll = kmalloc(sizeof(*poll), GFP_ATOMIC); in __io_queue_proc()
506 if (!poll) { in __io_queue_proc()
513 io_init_poll_iocb(poll, first->events); in __io_queue_proc()
516 kfree(poll); in __io_queue_proc()
519 *poll_ptr = poll; in __io_queue_proc()
521 /* fine to modify, there is no poll queued to race with us */ in __io_queue_proc()
526 poll->head = head; in __io_queue_proc()
527 poll->wait.private = (void *) wqe_private; in __io_queue_proc()
529 if (poll->events & EPOLLEXCLUSIVE) in __io_queue_proc()
530 add_wait_queue_exclusive(head, &poll->wait); in __io_queue_proc()
532 add_wait_queue(head, &poll->wait); in __io_queue_proc()
539 struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll); in io_poll_queue_proc() local
541 __io_queue_proc(poll, pt, head, in io_poll_queue_proc()
566 struct io_poll *poll, in __io_arm_poll_handler() argument
574 io_init_poll_iocb(poll, mask); in __io_arm_poll_handler()
575 poll->file = req->file; in __io_arm_poll_handler()
576 req->apoll_events = poll->events; in __io_arm_poll_handler()
600 mask = vfs_poll(req->file, &ipt->pt) & poll->events; in __io_arm_poll_handler()
608 } else if (mask && (poll->events & EPOLLET)) { in __io_arm_poll_handler()
616 ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) { in __io_arm_poll_handler()
629 if (mask && (poll->events & EPOLLET) && in __io_arm_poll_handler()
638 * poll was waken up, queue up a tw, it'll deal with it. in __io_arm_poll_handler()
652 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll); in io_async_queue_proc()
656 * We can't reliably detect loops in repeated poll triggers and issue
678 apoll->poll.retries = APOLL_MAX_RETRY; in io_req_alloc_apoll()
684 apoll->poll.retries = APOLL_MAX_RETRY; in io_req_alloc_apoll()
688 if (unlikely(!--apoll->poll.retries)) in io_req_alloc_apoll()
735 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); in io_arm_poll_handler()
738 trace_io_uring_poll_arm(req, mask, apoll->poll.events); in io_arm_poll_handler()
769 * Returns true if we found and killed one or more poll requests
931 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_add_prep() local
942 poll->events = io_poll_parse_events(sqe, flags); in io_poll_add_prep()
948 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_add() local
961 ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); in io_poll_add()
1009 struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll); in io_poll_remove() local
1011 poll->events &= ~0xffff; in io_poll_remove()
1012 poll->events |= poll_update->events & 0xffff; in io_poll_remove()
1013 poll->events |= IO_POLL_UNMASK; in io_poll_remove()
1019 /* successfully updated, don't complete poll request */ in io_poll_remove()