8 #include <linux/poll.h>
20 #include "poll.h"
38 	/* output value, set only if arm poll returns >0 */
87  * arming poll and wakeups.
103 	/* pure poll stashes this in ->async_data, poll driven retry elsewhere */  in io_poll_get_double()
113 	return &req->apoll->poll;  in io_poll_get_single()
167 static void io_init_poll_iocb(struct io_poll *poll, __poll_t events,  in io_init_poll_iocb()  argument
170 	poll->head = NULL;  in io_init_poll_iocb()
173 	poll->events = events | IO_POLL_UNMASK;  in io_init_poll_iocb()
174 	INIT_LIST_HEAD(&poll->wait.entry);  in io_init_poll_iocb()
175 	init_waitqueue_func_entry(&poll->wait, wake_func);  in io_init_poll_iocb()
178 static inline void io_poll_remove_entry(struct io_poll *poll)  in io_poll_remove_entry()  argument
180 	struct wait_queue_head *head = smp_load_acquire(&poll->head);  in io_poll_remove_entry()
184 		list_del_init(&poll->wait.entry);  in io_poll_remove_entry()
185 		poll->head = NULL;  in io_poll_remove_entry()
194 	 * into the poll/apoll/double cachelines if we can.  in io_poll_remove_entries()
229  * All poll tw should go through this. Checks for poll events, manages
235  * IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot poll and that the result
326 		struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_task_func()  local
327 		req->cqe.res = mangle_poll(req->cqe.res & poll->events);  in io_poll_task_func()
363 	 * This is useful for poll that is armed on behalf of another  in __io_poll_execute()
392 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)  in io_pollfree_wake()  argument
405 	list_del_init(&poll->wait.entry);  in io_pollfree_wake()
413 	smp_store_release(&poll->head, NULL);  in io_pollfree_wake()
421 	struct io_poll *poll = container_of(wait, struct io_poll, wait);  in io_poll_wake()  local
425 		return io_pollfree_wake(req, poll);  in io_poll_wake()
428 	if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON)))  in io_poll_wake()
433 		if (mask && poll->events & EPOLLONESHOT) {  in io_poll_wake()
434 			list_del_init(&poll->wait.entry);  in io_poll_wake()
435 			poll->head = NULL;  in io_poll_wake()
450 	struct io_poll *poll = io_poll_get_single(req);  in io_poll_double_prepare()  local
454 	head = smp_load_acquire(&poll->head);  in io_poll_double_prepare()
456 	 * poll arm might not hold ownership and so race for req->flags with  in io_poll_double_prepare()
457 	 * io_poll_wake(). There is only one poll entry queued, serialise with  in io_poll_double_prepare()
472 static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,  in __io_queue_proc()  argument
480 	 * The file being polled uses multiple waitqueues for poll handling  in __io_queue_proc()
485 		struct io_poll *first = poll;  in __io_queue_proc()
498 		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);  in __io_queue_proc()
499 		if (!poll) {  in __io_queue_proc()
506 		io_init_poll_iocb(poll, first->events, first->wait.func);  in __io_queue_proc()
509 			kfree(poll);  in __io_queue_proc()
512 		*poll_ptr = poll;  in __io_queue_proc()
514 		/* fine to modify, there is no poll queued to race with us */  in __io_queue_proc()
519 	poll->head = head;  in __io_queue_proc()
520 	poll->wait.private = (void *) wqe_private;  in __io_queue_proc()
522 	if (poll->events & EPOLLEXCLUSIVE)  in __io_queue_proc()
523 		add_wait_queue_exclusive(head, &poll->wait);  in __io_queue_proc()
525 		add_wait_queue(head, &poll->wait);  in __io_queue_proc()
532 	struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll);  in io_poll_queue_proc()  local
534 	__io_queue_proc(poll, pt, head,  in io_poll_queue_proc()
551 				 struct io_poll *poll,  in __io_arm_poll_handler()  argument
559 	io_init_poll_iocb(poll, mask, io_poll_wake);  in __io_arm_poll_handler()
560 	poll->file = req->file;  in __io_arm_poll_handler()
561 	req->apoll_events = poll->events;  in __io_arm_poll_handler()
585 	mask = vfs_poll(req->file, &ipt->pt) & poll->events;  in __io_arm_poll_handler()
593 		} else if (mask && (poll->events & EPOLLET)) {  in __io_arm_poll_handler()
601 	   ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) {  in __io_arm_poll_handler()
615 	if (mask && (poll->events & EPOLLET) &&  in __io_arm_poll_handler()
624 		 * poll was waken up, queue up a tw, it'll deal with it.  in __io_arm_poll_handler()
638 	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);  in io_async_queue_proc()
707 	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);  in io_arm_poll_handler()
710 	trace_io_uring_poll_arm(req, mask, apoll->poll.events);  in io_arm_poll_handler()
741  * Returns true if we found and killed one or more poll requests
906 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add_prep()  local
917 	poll->events = io_poll_parse_events(sqe, flags);  in io_poll_add_prep()
923 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add()  local
936 	ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags);  in io_poll_add()
985 			struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll);  in io_poll_remove()  local
987 			poll->events &= ~0xffff;  in io_poll_remove()
988 			poll->events |= poll_update->events & 0xffff;  in io_poll_remove()
989 			poll->events |= IO_POLL_UNMASK;  in io_poll_remove()
995 		/* successfully updated, don't complete poll request */  in io_poll_remove()