34 	struct io_kiocb *req;  member
71 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)  in io_poll_get_ownership_slowpath()  argument
80 	v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs);  in io_poll_get_ownership_slowpath()
83 	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);  in io_poll_get_ownership_slowpath()
92 static inline bool io_poll_get_ownership(struct io_kiocb *req)  in io_poll_get_ownership()  argument
94 	if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))  in io_poll_get_ownership()
95 		return io_poll_get_ownership_slowpath(req);  in io_poll_get_ownership()
96 	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);  in io_poll_get_ownership()
99 static void io_poll_mark_cancelled(struct io_kiocb *req)  in io_poll_mark_cancelled()  argument
101 	atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs);  in io_poll_mark_cancelled()
104 static struct io_poll *io_poll_get_double(struct io_kiocb *req)  in io_poll_get_double()  argument
107 	if (req->opcode == IORING_OP_POLL_ADD)  in io_poll_get_double()
108 		return req->async_data;  in io_poll_get_double()
109 	return req->apoll->double_poll;  in io_poll_get_double()
112 static struct io_poll *io_poll_get_single(struct io_kiocb *req)  in io_poll_get_single()  argument
114 	if (req->opcode == IORING_OP_POLL_ADD)  in io_poll_get_single()
115 		return io_kiocb_to_cmd(req, struct io_poll);  in io_poll_get_single()
116 	return &req->apoll->poll;  in io_poll_get_single()
119 static void io_poll_req_insert(struct io_kiocb *req)  in io_poll_req_insert()  argument
121 	struct io_hash_table *table = &req->ctx->cancel_table;  in io_poll_req_insert()
122 	u32 index = hash_long(req->cqe.user_data, table->hash_bits);  in io_poll_req_insert()
126 	hlist_add_head(&req->hash_node, &hb->list);  in io_poll_req_insert()
130 static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_poll_req_delete()  argument
132 	struct io_hash_table *table = &req->ctx->cancel_table;  in io_poll_req_delete()
133 	u32 index = hash_long(req->cqe.user_data, table->hash_bits);  in io_poll_req_delete()
137 	hash_del(&req->hash_node);  in io_poll_req_delete()
141 static void io_poll_req_insert_locked(struct io_kiocb *req)  in io_poll_req_insert_locked()  argument
143 	struct io_hash_table *table = &req->ctx->cancel_table_locked;  in io_poll_req_insert_locked()
144 	u32 index = hash_long(req->cqe.user_data, table->hash_bits);  in io_poll_req_insert_locked()
146 	lockdep_assert_held(&req->ctx->uring_lock);  in io_poll_req_insert_locked()
148 	hlist_add_head(&req->hash_node, &table->hbs[index].list);  in io_poll_req_insert_locked()
151 static void io_poll_tw_hash_eject(struct io_kiocb *req, struct io_tw_state *ts)  in io_poll_tw_hash_eject()  argument
153 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_tw_hash_eject()
155 	if (req->flags & REQ_F_HASH_LOCKED) {  in io_poll_tw_hash_eject()
163 		hash_del(&req->hash_node);  in io_poll_tw_hash_eject()
164 		req->flags &= ~REQ_F_HASH_LOCKED;  in io_poll_tw_hash_eject()
166 		io_poll_req_delete(req, ctx);  in io_poll_tw_hash_eject()
192 static void io_poll_remove_entries(struct io_kiocb *req)  in io_poll_remove_entries()  argument
198 	if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL)))  in io_poll_remove_entries()
217 	if (req->flags & REQ_F_SINGLE_POLL)  in io_poll_remove_entries()
218 		io_poll_remove_entry(io_poll_get_single(req));  in io_poll_remove_entries()
219 	if (req->flags & REQ_F_DOUBLE_POLL)  in io_poll_remove_entries()
220 		io_poll_remove_entry(io_poll_get_double(req));  in io_poll_remove_entries()
241 static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)  in io_poll_check_events()  argument
246 	if (unlikely(req->task->flags & PF_EXITING))  in io_poll_check_events()
250 		v = atomic_read(&req->poll_refs);  in io_poll_check_events()
264 				req->cqe.res = 0;  in io_poll_check_events()
267 				req->cqe.res = 0;  in io_poll_check_events()
273 				atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);  in io_poll_check_events()
279 		if (!req->cqe.res) {  in io_poll_check_events()
280 			struct poll_table_struct pt = { ._key = req->apoll_events };  in io_poll_check_events()
281 			req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events;  in io_poll_check_events()
288 			if (unlikely(!req->cqe.res)) {  in io_poll_check_events()
290 				if (!(req->apoll_events & EPOLLONESHOT))  in io_poll_check_events()
295 		if (req->apoll_events & EPOLLONESHOT)  in io_poll_check_events()
299 		if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {  in io_poll_check_events()
300 			__poll_t mask = mangle_poll(req->cqe.res &  in io_poll_check_events()
301 						    req->apoll_events);  in io_poll_check_events()
303 			if (!io_fill_cqe_req_aux(req, ts->locked, mask,  in io_poll_check_events()
305 				io_req_set_res(req, mask, 0);  in io_poll_check_events()
309 			int ret = io_poll_issue(req, ts);  in io_poll_check_events()
317 		req->cqe.res = 0;  in io_poll_check_events()
323 	} while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs) &  in io_poll_check_events()
329 void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)  in io_poll_task_func()  argument
333 	ret = io_poll_check_events(req, ts);  in io_poll_task_func()
336 	io_poll_remove_entries(req);  in io_poll_task_func()
337 	io_poll_tw_hash_eject(req, ts);  in io_poll_task_func()
339 	if (req->opcode == IORING_OP_POLL_ADD) {  in io_poll_task_func()
343 			poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_task_func()
344 			req->cqe.res = mangle_poll(req->cqe.res & poll->events);  in io_poll_task_func()
346 			io_req_task_submit(req, ts);  in io_poll_task_func()
349 			req->cqe.res = ret;  in io_poll_task_func()
350 			req_set_fail(req);  in io_poll_task_func()
353 		io_req_set_res(req, req->cqe.res, 0);  in io_poll_task_func()
354 		io_req_task_complete(req, ts);  in io_poll_task_func()
356 		io_tw_lock(req->ctx, ts);  in io_poll_task_func()
359 			io_req_task_complete(req, ts);  in io_poll_task_func()
361 			io_req_task_submit(req, ts);  in io_poll_task_func()
363 			io_req_defer_failed(req, ret);  in io_poll_task_func()
367 static void __io_poll_execute(struct io_kiocb *req, int mask)  in __io_poll_execute()  argument
369 	io_req_set_res(req, mask, 0);  in __io_poll_execute()
370 	req->io_task_work.func = io_poll_task_func;  in __io_poll_execute()
372 	trace_io_uring_task_add(req, mask);  in __io_poll_execute()
373 	io_req_task_work_add(req);  in __io_poll_execute()
376 static inline void io_poll_execute(struct io_kiocb *req, int res)  in io_poll_execute()  argument
378 	if (io_poll_get_ownership(req))  in io_poll_execute()
379 		__io_poll_execute(req, res);  in io_poll_execute()
382 static void io_poll_cancel_req(struct io_kiocb *req)  in io_poll_cancel_req()  argument
384 	io_poll_mark_cancelled(req);  in io_poll_cancel_req()
386 	io_poll_execute(req, 0);  in io_poll_cancel_req()
391 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)  in io_pollfree_wake()  argument
393 	io_poll_mark_cancelled(req);  in io_pollfree_wake()
395 	io_poll_execute(req, 0);  in io_pollfree_wake()
419 	struct io_kiocb *req = wqe_to_req(wait);  in io_poll_wake()  local
424 		return io_pollfree_wake(req, poll);  in io_poll_wake()
430 	if (io_poll_get_ownership(req)) {  in io_poll_wake()
444 				req->flags &= ~REQ_F_DOUBLE_POLL;  in io_poll_wake()
446 				req->flags &= ~REQ_F_SINGLE_POLL;  in io_poll_wake()
448 		__io_poll_execute(req, mask);  in io_poll_wake()
454 static bool io_poll_double_prepare(struct io_kiocb *req)  in io_poll_double_prepare()  argument
457 	struct io_poll *poll = io_poll_get_single(req);  in io_poll_double_prepare()
470 		req->flags |= REQ_F_DOUBLE_POLL;  in io_poll_double_prepare()
471 		if (req->opcode == IORING_OP_POLL_ADD)  in io_poll_double_prepare()
472 			req->flags |= REQ_F_ASYNC_DATA;  in io_poll_double_prepare()
483 	struct io_kiocb *req = pt->req;  in __io_queue_proc()  local
484 	unsigned long wqe_private = (unsigned long) req;  in __io_queue_proc()
514 		if (!io_poll_double_prepare(req)) {  in __io_queue_proc()
522 		req->flags |= REQ_F_SINGLE_POLL;  in __io_queue_proc()
539 	struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll);  in io_poll_queue_proc()
542 			(struct io_poll **) &pt->req->async_data);  in io_poll_queue_proc()
545 static bool io_poll_can_finish_inline(struct io_kiocb *req,  in io_poll_can_finish_inline()  argument
548 	return pt->owning || io_poll_get_ownership(req);  in io_poll_can_finish_inline()
551 static void io_poll_add_hash(struct io_kiocb *req)  in io_poll_add_hash()  argument
553 	if (req->flags & REQ_F_HASH_LOCKED)  in io_poll_add_hash()
554 		io_poll_req_insert_locked(req);  in io_poll_add_hash()
556 		io_poll_req_insert(req);  in io_poll_add_hash()
565 static int __io_arm_poll_handler(struct io_kiocb *req,  in __io_arm_poll_handler()  argument
570 	struct io_ring_ctx *ctx = req->ctx;  in __io_arm_poll_handler()
572 	INIT_HLIST_NODE(&req->hash_node);  in __io_arm_poll_handler()
573 	req->work.cancel_seq = atomic_read(&ctx->cancel_seq);  in __io_arm_poll_handler()
575 	poll->file = req->file;  in __io_arm_poll_handler()
576 	req->apoll_events = poll->events;  in __io_arm_poll_handler()
579 	ipt->req = req;  in __io_arm_poll_handler()
594 	atomic_set(&req->poll_refs, (int)ipt->owning);  in __io_arm_poll_handler()
598 		req->flags &= ~REQ_F_HASH_LOCKED;  in __io_arm_poll_handler()
600 	mask = vfs_poll(req->file, &ipt->pt) & poll->events;  in __io_arm_poll_handler()
603 		io_poll_remove_entries(req);  in __io_arm_poll_handler()
605 		if (!io_poll_can_finish_inline(req, ipt)) {  in __io_arm_poll_handler()
606 			io_poll_mark_cancelled(req);  in __io_arm_poll_handler()
617 		if (!io_poll_can_finish_inline(req, ipt)) {  in __io_arm_poll_handler()
618 			io_poll_add_hash(req);  in __io_arm_poll_handler()
621 		io_poll_remove_entries(req);  in __io_arm_poll_handler()
627 	io_poll_add_hash(req);  in __io_arm_poll_handler()
630 	    io_poll_can_finish_inline(req, ipt)) {  in __io_arm_poll_handler()
631 		__io_poll_execute(req, mask);  in __io_arm_poll_handler()
640 		if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)  in __io_arm_poll_handler()
641 			__io_poll_execute(req, 0);  in __io_arm_poll_handler()
650 	struct async_poll *apoll = pt->req->apoll;  in io_async_queue_proc()
663 static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req,  in io_req_alloc_apoll()  argument
666 	struct io_ring_ctx *ctx = req->ctx;  in io_req_alloc_apoll()
670 	if (req->flags & REQ_F_POLLED) {  in io_req_alloc_apoll()
671 		apoll = req->apoll;  in io_req_alloc_apoll()
687 	req->apoll = apoll;  in io_req_alloc_apoll()
693 int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)  in io_arm_poll_handler()  argument
695 	const struct io_issue_def *def = &io_issue_defs[req->opcode];  in io_arm_poll_handler()
705 	req->flags |= REQ_F_HASH_LOCKED;  in io_arm_poll_handler()
709 	if (!file_can_poll(req->file))  in io_arm_poll_handler()
711 	if (!(req->flags & REQ_F_APOLL_MULTISHOT))  in io_arm_poll_handler()
718 		if (req->flags & REQ_F_CLEAR_POLLIN)  in io_arm_poll_handler()
726 	apoll = io_req_alloc_apoll(req, issue_flags);  in io_arm_poll_handler()
729 	req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL);  in io_arm_poll_handler()
730 	req->flags |= REQ_F_POLLED;  in io_arm_poll_handler()
733 	io_kbuf_recycle(req, issue_flags);  in io_arm_poll_handler()
735 	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);  in io_arm_poll_handler()
738 	trace_io_uring_poll_arm(req, mask, apoll->poll.events);  in io_arm_poll_handler()
748 	struct io_kiocb *req;  in io_poll_remove_all_table()  local
756 		hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) {  in io_poll_remove_all_table()
757 			if (io_match_task_safe(req, tsk, cancel_all)) {  in io_poll_remove_all_table()
758 				hlist_del_init(&req->hash_node);  in io_poll_remove_all_table()
759 				io_poll_cancel_req(req);  in io_poll_remove_all_table()
787 	struct io_kiocb *req;  in io_poll_find()  local
794 	hlist_for_each_entry(req, &hb->list, hash_node) {  in io_poll_find()
795 		if (cd->data != req->cqe.user_data)  in io_poll_find()
797 		if (poll_only && req->opcode != IORING_OP_POLL_ADD)  in io_poll_find()
800 			if (cd->seq == req->work.cancel_seq)  in io_poll_find()
802 			req->work.cancel_seq = cd->seq;  in io_poll_find()
805 		return req;  in io_poll_find()
817 	struct io_kiocb *req;  in io_poll_file_find()  local
826 		hlist_for_each_entry(req, &hb->list, hash_node) {  in io_poll_file_find()
827 			if (io_cancel_req_match(req, cd)) {  in io_poll_file_find()
829 				return req;  in io_poll_file_find()
837 static int io_poll_disarm(struct io_kiocb *req)  in io_poll_disarm()  argument
839 	if (!req)  in io_poll_disarm()
841 	if (!io_poll_get_ownership(req))  in io_poll_disarm()
843 	io_poll_remove_entries(req);  in io_poll_disarm()
844 	hash_del(&req->hash_node);  in io_poll_disarm()
852 	struct io_kiocb *req;  in __io_poll_cancel()  local
856 		req = io_poll_file_find(ctx, cd, table, &bucket);  in __io_poll_cancel()
858 		req = io_poll_find(ctx, false, cd, table, &bucket);  in __io_poll_cancel()
860 	if (req)  in __io_poll_cancel()
861 		io_poll_cancel_req(req);  in __io_poll_cancel()
864 	return req ? 0 : -ENOENT;  in __io_poll_cancel()
899 int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_poll_remove_prep()  argument
901 	struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update);  in io_poll_remove_prep()
929 int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_poll_add_prep()  argument
931 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add_prep()
939 	if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP))  in io_poll_add_prep()
946 int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_add()  argument
948 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add()
958 	if (req->ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_SINGLE_ISSUER))  in io_poll_add()
959 		req->flags |= REQ_F_HASH_LOCKED;  in io_poll_add()
961 	ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags);  in io_poll_add()
963 		io_req_set_res(req, ipt.result_mask, 0);  in io_poll_add()
969 int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_remove()  argument
971 	struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update);  in io_poll_remove()
972 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_remove()
1030 		req_set_fail(req);  in io_poll_remove()
1034 	io_req_set_res(req, ret, 0);  in io_poll_remove()