37 static inline bool io_is_timeout_noseq(struct io_kiocb *req)  in io_is_timeout_noseq()  argument
39 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_is_timeout_noseq()
44 static inline void io_put_req(struct io_kiocb *req)  in io_put_req()  argument
46 	if (req_ref_put_and_test(req)) {  in io_put_req()
47 		io_queue_next(req);  in io_put_req()
48 		io_free_req(req);  in io_put_req()
52 static bool io_kill_timeout(struct io_kiocb *req, int status)  in io_kill_timeout()  argument
53 	__must_hold(&req->ctx->completion_lock)  in io_kill_timeout()
54 	__must_hold(&req->ctx->timeout_lock)  in io_kill_timeout()
56 	struct io_timeout_data *io = req->async_data;  in io_kill_timeout()
59 		struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_kill_timeout()
62 			req_set_fail(req);  in io_kill_timeout()
63 		atomic_set(&req->ctx->cq_timeouts,  in io_kill_timeout()
64 			atomic_read(&req->ctx->cq_timeouts) + 1);  in io_kill_timeout()
66 		io_req_tw_post_queue(req, status, 0);  in io_kill_timeout()
80 		struct io_kiocb *req = cmd_to_io_kiocb(timeout);  in io_flush_timeouts()  local
83 		if (io_is_timeout_noseq(req))  in io_flush_timeouts()
98 		io_kill_timeout(req, 0);  in io_flush_timeouts()
120 static void io_fail_links(struct io_kiocb *req)  in io_fail_links()  argument
121 	__must_hold(&req->ctx->completion_lock)  in io_fail_links()
123 	struct io_kiocb *link = req->link;  in io_fail_links()
124 	bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;  in io_fail_links()
134 		trace_io_uring_fail_link(req, link);  in io_fail_links()
138 	link = req->link;  in io_fail_links()
141 	req->link = NULL;  in io_fail_links()
144 static inline void io_remove_next_linked(struct io_kiocb *req)  in io_remove_next_linked()  argument
146 	struct io_kiocb *nxt = req->link;  in io_remove_next_linked()
148 	req->link = nxt->link;  in io_remove_next_linked()
152 void io_disarm_next(struct io_kiocb *req)  in io_disarm_next()  argument
153 	__must_hold(&req->ctx->completion_lock)  in io_disarm_next()
157 	if (req->flags & REQ_F_ARM_LTIMEOUT) {  in io_disarm_next()
158 		link = req->link;  in io_disarm_next()
159 		req->flags &= ~REQ_F_ARM_LTIMEOUT;  in io_disarm_next()
161 			io_remove_next_linked(req);  in io_disarm_next()
164 	} else if (req->flags & REQ_F_LINK_TIMEOUT) {  in io_disarm_next()
165 		struct io_ring_ctx *ctx = req->ctx;  in io_disarm_next()
168 		link = io_disarm_linked_timeout(req);  in io_disarm_next()
173 	if (unlikely((req->flags & REQ_F_FAIL) &&  in io_disarm_next()
174 		     !(req->flags & REQ_F_HARDLINK)))  in io_disarm_next()
175 		io_fail_links(req);  in io_disarm_next()
178 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,  in __io_disarm_linked_timeout()  argument
180 	__must_hold(&req->ctx->completion_lock)  in __io_disarm_linked_timeout()
181 	__must_hold(&req->ctx->timeout_lock)  in __io_disarm_linked_timeout()
186 	io_remove_next_linked(req);  in __io_disarm_linked_timeout()
200 	struct io_kiocb *req = data->req;  in io_timeout_fn()  local
201 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_fn()
202 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_fn()
207 	atomic_set(&req->ctx->cq_timeouts,  in io_timeout_fn()
208 		atomic_read(&req->ctx->cq_timeouts) + 1);  in io_timeout_fn()
212 		req_set_fail(req);  in io_timeout_fn()
214 	io_req_set_res(req, -ETIME, 0);  in io_timeout_fn()
215 	req->io_task_work.func = io_req_task_complete;  in io_timeout_fn()
216 	io_req_task_work_add(req);  in io_timeout_fn()
226 	struct io_kiocb *req = NULL;  in io_timeout_extract()  local
239 		req = tmp;  in io_timeout_extract()
242 	if (!req)  in io_timeout_extract()
245 	io = req->async_data;  in io_timeout_extract()
248 	timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_extract()
250 	return req;  in io_timeout_extract()
256 	struct io_kiocb *req;  in io_timeout_cancel()  local
259 	req = io_timeout_extract(ctx, cd);  in io_timeout_cancel()
262 	if (IS_ERR(req))  in io_timeout_cancel()
263 		return PTR_ERR(req);  in io_timeout_cancel()
264 	io_req_task_queue_fail(req, -ECANCELED);  in io_timeout_cancel()
268 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)  in io_req_task_link_timeout()  argument
271 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_req_task_link_timeout()
276 		if (!(req->task->flags & PF_EXITING)) {  in io_req_task_link_timeout()
278 				.ctx		= req->ctx,  in io_req_task_link_timeout()
282 			ret = io_try_cancel(req->task->io_uring, &cd, issue_flags);  in io_req_task_link_timeout()
284 		io_req_set_res(req, ret ?: -ETIME, 0);  in io_req_task_link_timeout()
285 		io_req_complete_post(req);  in io_req_task_link_timeout()
288 		io_req_set_res(req, -ETIME, 0);  in io_req_task_link_timeout()
289 		io_req_complete_post(req);  in io_req_task_link_timeout()
297 	struct io_kiocb *prev, *req = data->req;  in io_link_timeout_fn()  local
298 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_link_timeout_fn()
299 	struct io_ring_ctx *ctx = req->ctx;  in io_link_timeout_fn()
319 	req->io_task_work.func = io_req_task_link_timeout;  in io_link_timeout_fn()
320 	io_req_task_work_add(req);  in io_link_timeout_fn()
346 	struct io_kiocb *req = NULL;  in io_linked_timeout_update()  local
352 			req = tmp;  in io_linked_timeout_update()
356 	if (!req)  in io_linked_timeout_update()
359 	io = req->async_data;  in io_linked_timeout_update()
373 	struct io_kiocb *req = io_timeout_extract(ctx, &cd);  in io_timeout_update()  local
374 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_update()
377 	if (IS_ERR(req))  in io_timeout_update()
378 		return PTR_ERR(req);  in io_timeout_update()
381 	data = req->async_data;  in io_timeout_update()
389 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_timeout_remove_prep()  argument
391 	struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);  in io_timeout_remove_prep()
393 	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))  in io_timeout_remove_prep()
429 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout_remove()  argument
431 	struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);  in io_timeout_remove()
432 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_remove()
453 		req_set_fail(req);  in io_timeout_remove()
454 	io_req_set_res(req, ret, 0);  in io_timeout_remove()
458 static int __io_timeout_prep(struct io_kiocb *req,  in __io_timeout_prep()  argument
462 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in __io_timeout_prep()
481 	if (unlikely(off && !req->ctx->off_timeout_used))  in __io_timeout_prep()
482 		req->ctx->off_timeout_used = true;  in __io_timeout_prep()
484 	if (WARN_ON_ONCE(req_has_async_data(req)))  in __io_timeout_prep()
486 	if (io_alloc_async_data(req))  in __io_timeout_prep()
489 	data = req->async_data;  in __io_timeout_prep()
490 	data->req = req;  in __io_timeout_prep()
504 		struct io_submit_link *link = &req->ctx->submit_state.link;  in __io_timeout_prep()
516 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_timeout_prep()  argument
518 	return __io_timeout_prep(req, sqe, false);  in io_timeout_prep()
521 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_link_timeout_prep()  argument
523 	return __io_timeout_prep(req, sqe, true);  in io_link_timeout_prep()
526 int io_timeout(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout()  argument
528 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout()
529 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout()
530 	struct io_timeout_data *data = req->async_data;  in io_timeout()
541 	if (io_is_timeout_noseq(req)) {  in io_timeout()
577 void io_queue_linked_timeout(struct io_kiocb *req)  in io_queue_linked_timeout()  argument
579 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_queue_linked_timeout()
580 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_linked_timeout()
588 		struct io_timeout_data *data = req->async_data;  in io_queue_linked_timeout()
597 	io_put_req(req);  in io_queue_linked_timeout()
602 	__must_hold(&req->ctx->timeout_lock)  in io_match_task()
604 	struct io_kiocb *req;  in io_match_task()  local
611 	io_for_each_link(req, head) {  in io_match_task()
612 		if (req->flags & REQ_F_INFLIGHT)  in io_match_task()
628 		struct io_kiocb *req = cmd_to_io_kiocb(timeout);  in io_kill_timeouts()  local
630 		if (io_match_task(req, tsk, cancel_all) &&  in io_kill_timeouts()
631 		    io_kill_timeout(req, -ECANCELED))  in io_kill_timeouts()