210 struct io_ring_ctx;
245 typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
248 	struct io_ring_ctx		*ctx;
330 struct io_ring_ctx {  struct
469 	const struct io_ring_ctx *last;
868 	struct io_ring_ctx		*ctx;
893 	struct io_ring_ctx	*ctx;
1077 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1082 static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1088 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
1092 static struct file *io_file_get(struct io_ring_ctx *ctx,
1098 static void io_submit_flush_completions(struct io_ring_ctx *ctx);
1115 		struct io_ring_ctx *ctx = file->private_data;  in io_uring_get_socket()
1124 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)  in io_tw_lock()
1185 	struct io_ring_ctx *ctx = req->ctx;  in io_req_set_rsrc_node()
1235 	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);  in io_ring_ctx_ref_free()
1247 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,  in io_fallback_req_func()
1266 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)  in io_ring_ctx_alloc()
1268 	struct io_ring_ctx *ctx;  in io_ring_ctx_alloc()
1332 static void io_account_cq_overflow(struct io_ring_ctx *ctx)  in io_account_cq_overflow()
1343 		struct io_ring_ctx *ctx = req->ctx;  in req_need_defer()
1397 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_work()
1431 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_link()
1445 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_async_work()
1492 static void io_queue_deferred(struct io_ring_ctx *ctx)  in io_queue_deferred()
1506 static void io_flush_timeouts(struct io_ring_ctx *ctx)  in io_flush_timeouts()
1539 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)  in __io_commit_cqring_flush()
1547 static inline void io_commit_cqring(struct io_ring_ctx *ctx)  in io_commit_cqring()
1555 static inline bool io_sqring_full(struct io_ring_ctx *ctx)  in io_sqring_full()
1562 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)  in __io_cqring_events()
1567 static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)  in io_get_cqe()
1584 static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)  in io_should_trigger_evfd()
1600 static void io_cqring_ev_posted(struct io_ring_ctx *ctx)  in io_cqring_ev_posted()
1617 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)  in io_cqring_ev_posted_iopoll()
1633 static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)  in __io_cqring_overflow_flush()
1675 static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)  in io_cqring_overflow_flush()
1724 static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,  in io_cqring_event_overflow()
1752 static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,  in __io_cqring_fill_event()
1775 static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,  in io_cqring_fill_event()
1784 	struct io_ring_ctx *ctx = req->ctx;  in io_req_complete_post()
1868 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_preinit_req()
1877 static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,  in io_flush_cached_locked_reqs()
1887 static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)  in io_flush_cached_reqs()
1921 static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)  in io_alloc_req()
1979 	struct io_ring_ctx *ctx = req->ctx;  in __io_free_req()
2061 		struct io_ring_ctx *ctx = req->ctx;  in io_disarm_next()
2086 		struct io_ring_ctx *ctx = req->ctx;  in __io_req_find_next()
2109 static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)  in ctx_flush_and_put()
2125 	struct io_ring_ctx *ctx = NULL;  in tctx_task_work()
2217 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_cancel()
2226 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_submit()
2287 static void io_req_free_batch_finish(struct io_ring_ctx *ctx,  in io_req_free_batch_finish()
2317 static void io_submit_flush_completions(struct io_ring_ctx *ctx)  in io_submit_flush_completions()
2376 static unsigned io_cqring_events(struct io_ring_ctx *ctx)  in io_cqring_events()
2383 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)  in io_sqring_entries()
2426 static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_iopoll_complete()
2453 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_do_iopoll()
2503 static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)  in io_iopoll_try_reap_events()
2531 static int io_iopoll_check(struct io_ring_ctx *ctx, long min)  in io_iopoll_check()
2609 	struct io_ring_ctx *ctx = req->ctx;  in io_rw_should_reissue()
2664 		struct io_ring_ctx *ctx = req->ctx;  in io_req_task_complete()
2722 	struct io_ring_ctx *ctx = req->ctx;  in io_iopoll_req_issued()
2833 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
2946 			struct io_ring_ctx *ctx = req->ctx;  in kiocb_done()
3020 	struct io_ring_ctx *ctx = req->ctx;  in io_import_fixed()
3034 static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_unlock()
3040 static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_lock()
4057 	struct io_ring_ctx *ctx = req->ctx;  in io_nop()
4068 	struct io_ring_ctx *ctx = req->ctx;  in io_fsync_prep()
4289 static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,  in __io_remove_buffers()
4318 	struct io_ring_ctx *ctx = req->ctx;  in io_remove_buffers()
4405 	struct io_ring_ctx *ctx = req->ctx;  in io_provide_buffers()
4655 	struct io_ring_ctx *ctx = req->ctx;  in io_sfr_prep()
5271 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_rewait()
5329 	struct io_ring_ctx *ctx = req->ctx;  in __io_poll_complete()
5363 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_task_func()
5506 	struct io_ring_ctx *ctx = req->ctx;  in io_async_task_func()
5540 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_req_insert()
5553 	struct io_ring_ctx *ctx = req->ctx;  in __io_arm_poll_handler()
5602 	struct io_ring_ctx *ctx = req->ctx;  in io_arm_poll_handler()
5692 static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_poll_remove_all()
5717 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_find()
5735 static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_cancel()
5835 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_add()
5861 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_update()
5932 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_fn()
5946 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,  in io_timeout_extract()
5969 static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)  in io_timeout_cancel()
6000 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_linked_timeout_update()
6025 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_timeout_update()
6088 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_remove()
6168 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout()
6217 	struct io_ring_ctx *ctx;
6230 			       struct io_ring_ctx *ctx)  in io_async_cancel_one()
6257 	struct io_ring_ctx *ctx = req->ctx;  in io_try_cancel_userdata()
6295 	struct io_ring_ctx *ctx = req->ctx;  in io_async_cancel()
6340 	struct io_ring_ctx *ctx = req->ctx;  in io_files_update()
6484 	struct io_ring_ctx *ctx = req->ctx;  in io_drain_req()
6637 	struct io_ring_ctx *ctx = req->ctx;  in io_issue_sqe()
6823 static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,  in io_file_from_index()
6844 static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,  in io_file_get_fixed()
6862 static struct file *io_file_get_normal(struct io_ring_ctx *ctx,  in io_file_get_normal()
6875 static inline struct file *io_file_get(struct io_ring_ctx *ctx,  in io_file_get()
6903 	struct io_ring_ctx *ctx = req->ctx;  in io_link_timeout_fn()
6930 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_linked_timeout()
6965 			struct io_ring_ctx *ctx = req->ctx;  in __io_queue_sqe()
7026 static inline bool io_check_restriction(struct io_ring_ctx *ctx,  in io_check_restriction()
7047 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_init_req()
7109 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_submit_sqe()
7195 				struct io_ring_ctx *ctx)  in io_submit_state_end()
7217 static void io_commit_sqring(struct io_ring_ctx *ctx)  in io_commit_sqring()
7237 static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)  in io_get_sqe()
7261 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)  in io_submit_sqes()
7314 static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_set_wakeup_flag()
7323 static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_clear_wakeup_flag()
7331 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)  in __io_sq_thread()
7372 	struct io_ring_ctx *ctx;  in io_sqd_update_thread_idle()
7399 	struct io_ring_ctx *ctx;  in io_sq_thread()
7484 	struct io_ring_ctx *ctx;
7491 	struct io_ring_ctx *ctx = iowq->ctx;  in io_should_wake()
7529 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,  in io_cqring_wait_schedule()
7551 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,  in io_cqring_wait()
7656 	struct io_ring_ctx *ctx = node->rsrc_data->ctx;  in io_rsrc_node_ref_zero()
7678 static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)  in io_rsrc_node_alloc()
7697 static void io_rsrc_node_switch(struct io_ring_ctx *ctx,  in io_rsrc_node_switch()
7722 static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)  in io_rsrc_node_switch_start()
7730 static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)  in io_rsrc_ref_quiesce()
7786 static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,  in io_rsrc_data_alloc()
7839 static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)  in __io_sqe_files_unregister()
7866 static int io_sqe_files_unregister(struct io_ring_ctx *ctx)  in io_sqe_files_unregister()
7928 static void io_sq_thread_finish(struct io_ring_ctx *ctx)  in io_sq_thread_finish()
7945 	struct io_ring_ctx *ctx_attach;  in io_attach_sq_data()
8009 static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)  in __io_sqe_files_scm()
8063 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8094 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8100 static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_file_put()
8166 	struct io_ring_ctx *ctx = rsrc_data->ctx;  in __io_rsrc_put_work()
8196 	struct io_ring_ctx *ctx;  in io_rsrc_put_work()
8199 	ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);  in io_rsrc_put_work()
8212 static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_files_register()
8294 static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,  in io_sqe_file_register()
8355 	struct io_ring_ctx *ctx = req->ctx;  in io_install_fixed_file()
8411 	struct io_ring_ctx *ctx = req->ctx;  in io_close_fixed()
8446 static int __io_sqe_files_update(struct io_ring_ctx *ctx,  in __io_sqe_files_update()
8526 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,  in io_init_wq_offload()
8559 				       struct io_ring_ctx *ctx)  in io_uring_alloc_task_context()
8606 static int io_sq_offload_create(struct io_ring_ctx *ctx,  in io_sq_offload_create()
8717 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_unaccount_mem()
8726 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_account_mem()
8791 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)  in io_buffer_unmap()
8806 static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_buf_put()
8812 static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in __io_sqe_buffers_unregister()
8825 static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in io_sqe_buffers_unregister()
8838 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,  in io_copy_iov()
8872 static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,  in headpage_already_acct()
8900 static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,  in io_buffer_account_pin()
8932 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,  in io_sqe_buffer_register()
9032 static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)  in io_buffers_map_alloc()
9062 static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_buffers_register()
9114 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,  in __io_sqe_buffers_update()
9174 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)  in io_eventfd_register()
9196 static int io_eventfd_unregister(struct io_ring_ctx *ctx)  in io_eventfd_unregister()
9207 static void io_destroy_buffers(struct io_ring_ctx *ctx)  in io_destroy_buffers()
9228 static void io_req_caches_free(struct io_ring_ctx *ctx)  in io_req_caches_free()
9250 static void io_ring_ctx_free(struct io_ring_ctx *ctx)  in io_ring_ctx_free()
9309 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_poll()
9340 static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)  in io_unregister_personality()
9356 	struct io_ring_ctx		*ctx;
9383 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);  in io_ring_exit_work()
9450 static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_kill_timeouts()
9473 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)  in io_ring_ctx_wait_and_kill()
9504 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_release()
9523 		struct io_ring_ctx *ctx = req->ctx;  in io_cancel_task_cb()
9535 static bool io_cancel_defer_files(struct io_ring_ctx *ctx,  in io_cancel_defer_files()
9561 static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)  in io_uring_try_cancel_iowq()
9585 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,  in io_uring_try_cancel_requests()
9628 static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in __io_uring_add_tctx_node()
9674 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in io_uring_add_tctx_node()
9755 	struct io_ring_ctx *ctx;  in io_uring_cancel_generic()
9819 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_validate_mmap_request()
9886 static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)  in io_sqpoll_wait_sq()
9938 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE6()
10062 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)  in __io_uring_show_fdinfo()
10125 	struct io_ring_ctx *ctx = f->private_data;  in io_uring_show_fdinfo()
10147 static int io_allocate_scq_urings(struct io_ring_ctx *ctx,  in io_allocate_scq_urings()
10189 static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)  in io_uring_install_fd()
10212 static struct file *io_uring_get_file(struct io_ring_ctx *ctx)  in io_uring_get_file()
10240 	struct io_ring_ctx *ctx;  in io_uring_create()
10396 static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)  in io_probe()
10435 static int io_register_personality(struct io_ring_ctx *ctx)  in io_register_personality()
10452 static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,  in io_register_restrictions()
10522 static int io_register_enable_rings(struct io_ring_ctx *ctx)  in io_register_enable_rings()
10536 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,  in __io_register_rsrc_update()
10560 static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_files_update()
10573 static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc_update()
10587 static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc()
10613 static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,  in io_register_iowq_aff()
10640 static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)  in io_unregister_iowq_aff()
10650 static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,  in io_register_iowq_max_workers()
10756 static int io_ctx_quiesce(struct io_ring_ctx *ctx)  in io_ctx_quiesce()
10783 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,  in __io_uring_register()
10928 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE4()