49 	struct io_uring_task *tctx = tsk->io_uring;  in __io_uring_free()  local
51 	WARN_ON_ONCE(!xa_empty(&tctx->xa));  in __io_uring_free()
52 	WARN_ON_ONCE(tctx->io_wq);  in __io_uring_free()
53 	WARN_ON_ONCE(tctx->cached_refs);  in __io_uring_free()
55 	percpu_counter_destroy(&tctx->inflight);  in __io_uring_free()
56 	kfree(tctx);  in __io_uring_free()
63 	struct io_uring_task *tctx;  in io_uring_alloc_task_context()  local
66 	tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);  in io_uring_alloc_task_context()
67 	if (unlikely(!tctx))  in io_uring_alloc_task_context()
70 	ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);  in io_uring_alloc_task_context()
72 		kfree(tctx);  in io_uring_alloc_task_context()
76 	tctx->io_wq = io_init_wq_offload(ctx, task);  in io_uring_alloc_task_context()
77 	if (IS_ERR(tctx->io_wq)) {  in io_uring_alloc_task_context()
78 		ret = PTR_ERR(tctx->io_wq);  in io_uring_alloc_task_context()
79 		percpu_counter_destroy(&tctx->inflight);  in io_uring_alloc_task_context()
80 		kfree(tctx);  in io_uring_alloc_task_context()
84 	xa_init(&tctx->xa);  in io_uring_alloc_task_context()
85 	init_waitqueue_head(&tctx->wait);  in io_uring_alloc_task_context()
86 	atomic_set(&tctx->in_idle, 0);  in io_uring_alloc_task_context()
87 	atomic_set(&tctx->inflight_tracked, 0);  in io_uring_alloc_task_context()
88 	task->io_uring = tctx;  in io_uring_alloc_task_context()
89 	init_llist_head(&tctx->task_list);  in io_uring_alloc_task_context()
90 	init_task_work(&tctx->task_work, tctx_task_work);  in io_uring_alloc_task_context()
96 	struct io_uring_task *tctx = current->io_uring;  in __io_uring_add_tctx_node()  local
100 	if (unlikely(!tctx)) {  in __io_uring_add_tctx_node()
105 		tctx = current->io_uring;  in __io_uring_add_tctx_node()
110 			ret = io_wq_max_workers(tctx->io_wq, limits);  in __io_uring_add_tctx_node()
115 	if (!xa_load(&tctx->xa, (unsigned long)ctx)) {  in __io_uring_add_tctx_node()
122 		ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,  in __io_uring_add_tctx_node()
157 	struct io_uring_task *tctx = current->io_uring;  in io_uring_del_tctx_node()  local
160 	if (!tctx)  in io_uring_del_tctx_node()
162 	node = xa_erase(&tctx->xa, index);  in io_uring_del_tctx_node()
173 	if (tctx->last == node->ctx)  in io_uring_del_tctx_node()
174 		tctx->last = NULL;  in io_uring_del_tctx_node()
178 __cold void io_uring_clean_tctx(struct io_uring_task *tctx)  in io_uring_clean_tctx()  argument
180 	struct io_wq *wq = tctx->io_wq;  in io_uring_clean_tctx()
184 	xa_for_each(&tctx->xa, index, node) {  in io_uring_clean_tctx()
194 		tctx->io_wq = NULL;  in io_uring_clean_tctx()
200 	struct io_uring_task *tctx = current->io_uring;  in io_uring_unreg_ringfd()  local
204 		if (tctx->registered_rings[i]) {  in io_uring_unreg_ringfd()
205 			fput(tctx->registered_rings[i]);  in io_uring_unreg_ringfd()
206 			tctx->registered_rings[i] = NULL;  in io_uring_unreg_ringfd()
211 static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd,  in io_ring_add_registered_fd()  argument
219 		if (tctx->registered_rings[offset])  in io_ring_add_registered_fd()
229 		tctx->registered_rings[offset] = file;  in io_ring_add_registered_fd()
249 	struct io_uring_task *tctx;  in io_ringfd_register()  local
261 	tctx = current->io_uring;  in io_ringfd_register()
287 		ret = io_ring_add_registered_fd(tctx, reg.data, start, end);  in io_ringfd_register()
293 			fput(tctx->registered_rings[reg.offset]);  in io_ringfd_register()
294 			tctx->registered_rings[reg.offset] = NULL;  in io_ringfd_register()
307 	struct io_uring_task *tctx = current->io_uring;  in io_ringfd_unregister()  local
313 	if (!tctx)  in io_ringfd_unregister()
327 		if (tctx->registered_rings[reg.offset]) {  in io_ringfd_unregister()
328 			fput(tctx->registered_rings[reg.offset]);  in io_ringfd_unregister()
329 			tctx->registered_rings[reg.offset] = NULL;  in io_ringfd_unregister()