26 void get_io_context(struct io_context *ioc)  in get_io_context()  argument
28 	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);  in get_io_context()
29 	atomic_long_inc(&ioc->refcount);  in get_io_context()
62 	struct io_context *ioc = icq->ioc;  in ioc_destroy_icq()  local
66 	lockdep_assert_held(&ioc->lock);  in ioc_destroy_icq()
68 	radix_tree_delete(&ioc->icq_tree, icq->q->id);  in ioc_destroy_icq()
77 	if (rcu_access_pointer(ioc->icq_hint) == icq)  in ioc_destroy_icq()
78 		rcu_assign_pointer(ioc->icq_hint, NULL);  in ioc_destroy_icq()
96 	struct io_context *ioc = container_of(work, struct io_context,  in ioc_release_fn()  local
106 	spin_lock_irqsave_nested(&ioc->lock, flags, 1);  in ioc_release_fn()
108 	while (!hlist_empty(&ioc->icq_list)) {  in ioc_release_fn()
109 		struct io_cq *icq = hlist_entry(ioc->icq_list.first,  in ioc_release_fn()
117 			spin_unlock_irqrestore(&ioc->lock, flags);  in ioc_release_fn()
119 			spin_lock_irqsave_nested(&ioc->lock, flags, 1);  in ioc_release_fn()
123 	spin_unlock_irqrestore(&ioc->lock, flags);  in ioc_release_fn()
125 	kmem_cache_free(iocontext_cachep, ioc);  in ioc_release_fn()
135 void put_io_context(struct io_context *ioc)  in put_io_context()  argument
140 	if (ioc == NULL)  in put_io_context()
143 	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);  in put_io_context()
149 	if (atomic_long_dec_and_test(&ioc->refcount)) {  in put_io_context()
150 		spin_lock_irqsave(&ioc->lock, flags);  in put_io_context()
151 		if (!hlist_empty(&ioc->icq_list))  in put_io_context()
153 					&ioc->release_work);  in put_io_context()
156 		spin_unlock_irqrestore(&ioc->lock, flags);  in put_io_context()
160 		kmem_cache_free(iocontext_cachep, ioc);  in put_io_context()
170 void put_io_context_active(struct io_context *ioc)  in put_io_context_active()  argument
175 	if (!atomic_dec_and_test(&ioc->active_ref)) {  in put_io_context_active()
176 		put_io_context(ioc);  in put_io_context_active()
185 	spin_lock_irqsave_nested(&ioc->lock, flags, 1);  in put_io_context_active()
186 	hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) {  in put_io_context_active()
192 	spin_unlock_irqrestore(&ioc->lock, flags);  in put_io_context_active()
194 	put_io_context(ioc);  in put_io_context_active()
200 	struct io_context *ioc;  in exit_io_context()  local
203 	ioc = task->io_context;  in exit_io_context()
207 	atomic_dec(&ioc->nr_tasks);  in exit_io_context()
208 	put_io_context_active(ioc);  in exit_io_context()
218 		struct io_context *ioc = icq->ioc;  in __ioc_clear_queue()  local
220 		spin_lock_irqsave(&ioc->lock, flags);  in __ioc_clear_queue()
222 		spin_unlock_irqrestore(&ioc->lock, flags);  in __ioc_clear_queue()
245 	struct io_context *ioc;  in create_task_io_context()  local
248 	ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,  in create_task_io_context()
250 	if (unlikely(!ioc))  in create_task_io_context()
254 	atomic_long_set(&ioc->refcount, 1);  in create_task_io_context()
255 	atomic_set(&ioc->nr_tasks, 1);  in create_task_io_context()
256 	atomic_set(&ioc->active_ref, 1);  in create_task_io_context()
257 	spin_lock_init(&ioc->lock);  in create_task_io_context()
258 	INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC);  in create_task_io_context()
259 	INIT_HLIST_HEAD(&ioc->icq_list);  in create_task_io_context()
260 	INIT_WORK(&ioc->release_work, ioc_release_fn);  in create_task_io_context()
272 		task->io_context = ioc;  in create_task_io_context()
274 		kmem_cache_free(iocontext_cachep, ioc);  in create_task_io_context()
299 	struct io_context *ioc;  in get_task_io_context()  local
305 		ioc = task->io_context;  in get_task_io_context()
306 		if (likely(ioc)) {  in get_task_io_context()
307 			get_io_context(ioc);  in get_task_io_context()
309 			return ioc;  in get_task_io_context()
325 struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q)  in ioc_lookup_icq()  argument
338 	icq = rcu_dereference(ioc->icq_hint);  in ioc_lookup_icq()
342 	icq = radix_tree_lookup(&ioc->icq_tree, q->id);  in ioc_lookup_icq()
344 		rcu_assign_pointer(ioc->icq_hint, icq);	/* allowed to race */  in ioc_lookup_icq()
365 struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,  in ioc_create_icq()  argument
382 	icq->ioc = ioc;  in ioc_create_icq()
389 	spin_lock(&ioc->lock);  in ioc_create_icq()
391 	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {  in ioc_create_icq()
392 		hlist_add_head(&icq->ioc_node, &ioc->icq_list);  in ioc_create_icq()
398 		icq = ioc_lookup_icq(ioc, q);  in ioc_create_icq()
403 	spin_unlock(&ioc->lock);  in ioc_create_icq()