35 	struct io_cq *icq = container_of(head, struct io_cq, __rcu_head);  in icq_free_icq_rcu()  local
37 	kmem_cache_free(icq->__rcu_icq_cache, icq);  in icq_free_icq_rcu()
44 static void ioc_exit_icq(struct io_cq *icq)  in ioc_exit_icq()  argument
46 	struct elevator_type *et = icq->q->elevator->type;  in ioc_exit_icq()
48 	if (icq->flags & ICQ_EXITED)  in ioc_exit_icq()
52 		et->ops.mq.exit_icq(icq);  in ioc_exit_icq()
54 		et->ops.sq.elevator_exit_icq_fn(icq);  in ioc_exit_icq()
56 	icq->flags |= ICQ_EXITED;  in ioc_exit_icq()
63 static void ioc_destroy_icq(struct io_cq *icq)  in ioc_destroy_icq()  argument
65 	struct io_context *ioc = icq->ioc;  in ioc_destroy_icq()
66 	struct request_queue *q = icq->q;  in ioc_destroy_icq()
71 	radix_tree_delete(&ioc->icq_tree, icq->q->id);  in ioc_destroy_icq()
72 	hlist_del_init(&icq->ioc_node);  in ioc_destroy_icq()
73 	list_del_init(&icq->q_node);  in ioc_destroy_icq()
80 	if (rcu_access_pointer(ioc->icq_hint) == icq)  in ioc_destroy_icq()
83 	ioc_exit_icq(icq);  in ioc_destroy_icq()
89 	icq->__rcu_icq_cache = et->icq_cache;  in ioc_destroy_icq()
90 	call_rcu(&icq->__rcu_head, icq_free_icq_rcu);  in ioc_destroy_icq()
112 		struct io_cq *icq = hlist_entry(ioc->icq_list.first,  in ioc_release_fn()  local
114 		struct request_queue *q = icq->q;  in ioc_release_fn()
117 			ioc_destroy_icq(icq);  in ioc_release_fn()
178 	struct io_cq *icq;  in put_io_context_active()  local
192 	hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) {  in put_io_context_active()
193 		if (icq->flags & ICQ_EXITED)  in put_io_context_active()
196 		et = icq->q->elevator->type;  in put_io_context_active()
198 			ioc_exit_icq(icq);  in put_io_context_active()
200 			if (spin_trylock(icq->q->queue_lock)) {  in put_io_context_active()
201 				ioc_exit_icq(icq);  in put_io_context_active()
202 				spin_unlock(icq->q->queue_lock);  in put_io_context_active()
234 		struct io_cq *icq = list_entry(icq_list->next,  in __ioc_clear_queue()  local
236 		struct io_context *ioc = icq->ioc;  in __ioc_clear_queue()
239 		ioc_destroy_icq(icq);  in __ioc_clear_queue()
351 	struct io_cq *icq;  in ioc_lookup_icq()  local
362 	icq = rcu_dereference(ioc->icq_hint);  in ioc_lookup_icq()
363 	if (icq && icq->q == q)  in ioc_lookup_icq()
366 	icq = radix_tree_lookup(&ioc->icq_tree, q->id);  in ioc_lookup_icq()
367 	if (icq && icq->q == q)  in ioc_lookup_icq()
368 		rcu_assign_pointer(ioc->icq_hint, icq);	/* allowed to race */  in ioc_lookup_icq()
370 		icq = NULL;  in ioc_lookup_icq()
373 	return icq;  in ioc_lookup_icq()
393 	struct io_cq *icq;  in ioc_create_icq()  local
396 	icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,  in ioc_create_icq()
398 	if (!icq)  in ioc_create_icq()
402 		kmem_cache_free(et->icq_cache, icq);  in ioc_create_icq()
406 	icq->ioc = ioc;  in ioc_create_icq()
407 	icq->q = q;  in ioc_create_icq()
408 	INIT_LIST_HEAD(&icq->q_node);  in ioc_create_icq()
409 	INIT_HLIST_NODE(&icq->ioc_node);  in ioc_create_icq()
415 	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {  in ioc_create_icq()
416 		hlist_add_head(&icq->ioc_node, &ioc->icq_list);  in ioc_create_icq()
417 		list_add(&icq->q_node, &q->icq_list);  in ioc_create_icq()
419 			et->ops.mq.init_icq(icq);  in ioc_create_icq()
421 			et->ops.sq.elevator_init_icq_fn(icq);  in ioc_create_icq()
423 		kmem_cache_free(et->icq_cache, icq);  in ioc_create_icq()
424 		icq = ioc_lookup_icq(ioc, q);  in ioc_create_icq()
425 		if (!icq)  in ioc_create_icq()
432 	return icq;  in ioc_create_icq()