54 	struct blk_stat_callback *cb;  in blk_stat_add()  local
64 	list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {  in blk_stat_add()
65 		if (!blk_stat_is_active(cb))  in blk_stat_add()
68 		bucket = cb->bucket_fn(rq);  in blk_stat_add()
72 		stat = &get_cpu_ptr(cb->cpu_stat)[bucket];  in blk_stat_add()
74 		put_cpu_ptr(cb->cpu_stat);  in blk_stat_add()
81 	struct blk_stat_callback *cb = from_timer(cb, t, timer);  in blk_stat_timer_fn()  local
85 	for (bucket = 0; bucket < cb->buckets; bucket++)  in blk_stat_timer_fn()
86 		blk_rq_stat_init(&cb->stat[bucket]);  in blk_stat_timer_fn()
91 		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);  in blk_stat_timer_fn()
92 		for (bucket = 0; bucket < cb->buckets; bucket++) {  in blk_stat_timer_fn()
93 			blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);  in blk_stat_timer_fn()
98 	cb->timer_fn(cb);  in blk_stat_timer_fn()
106 	struct blk_stat_callback *cb;  in blk_stat_alloc_callback()  local
108 	cb = kmalloc(sizeof(*cb), GFP_KERNEL);  in blk_stat_alloc_callback()
109 	if (!cb)  in blk_stat_alloc_callback()
112 	cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),  in blk_stat_alloc_callback()
114 	if (!cb->stat) {  in blk_stat_alloc_callback()
115 		kfree(cb);  in blk_stat_alloc_callback()
118 	cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),  in blk_stat_alloc_callback()
120 	if (!cb->cpu_stat) {  in blk_stat_alloc_callback()
121 		kfree(cb->stat);  in blk_stat_alloc_callback()
122 		kfree(cb);  in blk_stat_alloc_callback()
126 	cb->timer_fn = timer_fn;  in blk_stat_alloc_callback()
127 	cb->bucket_fn = bucket_fn;  in blk_stat_alloc_callback()
128 	cb->data = data;  in blk_stat_alloc_callback()
129 	cb->buckets = buckets;  in blk_stat_alloc_callback()
130 	timer_setup(&cb->timer, blk_stat_timer_fn, 0);  in blk_stat_alloc_callback()
132 	return cb;  in blk_stat_alloc_callback()
136 			   struct blk_stat_callback *cb)  in blk_stat_add_callback()  argument
144 		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);  in blk_stat_add_callback()
145 		for (bucket = 0; bucket < cb->buckets; bucket++)  in blk_stat_add_callback()
150 	list_add_tail_rcu(&cb->list, &q->stats->callbacks);  in blk_stat_add_callback()
156 			      struct blk_stat_callback *cb)  in blk_stat_remove_callback()  argument
159 	list_del_rcu(&cb->list);  in blk_stat_remove_callback()
164 	del_timer_sync(&cb->timer);  in blk_stat_remove_callback()
169 	struct blk_stat_callback *cb;  in blk_stat_free_callback_rcu()  local
171 	cb = container_of(head, struct blk_stat_callback, rcu);  in blk_stat_free_callback_rcu()
172 	free_percpu(cb->cpu_stat);  in blk_stat_free_callback_rcu()
173 	kfree(cb->stat);  in blk_stat_free_callback_rcu()
174 	kfree(cb);  in blk_stat_free_callback_rcu()
177 void blk_stat_free_callback(struct blk_stat_callback *cb)  in blk_stat_free_callback()  argument
179 	if (cb)  in blk_stat_free_callback()
180 		call_rcu(&cb->rcu, blk_stat_free_callback_rcu);  in blk_stat_free_callback()