54 	struct blk_stat_callback *cb;  in blk_stat_add()  local
65 	list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {  in blk_stat_add()
66 		if (!blk_stat_is_active(cb))  in blk_stat_add()
69 		bucket = cb->bucket_fn(rq);  in blk_stat_add()
73 		stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];  in blk_stat_add()
82 	struct blk_stat_callback *cb = from_timer(cb, t, timer);  in blk_stat_timer_fn()  local
86 	for (bucket = 0; bucket < cb->buckets; bucket++)  in blk_stat_timer_fn()
87 		blk_rq_stat_init(&cb->stat[bucket]);  in blk_stat_timer_fn()
92 		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);  in blk_stat_timer_fn()
93 		for (bucket = 0; bucket < cb->buckets; bucket++) {  in blk_stat_timer_fn()
94 			blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);  in blk_stat_timer_fn()
99 	cb->timer_fn(cb);  in blk_stat_timer_fn()
107 	struct blk_stat_callback *cb;  in blk_stat_alloc_callback()  local
109 	cb = kmalloc(sizeof(*cb), GFP_KERNEL);  in blk_stat_alloc_callback()
110 	if (!cb)  in blk_stat_alloc_callback()
113 	cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),  in blk_stat_alloc_callback()
115 	if (!cb->stat) {  in blk_stat_alloc_callback()
116 		kfree(cb);  in blk_stat_alloc_callback()
119 	cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),  in blk_stat_alloc_callback()
121 	if (!cb->cpu_stat) {  in blk_stat_alloc_callback()
122 		kfree(cb->stat);  in blk_stat_alloc_callback()
123 		kfree(cb);  in blk_stat_alloc_callback()
127 	cb->timer_fn = timer_fn;  in blk_stat_alloc_callback()
128 	cb->bucket_fn = bucket_fn;  in blk_stat_alloc_callback()
129 	cb->data = data;  in blk_stat_alloc_callback()
130 	cb->buckets = buckets;  in blk_stat_alloc_callback()
131 	timer_setup(&cb->timer, blk_stat_timer_fn, 0);  in blk_stat_alloc_callback()
133 	return cb;  in blk_stat_alloc_callback()
137 			   struct blk_stat_callback *cb)  in blk_stat_add_callback()  argument
146 		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);  in blk_stat_add_callback()
147 		for (bucket = 0; bucket < cb->buckets; bucket++)  in blk_stat_add_callback()
152 	list_add_tail_rcu(&cb->list, &q->stats->callbacks);  in blk_stat_add_callback()
158 			      struct blk_stat_callback *cb)  in blk_stat_remove_callback()  argument
163 	list_del_rcu(&cb->list);  in blk_stat_remove_callback()
168 	del_timer_sync(&cb->timer);  in blk_stat_remove_callback()
173 	struct blk_stat_callback *cb;  in blk_stat_free_callback_rcu()  local
175 	cb = container_of(head, struct blk_stat_callback, rcu);  in blk_stat_free_callback_rcu()
176 	free_percpu(cb->cpu_stat);  in blk_stat_free_callback_rcu()
177 	kfree(cb->stat);  in blk_stat_free_callback_rcu()
178 	kfree(cb);  in blk_stat_free_callback_rcu()
181 void blk_stat_free_callback(struct blk_stat_callback *cb)  in blk_stat_free_callback()  argument
183 	if (cb)  in blk_stat_free_callback()
184 		call_rcu(&cb->rcu, blk_stat_free_callback_rcu);  in blk_stat_free_callback()