37 static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);
49 struct blkcg blkcg_root;
83 static int init_blkcg_llists(struct blkcg *blkcg)  in init_blkcg_llists()  argument
87 	blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL);  in init_blkcg_llists()
88 	if (!blkcg->lhead)  in init_blkcg_llists()
92 		init_llist_head(per_cpu_ptr(blkcg->lhead, cpu));  in init_blkcg_llists()
172 	struct blkcg *blkcg = blkg->blkcg;  in __blkg_release()  local
185 		__blkcg_rstat_flush(blkcg, cpu);  in __blkg_release()
188 	css_put(&blkg->blkcg->css);  in __blkg_release()
282 	return &bio->bi_blkg->blkcg->css;  in bio_blkcg_css()
292 static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)  in blkcg_parent()  argument
294 	return css_to_blkcg(blkcg->css.parent);  in blkcg_parent()
305 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,  in blkg_alloc()  argument
325 	blkg->blkcg = blkcg;  in blkg_alloc()
346 		pd = pol->pd_alloc_fn(disk, blkcg, gfp_mask);  in blkg_alloc()
375 static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,  in blkg_create()  argument
390 	if (!css_tryget_online(&blkcg->css)) {  in blkg_create()
397 		new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN);  in blkg_create()
406 	if (blkcg_parent(blkcg)) {  in blkg_create()
407 		blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);  in blkg_create()
424 	spin_lock(&blkcg->lock);  in blkg_create()
425 	ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);  in blkg_create()
427 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);  in blkg_create()
441 	spin_unlock(&blkcg->lock);  in blkg_create()
451 	css_put(&blkcg->css);  in blkg_create()
471 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,  in blkg_lookup_create()  argument
480 	blkg = blkg_lookup(blkcg, q);  in blkg_lookup_create()
485 	blkg = blkg_lookup(blkcg, q);  in blkg_lookup_create()
487 		if (blkcg != &blkcg_root &&  in blkg_lookup_create()
488 		    blkg != rcu_dereference(blkcg->blkg_hint))  in blkg_lookup_create()
489 			rcu_assign_pointer(blkcg->blkg_hint, blkg);  in blkg_lookup_create()
499 		struct blkcg *pos = blkcg;  in blkg_lookup_create()
500 		struct blkcg *parent = blkcg_parent(blkcg);  in blkg_lookup_create()
519 		if (pos == blkcg)  in blkg_lookup_create()
530 	struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy()  local
534 	lockdep_assert_held(&blkcg->lock);  in blkg_destroy()
557 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);  in blkg_destroy()
565 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)  in blkg_destroy()
566 		rcu_assign_pointer(blkcg->blkg_hint, NULL);  in blkg_destroy()
584 		struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy_all()  local
589 		spin_lock(&blkcg->lock);  in blkg_destroy_all()
591 		spin_unlock(&blkcg->lock);  in blkg_destroy_all()
612 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_reset_stats()  local
617 	spin_lock_irq(&blkcg->lock);  in blkcg_reset_stats()
624 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_reset_stats()
645 	spin_unlock_irq(&blkcg->lock);  in blkcg_reset_stats()
675 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,  in blkcg_print_blkgs()  argument
685 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_blkgs()
799 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,  in blkg_conf_prep()  argument
830 	blkg = blkg_lookup(blkcg, q);  in blkg_conf_prep()
839 		struct blkcg *pos = blkcg;  in blkg_conf_prep()
840 		struct blkcg *parent;  in blkg_conf_prep()
843 		parent = blkcg_parent(blkcg);  in blkg_conf_prep()
885 		if (pos == blkcg)  in blkg_conf_prep()
984 static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)  in __blkcg_rstat_flush()  argument
986 	struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);  in __blkcg_rstat_flush()
1146 	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));  in blkcg_print_stat()  local
1152 		cgroup_rstat_flush(blkcg->css.cgroup);  in blkcg_print_stat()
1155 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_stat()
1219 static void blkcg_destroy_blkgs(struct blkcg *blkcg)  in blkcg_destroy_blkgs()  argument
1223 	spin_lock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1225 	while (!hlist_empty(&blkcg->blkg_list)) {  in blkcg_destroy_blkgs()
1226 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,  in blkcg_destroy_blkgs()
1236 			spin_unlock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1238 			spin_lock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1246 	spin_unlock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1273 	struct blkcg *blkcg = css_to_blkcg(blkcg_css);  in blkcg_unpin_online()  local
1276 		if (!refcount_dec_and_test(&blkcg->online_pin))  in blkcg_unpin_online()
1278 		blkcg_destroy_blkgs(blkcg);  in blkcg_unpin_online()
1279 		blkcg = blkcg_parent(blkcg);  in blkcg_unpin_online()
1280 	} while (blkcg);  in blkcg_unpin_online()
1302 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_free()  local
1307 	list_del(&blkcg->all_blkcgs_node);  in blkcg_css_free()
1310 		if (blkcg->cpd[i])  in blkcg_css_free()
1311 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_free()
1315 	free_percpu(blkcg->lhead);  in blkcg_css_free()
1316 	kfree(blkcg);  in blkcg_css_free()
1322 	struct blkcg *blkcg;  in blkcg_css_alloc()  local
1328 		blkcg = &blkcg_root;  in blkcg_css_alloc()
1330 		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);  in blkcg_css_alloc()
1331 		if (!blkcg)  in blkcg_css_alloc()
1335 	if (init_blkcg_llists(blkcg))  in blkcg_css_alloc()
1355 		blkcg->cpd[i] = cpd;  in blkcg_css_alloc()
1356 		cpd->blkcg = blkcg;  in blkcg_css_alloc()
1360 	spin_lock_init(&blkcg->lock);  in blkcg_css_alloc()
1361 	refcount_set(&blkcg->online_pin, 1);  in blkcg_css_alloc()
1362 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);  in blkcg_css_alloc()
1363 	INIT_HLIST_HEAD(&blkcg->blkg_list);  in blkcg_css_alloc()
1365 	INIT_LIST_HEAD(&blkcg->cgwb_list);  in blkcg_css_alloc()
1367 	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);  in blkcg_css_alloc()
1370 	return &blkcg->css;  in blkcg_css_alloc()
1374 		if (blkcg->cpd[i])  in blkcg_css_alloc()
1375 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_alloc()
1376 	free_percpu(blkcg->lhead);  in blkcg_css_alloc()
1378 	if (blkcg != &blkcg_root)  in blkcg_css_alloc()
1379 		kfree(blkcg);  in blkcg_css_alloc()
1387 	struct blkcg *parent = blkcg_parent(css_to_blkcg(css));  in blkcg_css_online()
1526 			pd = pol->pd_alloc_fn(disk, blkg->blkcg,  in blkcg_activate_policy()
1544 			pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg,  in blkcg_activate_policy()
1552 		spin_lock(&blkg->blkcg->lock);  in blkcg_activate_policy()
1565 		spin_unlock(&blkg->blkcg->lock);  in blkcg_activate_policy()
1585 		struct blkcg *blkcg = blkg->blkcg;  in blkcg_activate_policy()  local
1588 		spin_lock(&blkcg->lock);  in blkcg_activate_policy()
1597 		spin_unlock(&blkcg->lock);  in blkcg_activate_policy()
1631 		struct blkcg *blkcg = blkg->blkcg;  in blkcg_deactivate_policy()  local
1633 		spin_lock(&blkcg->lock);  in blkcg_deactivate_policy()
1640 		spin_unlock(&blkcg->lock);  in blkcg_deactivate_policy()
1653 	struct blkcg *blkcg;  in blkcg_free_all_cpd()  local
1655 	list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_free_all_cpd()
1656 		if (blkcg->cpd[pol->plid]) {  in blkcg_free_all_cpd()
1657 			pol->cpd_free_fn(blkcg->cpd[pol->plid]);  in blkcg_free_all_cpd()
1658 			blkcg->cpd[pol->plid] = NULL;  in blkcg_free_all_cpd()
1672 	struct blkcg *blkcg;  in blkcg_policy_register()  local
1699 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_register()
1706 			blkcg->cpd[pol->plid] = cpd;  in blkcg_policy_register()
1707 			cpd->blkcg = blkcg;  in blkcg_policy_register()
1901 	struct blkcg *blkcg;  in blkcg_maybe_throttle_current()  local
1912 	blkcg = css_to_blkcg(blkcg_css());  in blkcg_maybe_throttle_current()
1913 	if (!blkcg)  in blkcg_maybe_throttle_current()
1915 	blkg = blkg_lookup(blkcg, disk->queue);  in blkcg_maybe_throttle_current()
2090 	struct blkcg *blkcg = bio->bi_blkg->blkcg;  in blk_cgroup_bio_start()  local
2099 	if (!cgroup_parent(blkcg->css.cgroup))  in blk_cgroup_bio_start()
2121 		struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);  in blk_cgroup_bio_start()
2128 	cgroup_rstat_updated(blkcg->css.cgroup, cpu);  in blk_cgroup_bio_start()