47 struct blkcg blkcg_root;
128 	css_put(&blkg->blkcg->css);  in __blkg_release()
187 	return &bio->bi_blkg->blkcg->css;  in bio_blkcg_css()
197 static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)  in blkcg_parent()  argument
199 	return css_to_blkcg(blkcg->css.parent);  in blkcg_parent()
210 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,  in blkg_alloc()  argument
236 	blkg->blkcg = blkcg;  in blkg_alloc()
250 		pd = pol->pd_alloc_fn(gfp_mask, disk->queue, blkcg);  in blkg_alloc()
270 static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,  in blkg_create()  argument
285 	if (!css_tryget_online(&blkcg->css)) {  in blkg_create()
292 		new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN);  in blkg_create()
301 	if (blkcg_parent(blkcg)) {  in blkg_create()
302 		blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);  in blkg_create()
319 	spin_lock(&blkcg->lock);  in blkg_create()
320 	ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);  in blkg_create()
322 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);  in blkg_create()
333 	spin_unlock(&blkcg->lock);  in blkg_create()
343 	css_put(&blkcg->css);  in blkg_create()
362 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,  in blkg_lookup_create()  argument
371 	blkg = blkg_lookup(blkcg, q);  in blkg_lookup_create()
376 	blkg = blkg_lookup(blkcg, q);  in blkg_lookup_create()
378 		if (blkcg != &blkcg_root &&  in blkg_lookup_create()
379 		    blkg != rcu_dereference(blkcg->blkg_hint))  in blkg_lookup_create()
380 			rcu_assign_pointer(blkcg->blkg_hint, blkg);  in blkg_lookup_create()
390 		struct blkcg *pos = blkcg;  in blkg_lookup_create()
391 		struct blkcg *parent = blkcg_parent(blkcg);  in blkg_lookup_create()
410 		if (pos == blkcg)  in blkg_lookup_create()
421 	struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy()  local
425 	lockdep_assert_held(&blkcg->lock);  in blkg_destroy()
440 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);  in blkg_destroy()
449 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)  in blkg_destroy()
450 		rcu_assign_pointer(blkcg->blkg_hint, NULL);  in blkg_destroy()
468 		struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy_all()  local
470 		spin_lock(&blkcg->lock);  in blkg_destroy_all()
472 		spin_unlock(&blkcg->lock);  in blkg_destroy_all()
493 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_reset_stats()  local
498 	spin_lock_irq(&blkcg->lock);  in blkcg_reset_stats()
505 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_reset_stats()
521 	spin_unlock_irq(&blkcg->lock);  in blkcg_reset_stats()
551 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,  in blkcg_print_blkgs()  argument
561 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_blkgs()
644 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,  in blkg_conf_prep()  argument
676 	blkg = blkg_lookup(blkcg, q);  in blkg_conf_prep()
685 		struct blkcg *pos = blkcg;  in blkg_conf_prep()
686 		struct blkcg *parent;  in blkg_conf_prep()
689 		parent = blkcg_parent(blkcg);  in blkg_conf_prep()
733 		if (pos == blkcg)  in blkg_conf_prep()
829 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_rstat_flush()  local
838 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_rstat_flush()
966 	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));  in blkcg_print_stat()  local
972 		cgroup_rstat_flush(blkcg->css.cgroup);  in blkcg_print_stat()
975 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_stat()
1039 static void blkcg_destroy_blkgs(struct blkcg *blkcg)  in blkcg_destroy_blkgs()  argument
1043 	spin_lock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1045 	while (!hlist_empty(&blkcg->blkg_list)) {  in blkcg_destroy_blkgs()
1046 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,  in blkcg_destroy_blkgs()
1056 			spin_unlock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1058 			spin_lock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1066 	spin_unlock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1093 	struct blkcg *blkcg = css_to_blkcg(blkcg_css);  in blkcg_unpin_online()  local
1096 		if (!refcount_dec_and_test(&blkcg->online_pin))  in blkcg_unpin_online()
1098 		blkcg_destroy_blkgs(blkcg);  in blkcg_unpin_online()
1099 		blkcg = blkcg_parent(blkcg);  in blkcg_unpin_online()
1100 	} while (blkcg);  in blkcg_unpin_online()
1122 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_free()  local
1127 	list_del(&blkcg->all_blkcgs_node);  in blkcg_css_free()
1130 		if (blkcg->cpd[i])  in blkcg_css_free()
1131 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_free()
1135 	kfree(blkcg);  in blkcg_css_free()
1141 	struct blkcg *blkcg;  in blkcg_css_alloc()  local
1148 		blkcg = &blkcg_root;  in blkcg_css_alloc()
1150 		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);  in blkcg_css_alloc()
1151 		if (!blkcg) {  in blkcg_css_alloc()
1175 		blkcg->cpd[i] = cpd;  in blkcg_css_alloc()
1176 		cpd->blkcg = blkcg;  in blkcg_css_alloc()
1182 	spin_lock_init(&blkcg->lock);  in blkcg_css_alloc()
1183 	refcount_set(&blkcg->online_pin, 1);  in blkcg_css_alloc()
1184 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);  in blkcg_css_alloc()
1185 	INIT_HLIST_HEAD(&blkcg->blkg_list);  in blkcg_css_alloc()
1187 	INIT_LIST_HEAD(&blkcg->cgwb_list);  in blkcg_css_alloc()
1189 	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);  in blkcg_css_alloc()
1192 	return &blkcg->css;  in blkcg_css_alloc()
1196 		if (blkcg->cpd[i])  in blkcg_css_alloc()
1197 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_alloc()
1199 	if (blkcg != &blkcg_root)  in blkcg_css_alloc()
1200 		kfree(blkcg);  in blkcg_css_alloc()
1208 	struct blkcg *parent = blkcg_parent(css_to_blkcg(css));  in blkcg_css_online()
1289 		struct blkcg *blkcg;  in blkcg_bind()  local
1294 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)  in blkcg_bind()
1295 			if (blkcg->cpd[pol->plid])  in blkcg_bind()
1296 				pol->cpd_bind_fn(blkcg->cpd[pol->plid]);  in blkcg_bind()
1374 					      blkg->blkcg);  in blkcg_activate_policy()
1392 						       blkg->blkcg);  in blkcg_activate_policy()
1426 		struct blkcg *blkcg = blkg->blkcg;  in blkcg_activate_policy()  local
1428 		spin_lock(&blkcg->lock);  in blkcg_activate_policy()
1433 		spin_unlock(&blkcg->lock);  in blkcg_activate_policy()
1465 		struct blkcg *blkcg = blkg->blkcg;  in blkcg_deactivate_policy()  local
1467 		spin_lock(&blkcg->lock);  in blkcg_deactivate_policy()
1474 		spin_unlock(&blkcg->lock);  in blkcg_deactivate_policy()
1486 	struct blkcg *blkcg;  in blkcg_free_all_cpd()  local
1488 	list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_free_all_cpd()
1489 		if (blkcg->cpd[pol->plid]) {  in blkcg_free_all_cpd()
1490 			pol->cpd_free_fn(blkcg->cpd[pol->plid]);  in blkcg_free_all_cpd()
1491 			blkcg->cpd[pol->plid] = NULL;  in blkcg_free_all_cpd()
1505 	struct blkcg *blkcg;  in blkcg_policy_register()  local
1532 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_register()
1539 			blkcg->cpd[pol->plid] = cpd;  in blkcg_policy_register()
1540 			cpd->blkcg = blkcg;  in blkcg_policy_register()
1755 	struct blkcg *blkcg;  in blkcg_maybe_throttle_current()  local
1766 	blkcg = css_to_blkcg(blkcg_css());  in blkcg_maybe_throttle_current()
1767 	if (!blkcg)  in blkcg_maybe_throttle_current()
1769 	blkg = blkg_lookup(blkcg, q);  in blkcg_maybe_throttle_current()
1966 		cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu);  in blk_cgroup_bio_start()