1 /* SPDX-License-Identifier: GPL-2.0 */
22 #include <linux/radix-tree.h>
28 /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
82  * information per blkcg - q pair.
91 	/* the blkg and policy id this per-policy data belongs to */
97  * Policies that need to keep per-blkcg data which is independent from any
101  * cpd_init() is invoked to let each policy handle per-blkcg data.
104 	/* the blkcg and policy id this per-policy data belongs to */
117 	/* all non-root blkcg_gq's are guaranteed to have access to parent */
129 	struct blkg_policy_data		*pd[BLKCG_MAX_POLS];  member
150 typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
151 typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
152 typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
153 typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
154 typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
155 typedef bool (blkcg_pol_stat_pd_fn)(struct blkg_policy_data *pd,
202 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
216  * blkcg_css - find the current css
238  * __bio_blkcg - internal, inconsistent version to get blkcg
253 	if (bio && bio->bi_blkg)  in __bio_blkcg()
254 		return bio->bi_blkg->blkcg;  in __bio_blkcg()
259  * bio_blkcg - grab the blkcg associated with a bio
268 	if (bio && bio->bi_blkg)  in bio_blkcg()
269 		return bio->bi_blkg->blkcg;  in bio_blkcg()
283 		if (atomic_read(&css->cgroup->congestion_count)) {  in blk_cgroup_congested()
287 		css = css->parent;  in blk_cgroup_congested()
294  * bio_issue_as_root_blkg - see if this bio needs to be issued as root blkg
306 	return (bio->bi_opf & (REQ_META | REQ_SWAP)) != 0;  in bio_issue_as_root_blkg()
310  * blkcg_parent - get the parent of a blkcg
317 	return css_to_blkcg(blkcg->css.parent);  in blkcg_parent()
321  * __blkg_lookup - internal version of blkg_lookup()
327  * implementations.  Looks up blkgs for the @blkcg - @q pair regardless of
329  * holding @q->queue_lock and lookup hint is updated on success.
338 		return q->root_blkg;  in __blkg_lookup()
340 	blkg = rcu_dereference(blkcg->blkg_hint);  in __blkg_lookup()
341 	if (blkg && blkg->q == q)  in __blkg_lookup()
348  * blkg_lookup - lookup blkg for the specified blkcg - q pair
352  * Lookup blkg for the @blkcg - @q pair.  This function should be called
363  * blk_queue_root_blkg - return blkg for the (blkcg_root, @q) pair
370 	return q->root_blkg;  in blk_queue_root_blkg()
374  * blkg_to_pdata - get policy private data
378  * Return pointer to private data associated with the @blkg-@pol pair.
383 	return blkg ? blkg->pd[pol->plid] : NULL;  in blkg_to_pd()
389 	return blkcg ? blkcg->cpd[pol->plid] : NULL;  in blkcg_to_cpd()
393  * pdata_to_blkg - get blkg associated with policy private data
394  * @pd: policy private data of interest
396  * @pd is policy private data.  Determine the blkg it's associated with.
398 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)  in pd_to_blkg()  argument
400 	return pd ? pd->blkg : NULL;  in pd_to_blkg()
405 	return cpd ? cpd->blkcg : NULL;  in cpd_to_blkcg()
411  * blkcg_pin_online - pin online state
415  * impedance-match blkg and cgwb lifetimes so that blkg doesn't go offline
420 	refcount_inc(&blkcg->online_pin);  in blkcg_pin_online()
424  * blkcg_unpin_online - unpin online state
427  * This is primarily used to impedance-match blkg and cgwb lifetimes so
435 		if (!refcount_dec_and_test(&blkcg->online_pin))  in blkcg_unpin_online()
443  * blkg_path - format cgroup path of blkg
452 	return cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);  in blkg_path()
456  * blkg_get - get a blkg reference
463 	percpu_ref_get(&blkg->refcnt);  in blkg_get()
467  * blkg_tryget - try and get a blkg reference
475 	return blkg && percpu_ref_tryget(&blkg->refcnt);  in blkg_tryget()
479  * blkg_put - put a blkg reference
484 	percpu_ref_put(&blkg->refcnt);  in blkg_put()
488  * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
500 	css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css)	\
502 					      (p_blkg)->q, false)))
505  * blkg_for_each_descendant_post - post-order walk of a blkg's descendants
510  * Similar to blkg_for_each_descendant_pre() but performs post-order
515 	css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css)	\
517 					      (p_blkg)->q, false)))
523 	if (bio->bi_opf & REQ_CGROUP_PUNT)  in blkcg_punt_bio_submit()
531 	bio_issue_init(&bio->bi_issue, bio_sectors(bio));  in blkcg_bio_issue_init()
536 	if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))  in blkcg_use_delay()
538 	if (atomic_add_return(1, &blkg->use_delay) == 1)  in blkcg_use_delay()
539 		atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_use_delay()
544 	int old = atomic_read(&blkg->use_delay);  in blkcg_unuse_delay()
559 		int cur = atomic_cmpxchg(&blkg->use_delay, old, old - 1);  in blkcg_unuse_delay()
568 		atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_unuse_delay()
573  * blkcg_set_delay - Enable allocator delay mechanism with the specified delay amount
583 	int old = atomic_read(&blkg->use_delay);  in blkcg_set_delay()
586 	if (!old && atomic_cmpxchg(&blkg->use_delay, old, -1) == old)  in blkcg_set_delay()
587 		atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_set_delay()
589 	atomic64_set(&blkg->delay_nsec, delay);  in blkcg_set_delay()
593  * blkcg_clear_delay - Disable allocator delay mechanism
596  * Disable use_delay mechanism. See blkcg_set_delay().
600 	int old = atomic_read(&blkg->use_delay);  in blkcg_clear_delay()
603 	if (old && atomic_cmpxchg(&blkg->use_delay, old, 0) == old)  in blkcg_clear_delay()
604 		atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_clear_delay()
628 #define blkcg_root_css	((struct cgroup_subsys_state *)ERR_PTR(-EINVAL))
654 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }  in pd_to_blkg()  argument
664 	for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
684 		return -EINVAL;  in blkcg_set_fc_appid()
688 		return -ENOENT;  in blkcg_set_fc_appid()
691 		ret = -ENOENT;  in blkcg_set_fc_appid()
702 	strlcpy(blkcg->fc_app_id, app_id, app_id_len);  in blkcg_set_fc_appid()
710  * blkcg_get_fc_appid - get the fc app identifier associated with a bio
717 	if (bio && bio->bi_blkg &&  in blkcg_get_fc_appid()
718 		(bio->bi_blkg->blkcg->fc_app_id[0] != '\0'))  in blkcg_get_fc_appid()
719 		return bio->bi_blkg->blkcg->fc_app_id;  in blkcg_get_fc_appid()
723 static inline int blkcg_set_fc_appid(char *buf, u64 id, size_t len) { return -EINVAL; }  in blkcg_set_fc_appid()