9 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
11 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)  in cgroup_rstat_cpu()  argument
13 	return per_cpu_ptr(cgrp->rstat_cpu, cpu);  in cgroup_rstat_cpu()
25 void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)  in cgroup_rstat_updated()  argument
32 	if (!cgroup_parent(cgrp))  in cgroup_rstat_updated()
43 	if (cgroup_rstat_cpu(cgrp, cpu)->updated_next)  in cgroup_rstat_updated()
49 	for (parent = cgroup_parent(cgrp); parent;  in cgroup_rstat_updated()
50 	     cgrp = parent, parent = cgroup_parent(cgrp)) {  in cgroup_rstat_updated()
51 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_updated()
62 		prstatc->updated_children = cgrp;  in cgroup_rstat_updated()
141 static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)  in cgroup_rstat_flush_locked()  argument
154 		while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) {  in cgroup_rstat_flush_locked()
191 void cgroup_rstat_flush(struct cgroup *cgrp)  in cgroup_rstat_flush()  argument
196 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush()
206 void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp)  in cgroup_rstat_flush_irqsafe()  argument
211 	cgroup_rstat_flush_locked(cgrp, false);  in cgroup_rstat_flush_irqsafe()
224 void cgroup_rstat_flush_hold(struct cgroup *cgrp)  in cgroup_rstat_flush_hold()  argument
229 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush_hold()
241 int cgroup_rstat_init(struct cgroup *cgrp)  in cgroup_rstat_init()  argument
246 	if (!cgrp->rstat_cpu) {  in cgroup_rstat_init()
247 		cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);  in cgroup_rstat_init()
248 		if (!cgrp->rstat_cpu)  in cgroup_rstat_init()
254 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_init()
256 		rstatc->updated_children = cgrp;  in cgroup_rstat_init()
263 void cgroup_rstat_exit(struct cgroup *cgrp)  in cgroup_rstat_exit()  argument
267 	cgroup_rstat_flush(cgrp);  in cgroup_rstat_exit()
271 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_exit()
273 		if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||  in cgroup_rstat_exit()
278 	free_percpu(cgrp->rstat_cpu);  in cgroup_rstat_exit()
279 	cgrp->rstat_cpu = NULL;  in cgroup_rstat_exit()
289 	BUG_ON(cgroup_rstat_init(&cgrp_dfl_root.cgrp));  in cgroup_rstat_boot()
312 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
314 	struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_base_stat_flush()
315 	struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_base_stat_flush()
328 	cgroup_base_stat_add(&cgrp->bstat, &delta);  in cgroup_base_stat_flush()
333 		delta = cgrp->bstat;  in cgroup_base_stat_flush()
334 		cgroup_base_stat_sub(&delta, &cgrp->last_bstat);  in cgroup_base_stat_flush()
336 		cgroup_base_stat_add(&cgrp->last_bstat, &delta);  in cgroup_base_stat_flush()
341 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp)  in cgroup_base_stat_cputime_account_begin()  argument
345 	rstatc = get_cpu_ptr(cgrp->rstat_cpu);  in cgroup_base_stat_cputime_account_begin()
350 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,  in cgroup_base_stat_cputime_account_end()  argument
354 	cgroup_rstat_updated(cgrp, smp_processor_id());  in cgroup_base_stat_cputime_account_end()
358 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)  in __cgroup_account_cputime()  argument
362 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp);  in __cgroup_account_cputime()
364 	cgroup_base_stat_cputime_account_end(cgrp, rstatc);  in __cgroup_account_cputime()
367 void __cgroup_account_cputime_field(struct cgroup *cgrp,  in __cgroup_account_cputime_field()  argument
372 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp);  in __cgroup_account_cputime_field()
388 	cgroup_base_stat_cputime_account_end(cgrp, rstatc);  in __cgroup_account_cputime_field()
431 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_base_stat_cputime_show()  local
435 	if (cgroup_parent(cgrp)) {  in cgroup_base_stat_cputime_show()
436 		cgroup_rstat_flush_hold(cgrp);  in cgroup_base_stat_cputime_show()
437 		usage = cgrp->bstat.cputime.sum_exec_runtime;  in cgroup_base_stat_cputime_show()
438 		cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,  in cgroup_base_stat_cputime_show()