8 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
10 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)  in cgroup_rstat_cpu()  argument
12 	return per_cpu_ptr(cgrp->rstat_cpu, cpu);  in cgroup_rstat_cpu()
24 void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)  in cgroup_rstat_updated()  argument
31 	if (!cgroup_parent(cgrp))  in cgroup_rstat_updated()
45 	if (cgroup_rstat_cpu(cgrp, cpu)->updated_next)  in cgroup_rstat_updated()
51 	for (parent = cgroup_parent(cgrp); parent;  in cgroup_rstat_updated()
52 	     cgrp = parent, parent = cgroup_parent(cgrp)) {  in cgroup_rstat_updated()
53 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_updated()
64 		prstatc->updated_children = cgrp;  in cgroup_rstat_updated()
149 static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)  in cgroup_rstat_flush_locked()  argument
162 		while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) {  in cgroup_rstat_flush_locked()
199 void cgroup_rstat_flush(struct cgroup *cgrp)  in cgroup_rstat_flush()  argument
204 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush()
214 void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp)  in cgroup_rstat_flush_irqsafe()  argument
219 	cgroup_rstat_flush_locked(cgrp, false);  in cgroup_rstat_flush_irqsafe()
232 void cgroup_rstat_flush_hold(struct cgroup *cgrp)  in cgroup_rstat_flush_hold()  argument
237 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush_hold()
249 int cgroup_rstat_init(struct cgroup *cgrp)  in cgroup_rstat_init()  argument
254 	if (!cgrp->rstat_cpu) {  in cgroup_rstat_init()
255 		cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);  in cgroup_rstat_init()
256 		if (!cgrp->rstat_cpu)  in cgroup_rstat_init()
262 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_init()
264 		rstatc->updated_children = cgrp;  in cgroup_rstat_init()
271 void cgroup_rstat_exit(struct cgroup *cgrp)  in cgroup_rstat_exit()  argument
275 	cgroup_rstat_flush(cgrp);  in cgroup_rstat_exit()
279 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_exit()
281 		if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||  in cgroup_rstat_exit()
286 	free_percpu(cgrp->rstat_cpu);  in cgroup_rstat_exit()
287 	cgrp->rstat_cpu = NULL;  in cgroup_rstat_exit()
297 	BUG_ON(cgroup_rstat_init(&cgrp_dfl_root.cgrp));  in cgroup_rstat_boot()
312 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
314 	struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_base_stat_flush()
315 	struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_base_stat_flush()
335 	cgroup_base_stat_accumulate(&delta, &cgrp->pending_bstat);  in cgroup_base_stat_flush()
336 	memset(&cgrp->pending_bstat, 0, sizeof(cgrp->pending_bstat));  in cgroup_base_stat_flush()
339 	cgroup_base_stat_accumulate(&cgrp->bstat, &delta);  in cgroup_base_stat_flush()
345 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp)  in cgroup_base_stat_cputime_account_begin()  argument
349 	rstatc = get_cpu_ptr(cgrp->rstat_cpu);  in cgroup_base_stat_cputime_account_begin()
354 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,  in cgroup_base_stat_cputime_account_end()  argument
358 	cgroup_rstat_updated(cgrp, smp_processor_id());  in cgroup_base_stat_cputime_account_end()
362 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)  in __cgroup_account_cputime()  argument
366 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp);  in __cgroup_account_cputime()
368 	cgroup_base_stat_cputime_account_end(cgrp, rstatc);  in __cgroup_account_cputime()
371 void __cgroup_account_cputime_field(struct cgroup *cgrp,  in __cgroup_account_cputime_field()  argument
376 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp);  in __cgroup_account_cputime_field()
392 	cgroup_base_stat_cputime_account_end(cgrp, rstatc);  in __cgroup_account_cputime_field()
397 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_base_stat_cputime_show()  local
400 	if (!cgroup_parent(cgrp))  in cgroup_base_stat_cputime_show()
403 	cgroup_rstat_flush_hold(cgrp);  in cgroup_base_stat_cputime_show()
404 	usage = cgrp->bstat.cputime.sum_exec_runtime;  in cgroup_base_stat_cputime_show()
405 	cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime, &utime, &stime);  in cgroup_base_stat_cputime_show()