9 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
11 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)  in cgroup_rstat_cpu()  argument
13 	return per_cpu_ptr(cgrp->rstat_cpu, cpu);  in cgroup_rstat_cpu()
25 void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)  in cgroup_rstat_updated()  argument
32 	if (!cgroup_parent(cgrp))  in cgroup_rstat_updated()
46 	if (cgroup_rstat_cpu(cgrp, cpu)->updated_next)  in cgroup_rstat_updated()
52 	for (parent = cgroup_parent(cgrp); parent;  in cgroup_rstat_updated()
53 	     cgrp = parent, parent = cgroup_parent(cgrp)) {  in cgroup_rstat_updated()
54 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_updated()
65 		prstatc->updated_children = cgrp;  in cgroup_rstat_updated()
152 static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)  in cgroup_rstat_flush_locked()  argument
165 		while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) {  in cgroup_rstat_flush_locked()
202 void cgroup_rstat_flush(struct cgroup *cgrp)  in cgroup_rstat_flush()  argument
207 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush()
217 void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp)  in cgroup_rstat_flush_irqsafe()  argument
222 	cgroup_rstat_flush_locked(cgrp, false);  in cgroup_rstat_flush_irqsafe()
235 void cgroup_rstat_flush_hold(struct cgroup *cgrp)  in cgroup_rstat_flush_hold()  argument
240 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush_hold()
252 int cgroup_rstat_init(struct cgroup *cgrp)  in cgroup_rstat_init()  argument
257 	if (!cgrp->rstat_cpu) {  in cgroup_rstat_init()
258 		cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);  in cgroup_rstat_init()
259 		if (!cgrp->rstat_cpu)  in cgroup_rstat_init()
265 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_init()
267 		rstatc->updated_children = cgrp;  in cgroup_rstat_init()
274 void cgroup_rstat_exit(struct cgroup *cgrp)  in cgroup_rstat_exit()  argument
278 	cgroup_rstat_flush(cgrp);  in cgroup_rstat_exit()
282 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_exit()
284 		if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||  in cgroup_rstat_exit()
289 	free_percpu(cgrp->rstat_cpu);  in cgroup_rstat_exit()
290 	cgrp->rstat_cpu = NULL;  in cgroup_rstat_exit()
300 	BUG_ON(cgroup_rstat_init(&cgrp_dfl_root.cgrp));  in cgroup_rstat_boot()
315 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
317 	struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_base_stat_flush()
318 	struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_base_stat_flush()
338 	cgroup_base_stat_accumulate(&delta, &cgrp->pending_bstat);  in cgroup_base_stat_flush()
339 	memset(&cgrp->pending_bstat, 0, sizeof(cgrp->pending_bstat));  in cgroup_base_stat_flush()
342 	cgroup_base_stat_accumulate(&cgrp->bstat, &delta);  in cgroup_base_stat_flush()
348 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp)  in cgroup_base_stat_cputime_account_begin()  argument
352 	rstatc = get_cpu_ptr(cgrp->rstat_cpu);  in cgroup_base_stat_cputime_account_begin()
357 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,  in cgroup_base_stat_cputime_account_end()  argument
361 	cgroup_rstat_updated(cgrp, smp_processor_id());  in cgroup_base_stat_cputime_account_end()
365 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)  in __cgroup_account_cputime()  argument
369 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp);  in __cgroup_account_cputime()
371 	cgroup_base_stat_cputime_account_end(cgrp, rstatc);  in __cgroup_account_cputime()
374 void __cgroup_account_cputime_field(struct cgroup *cgrp,  in __cgroup_account_cputime_field()  argument
379 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp);  in __cgroup_account_cputime_field()
395 	cgroup_base_stat_cputime_account_end(cgrp, rstatc);  in __cgroup_account_cputime_field()
400 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_base_stat_cputime_show()  local
403 	if (!cgroup_parent(cgrp))  in cgroup_base_stat_cputime_show()
406 	cgroup_rstat_flush_hold(cgrp);  in cgroup_base_stat_cputime_show()
407 	usage = cgrp->bstat.cputime.sum_exec_runtime;  in cgroup_base_stat_cputime_show()
408 	cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime, &utime, &stime);  in cgroup_base_stat_cputime_show()