13 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
15 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)  in cgroup_rstat_cpu()  argument
17 	return per_cpu_ptr(cgrp->rstat_cpu, cpu);  in cgroup_rstat_cpu()
29 void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)  in cgroup_rstat_updated()  argument
42 	if (data_race(cgroup_rstat_cpu(cgrp, cpu)->updated_next))  in cgroup_rstat_updated()
49 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_updated()
50 		struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_rstat_updated()
62 			rstatc->updated_next = cgrp;  in cgroup_rstat_updated()
68 		prstatc->updated_children = cgrp;  in cgroup_rstat_updated()
70 		cgrp = parent;  in cgroup_rstat_updated()
166 __weak noinline void bpf_rstat_flush(struct cgroup *cgrp,  in bpf_rstat_flush()  argument
174 static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)  in cgroup_rstat_flush_locked()  argument
196 		while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) {  in cgroup_rstat_flush_locked()
234 void cgroup_rstat_flush(struct cgroup *cgrp)  in cgroup_rstat_flush()  argument
239 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush()
249 void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp)  in cgroup_rstat_flush_irqsafe()  argument
254 	cgroup_rstat_flush_locked(cgrp, false);  in cgroup_rstat_flush_irqsafe()
267 void cgroup_rstat_flush_hold(struct cgroup *cgrp)  in cgroup_rstat_flush_hold()  argument
272 	cgroup_rstat_flush_locked(cgrp, true);  in cgroup_rstat_flush_hold()
284 int cgroup_rstat_init(struct cgroup *cgrp)  in cgroup_rstat_init()  argument
289 	if (!cgrp->rstat_cpu) {  in cgroup_rstat_init()
290 		cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);  in cgroup_rstat_init()
291 		if (!cgrp->rstat_cpu)  in cgroup_rstat_init()
297 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_init()
299 		rstatc->updated_children = cgrp;  in cgroup_rstat_init()
306 void cgroup_rstat_exit(struct cgroup *cgrp)  in cgroup_rstat_exit()  argument
310 	cgroup_rstat_flush(cgrp);  in cgroup_rstat_exit()
314 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_exit()
316 		if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||  in cgroup_rstat_exit()
321 	free_percpu(cgrp->rstat_cpu);  in cgroup_rstat_exit()
322 	cgrp->rstat_cpu = NULL;  in cgroup_rstat_exit()
359 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
361 	struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_base_stat_flush()
362 	struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_base_stat_flush()
378 	cgroup_base_stat_add(&cgrp->bstat, &delta);  in cgroup_base_stat_flush()
383 		delta = cgrp->bstat;  in cgroup_base_stat_flush()
384 		cgroup_base_stat_sub(&delta, &cgrp->last_bstat);  in cgroup_base_stat_flush()
386 		cgroup_base_stat_add(&cgrp->last_bstat, &delta);  in cgroup_base_stat_flush()
391 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags)  in cgroup_base_stat_cputime_account_begin()  argument
395 	rstatc = get_cpu_ptr(cgrp->rstat_cpu);  in cgroup_base_stat_cputime_account_begin()
400 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,  in cgroup_base_stat_cputime_account_end()  argument
405 	cgroup_rstat_updated(cgrp, smp_processor_id());  in cgroup_base_stat_cputime_account_end()
409 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)  in __cgroup_account_cputime()  argument
414 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime()
416 	cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);  in __cgroup_account_cputime()
419 void __cgroup_account_cputime_field(struct cgroup *cgrp,  in __cgroup_account_cputime_field()  argument
425 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime_field()
446 	cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);  in __cgroup_account_cputime_field()
492 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_base_stat_cputime_show()  local
499 	if (cgroup_parent(cgrp)) {  in cgroup_base_stat_cputime_show()
500 		cgroup_rstat_flush_hold(cgrp);  in cgroup_base_stat_cputime_show()
501 		usage = cgrp->bstat.cputime.sum_exec_runtime;  in cgroup_base_stat_cputime_show()
502 		cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,  in cgroup_base_stat_cputime_show()
505 		forceidle_time = cgrp->bstat.forceidle_sum;  in cgroup_base_stat_cputime_show()