734 	struct sched_avg *sa = &se->avg;  in init_entity_runnable_average()
784 	struct sched_avg *sa = &se->avg;  in post_init_entity_util_avg()
786 	long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2;  in post_init_entity_util_avg()
789 		if (cfs_rq->avg.util_avg != 0) {  in post_init_entity_util_avg()
790 			sa->util_avg  = cfs_rq->avg.util_avg * se->load.weight;  in post_init_entity_util_avg()
791 			sa->util_avg /= (cfs_rq->avg.load_avg + 1);  in post_init_entity_util_avg()
811 		se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq);  in post_init_entity_util_avg()
2025 		delta = p->se.avg.load_sum;  in numa_get_avg_runtime()
2838 	cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg;  in enqueue_runnable_load_avg()
2839 	cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum;  in enqueue_runnable_load_avg()
2847 	sub_positive(&cfs_rq->avg.runnable_load_avg, se->avg.runnable_load_avg);  in dequeue_runnable_load_avg()
2848 	sub_positive(&cfs_rq->avg.runnable_load_sum,  in dequeue_runnable_load_avg()
2849 		     se_runnable(se) * se->avg.runnable_load_sum);  in dequeue_runnable_load_avg()
2855 	cfs_rq->avg.load_avg += se->avg.load_avg;  in enqueue_load_avg()
2856 	cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;  in enqueue_load_avg()
2862 	sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);  in dequeue_load_avg()
2863 	sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);  in dequeue_load_avg()
2893 		u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib;  in reweight_entity()
2895 		se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);  in reweight_entity()
2896 		se->avg.runnable_load_avg =  in reweight_entity()
2897 			div_u64(se_runnable(se) * se->avg.runnable_load_sum, divider);  in reweight_entity()
3001 	load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);  in calc_group_shares()
3059 	load_avg = max(cfs_rq->avg.load_avg,  in calc_group_runnable()
3062 	runnable = max(cfs_rq->avg.runnable_load_avg,  in calc_group_runnable()
3151 	long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;  in update_tg_load_avg()
3161 		cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;  in update_tg_load_avg()
3186 	if (!(se->avg.last_update_time && prev))  in set_task_rq_fair()
3200 			p_last_update_time = prev->avg.last_update_time;  in set_task_rq_fair()
3201 			n_last_update_time = next->avg.last_update_time;  in set_task_rq_fair()
3207 	p_last_update_time = prev->avg.last_update_time;  in set_task_rq_fair()
3208 	n_last_update_time = next->avg.last_update_time;  in set_task_rq_fair()
3211 	se->avg.last_update_time = n_last_update_time;  in set_task_rq_fair()
3286 	long delta = gcfs_rq->avg.util_avg - se->avg.util_avg;  in update_tg_cfs_util()
3301 	se->avg.util_avg = gcfs_rq->avg.util_avg;  in update_tg_cfs_util()
3302 	se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX;  in update_tg_cfs_util()
3305 	add_positive(&cfs_rq->avg.util_avg, delta);  in update_tg_cfs_util()
3306 	cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX;  in update_tg_cfs_util()
3327 		runnable_sum += se->avg.load_sum;  in update_tg_cfs_runnable()
3335 			load_sum = div_s64(gcfs_rq->avg.load_sum,  in update_tg_cfs_runnable()
3340 		runnable_sum = min(se->avg.load_sum, load_sum);  in update_tg_cfs_runnable()
3349 	running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;  in update_tg_cfs_runnable()
3355 	delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum;  in update_tg_cfs_runnable()
3356 	delta_avg = load_avg - se->avg.load_avg;  in update_tg_cfs_runnable()
3358 	se->avg.load_sum = runnable_sum;  in update_tg_cfs_runnable()
3359 	se->avg.load_avg = load_avg;  in update_tg_cfs_runnable()
3360 	add_positive(&cfs_rq->avg.load_avg, delta_avg);  in update_tg_cfs_runnable()
3361 	add_positive(&cfs_rq->avg.load_sum, delta_sum);  in update_tg_cfs_runnable()
3365 	delta_sum = runnable_load_sum - se_weight(se) * se->avg.runnable_load_sum;  in update_tg_cfs_runnable()
3366 	delta_avg = runnable_load_avg - se->avg.runnable_load_avg;  in update_tg_cfs_runnable()
3368 	se->avg.runnable_load_sum = runnable_sum;  in update_tg_cfs_runnable()
3369 	se->avg.runnable_load_avg = runnable_load_avg;  in update_tg_cfs_runnable()
3372 		add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg);  in update_tg_cfs_runnable()
3373 		add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum);  in update_tg_cfs_runnable()
3422 	if (se->avg.load_avg || se->avg.util_avg)  in skip_blocked_update()
3473 	struct sched_avg *sa = &cfs_rq->avg;  in update_cfs_rq_load_avg()
3524 	u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib;  in attach_entity_load_avg()
3533 	se->avg.last_update_time = cfs_rq->avg.last_update_time;  in attach_entity_load_avg()
3534 	se->avg.period_contrib = cfs_rq->avg.period_contrib;  in attach_entity_load_avg()
3542 	se->avg.util_sum = se->avg.util_avg * divider;  in attach_entity_load_avg()
3544 	se->avg.load_sum = divider;  in attach_entity_load_avg()
3546 		se->avg.load_sum =  in attach_entity_load_avg()
3547 			div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se));  in attach_entity_load_avg()
3550 	se->avg.runnable_load_sum = se->avg.load_sum;  in attach_entity_load_avg()
3553 	cfs_rq->avg.util_avg += se->avg.util_avg;  in attach_entity_load_avg()
3554 	cfs_rq->avg.util_sum += se->avg.util_sum;  in attach_entity_load_avg()
3556 	add_tg_cfs_propagate(cfs_rq, se->avg.load_sum);  in attach_entity_load_avg()
3574 	sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);  in detach_entity_load_avg()
3575 	sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);  in detach_entity_load_avg()
3577 	add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);  in detach_entity_load_avg()
3601 	if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))  in update_load_avg()
3607 	if (!se->avg.last_update_time && (flags & DO_ATTACH)) {  in update_load_avg()
3632 		last_update_time = cfs_rq->avg.last_update_time;  in cfs_rq_last_update_time()
3640 	return cfs_rq->avg.last_update_time;  in cfs_rq_last_update_time()
3676 	cfs_rq->removed.util_avg	+= se->avg.util_avg;  in remove_entity_load_avg()
3677 	cfs_rq->removed.load_avg	+= se->avg.load_avg;  in remove_entity_load_avg()
3678 	cfs_rq->removed.runnable_sum	+= se->avg.load_sum; /* == runnable_sum */  in remove_entity_load_avg()
3684 	return cfs_rq->avg.runnable_load_avg;  in cfs_rq_runnable_load_avg()
3689 	return cfs_rq->avg.load_avg;  in cfs_rq_load_avg()
3694 	return READ_ONCE(p->se.avg.util_avg);  in task_util()
3699 	struct util_est ue = READ_ONCE(p->se.avg.util_est);  in _task_util_est()
3718 	enqueued  = cfs_rq->avg.util_est.enqueued;  in util_est_enqueue()
3720 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);  in util_est_enqueue()
3747 	ue.enqueued  = cfs_rq->avg.util_est.enqueued;  in util_est_dequeue()
3749 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, ue.enqueued);  in util_est_dequeue()
3762 	ue = p->se.avg.util_est;  in util_est_dequeue()
3803 	WRITE_ONCE(p->se.avg.util_est, ue);  in util_est_dequeue()
6081 	util = READ_ONCE(cfs_rq->avg.util_avg);  in cpu_util()
6084 		util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued));  in cpu_util()
6108 	if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))  in cpu_util_without()
6112 	util = READ_ONCE(cfs_rq->avg.util_avg);  in cpu_util_without()
6145 			READ_ONCE(cfs_rq->avg.util_est.enqueued);  in cpu_util_without()
6212 	unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg);  in cpu_util_next()
6226 		util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);  in cpu_util_next()
6561 	p->se.avg.last_update_time = 0;  in migrate_task_rq_fair()
7482 	if (cfs_rq->avg.load_avg)  in cfs_rq_has_blocked()
7485 	if (cfs_rq->avg.util_avg)  in cfs_rq_has_blocked()
7527 	if (cfs_rq->avg.load_sum)  in cfs_rq_is_decayed()
7530 	if (cfs_rq->avg.util_sum)  in cfs_rq_is_decayed()
7533 	if (cfs_rq->avg.runnable_load_sum)  in cfs_rq_is_decayed()
7624 		load = div64_ul(load * se->avg.load_avg,  in update_cfs_rq_h_load()
7637 	return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,  in task_h_load()
7668 	return p->se.avg.load_avg;  in task_h_load()
10205 	p->se.avg.last_update_time = 0;  in task_move_group_fair()
10529 	return cfs_rq ? &cfs_rq->avg : NULL;  in sched_trace_cfs_rq_avg()