726 	struct sched_avg *sa = &se->avg;  in init_entity_runnable_average()
774 	struct sched_avg *sa = &se->avg;  in post_init_entity_util_avg()
776 	long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2;  in post_init_entity_util_avg()
779 		if (cfs_rq->avg.util_avg != 0) {  in post_init_entity_util_avg()
780 			sa->util_avg  = cfs_rq->avg.util_avg * se->load.weight;  in post_init_entity_util_avg()
781 			sa->util_avg /= (cfs_rq->avg.load_avg + 1);  in post_init_entity_util_avg()
803 		se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq);  in post_init_entity_util_avg()
2254 		delta = p->se.avg.load_sum;  in numa_get_avg_runtime()
3065 	cfs_rq->avg.load_avg += se->avg.load_avg;  in enqueue_load_avg()
3066 	cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;  in enqueue_load_avg()
3072 	u32 divider = get_pelt_divider(&se->avg);  in dequeue_load_avg()
3073 	sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);  in dequeue_load_avg()
3074 	cfs_rq->avg.load_sum = cfs_rq->avg.load_avg * divider;  in dequeue_load_avg()
3098 		u32 divider = get_pelt_divider(&se->avg);  in reweight_entity()
3100 		se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);  in reweight_entity()
3203 	load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);  in calc_group_shares()
3322 	if (cfs_rq->avg.load_sum)  in cfs_rq_is_decayed()
3325 	if (cfs_rq->avg.util_sum)  in cfs_rq_is_decayed()
3328 	if (cfs_rq->avg.runnable_sum)  in cfs_rq_is_decayed()
3339 	SCHED_WARN_ON(cfs_rq->avg.load_avg ||  in cfs_rq_is_decayed()
3340 		      cfs_rq->avg.util_avg ||  in cfs_rq_is_decayed()
3341 		      cfs_rq->avg.runnable_avg);  in cfs_rq_is_decayed()
3362 	long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;  in update_tg_load_avg()
3372 		cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;  in update_tg_load_avg()
3397 	if (!(se->avg.last_update_time && prev))  in set_task_rq_fair()
3411 			p_last_update_time = prev->avg.last_update_time;  in set_task_rq_fair()
3412 			n_last_update_time = next->avg.last_update_time;  in set_task_rq_fair()
3418 	p_last_update_time = prev->avg.last_update_time;  in set_task_rq_fair()
3419 	n_last_update_time = next->avg.last_update_time;  in set_task_rq_fair()
3422 	se->avg.last_update_time = n_last_update_time;  in set_task_rq_fair()
3497 	long delta = gcfs_rq->avg.util_avg - se->avg.util_avg;  in update_tg_cfs_util()
3508 	divider = get_pelt_divider(&cfs_rq->avg);  in update_tg_cfs_util()
3511 	se->avg.util_avg = gcfs_rq->avg.util_avg;  in update_tg_cfs_util()
3512 	se->avg.util_sum = se->avg.util_avg * divider;  in update_tg_cfs_util()
3515 	add_positive(&cfs_rq->avg.util_avg, delta);  in update_tg_cfs_util()
3516 	cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * divider;  in update_tg_cfs_util()
3522 	long delta = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;  in update_tg_cfs_runnable()
3533 	divider = get_pelt_divider(&cfs_rq->avg);  in update_tg_cfs_runnable()
3536 	se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;  in update_tg_cfs_runnable()
3537 	se->avg.runnable_sum = se->avg.runnable_avg * divider;  in update_tg_cfs_runnable()
3540 	add_positive(&cfs_rq->avg.runnable_avg, delta);  in update_tg_cfs_runnable()
3541 	cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * divider;  in update_tg_cfs_runnable()
3561 	divider = get_pelt_divider(&cfs_rq->avg);  in update_tg_cfs_load()
3568 		runnable_sum += se->avg.load_sum;  in update_tg_cfs_load()
3576 			load_sum = div_s64(gcfs_rq->avg.load_sum,  in update_tg_cfs_load()
3581 		runnable_sum = min(se->avg.load_sum, load_sum);  in update_tg_cfs_load()
3590 	running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;  in update_tg_cfs_load()
3596 	se->avg.load_sum = runnable_sum;  in update_tg_cfs_load()
3598 	delta = load_avg - se->avg.load_avg;  in update_tg_cfs_load()
3602 	se->avg.load_avg = load_avg;  in update_tg_cfs_load()
3604 	add_positive(&cfs_rq->avg.load_avg, delta);  in update_tg_cfs_load()
3605 	cfs_rq->avg.load_sum = cfs_rq->avg.load_avg * divider;  in update_tg_cfs_load()
3654 	if (se->avg.load_avg || se->avg.util_avg)  in skip_blocked_update()
3705 	struct sched_avg *sa = &cfs_rq->avg;  in update_cfs_rq_load_avg()
3710 		u32 divider = get_pelt_divider(&cfs_rq->avg);  in update_cfs_rq_load_avg()
3765 	u32 divider = get_pelt_divider(&cfs_rq->avg);  in attach_entity_load_avg()
3774 	se->avg.last_update_time = cfs_rq->avg.last_update_time;  in attach_entity_load_avg()
3775 	se->avg.period_contrib = cfs_rq->avg.period_contrib;  in attach_entity_load_avg()
3783 	se->avg.util_sum = se->avg.util_avg * divider;  in attach_entity_load_avg()
3785 	se->avg.runnable_sum = se->avg.runnable_avg * divider;  in attach_entity_load_avg()
3787 	se->avg.load_sum = divider;  in attach_entity_load_avg()
3789 		se->avg.load_sum =  in attach_entity_load_avg()
3790 			div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se));  in attach_entity_load_avg()
3794 	cfs_rq->avg.util_avg += se->avg.util_avg;  in attach_entity_load_avg()
3795 	cfs_rq->avg.util_sum += se->avg.util_sum;  in attach_entity_load_avg()
3796 	cfs_rq->avg.runnable_avg += se->avg.runnable_avg;  in attach_entity_load_avg()
3797 	cfs_rq->avg.runnable_sum += se->avg.runnable_sum;  in attach_entity_load_avg()
3799 	add_tg_cfs_propagate(cfs_rq, se->avg.load_sum);  in attach_entity_load_avg()
3820 	u32 divider = get_pelt_divider(&cfs_rq->avg);  in detach_entity_load_avg()
3823 	sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);  in detach_entity_load_avg()
3824 	cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * divider;  in detach_entity_load_avg()
3825 	sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg);  in detach_entity_load_avg()
3826 	cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * divider;  in detach_entity_load_avg()
3828 	add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);  in detach_entity_load_avg()
3852 	if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))  in update_load_avg()
3858 	if (!se->avg.last_update_time && (flags & DO_ATTACH)) {  in update_load_avg()
3887 		last_update_time = cfs_rq->avg.last_update_time;  in cfs_rq_last_update_time()
3895 	return cfs_rq->avg.last_update_time;  in cfs_rq_last_update_time()
3931 	cfs_rq->removed.util_avg	+= se->avg.util_avg;  in remove_entity_load_avg()
3932 	cfs_rq->removed.load_avg	+= se->avg.load_avg;  in remove_entity_load_avg()
3933 	cfs_rq->removed.runnable_avg	+= se->avg.runnable_avg;  in remove_entity_load_avg()
3939 	return cfs_rq->avg.runnable_avg;  in cfs_rq_runnable_avg()
3944 	return cfs_rq->avg.load_avg;  in cfs_rq_load_avg()
3951 	return READ_ONCE(p->se.avg.util_avg);  in task_util()
3956 	struct util_est ue = READ_ONCE(p->se.avg.util_est);  in _task_util_est()
3989 	enqueued  = cfs_rq->avg.util_est.enqueued;  in util_est_enqueue()
3991 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);  in util_est_enqueue()
4005 	enqueued  = cfs_rq->avg.util_est.enqueued;  in util_est_dequeue()
4007 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);  in util_est_dequeue()
4048 	ue = p->se.avg.util_est;  in util_est_update()
4108 	WRITE_ONCE(p->se.avg.util_est, ue);  in util_est_update()
5820 	if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))  in cpu_load_without()
5824 	load = READ_ONCE(cfs_rq->avg.load_avg);  in cpu_load_without()
5843 	if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))  in cpu_runnable_without()
5847 	runnable = READ_ONCE(cfs_rq->avg.runnable_avg);  in cpu_runnable_without()
5850 	lsub_positive(&runnable, p->se.avg.runnable_avg);  in cpu_runnable_without()
6539 	util = READ_ONCE(cfs_rq->avg.util_avg);  in cpu_util()
6542 		util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued));  in cpu_util()
6566 	if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))  in cpu_util_without()
6570 	util = READ_ONCE(cfs_rq->avg.util_avg);  in cpu_util_without()
6603 			READ_ONCE(cfs_rq->avg.util_est.enqueued);  in cpu_util_without()
6643 	unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg);  in cpu_util_next()
6657 		util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);  in cpu_util_next()
7028 	p->se.avg.last_update_time = 0;  in migrate_task_rq_fair()
8108 	if (cfs_rq->avg.load_avg)  in cfs_rq_has_blocked()
8111 	if (cfs_rq->avg.util_avg)  in cfs_rq_has_blocked()
8251 		load = div64_ul(load * se->avg.load_avg,  in update_cfs_rq_h_load()
8264 	return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,  in task_h_load()
8282 	return p->se.avg.load_avg;  in task_h_load()
8844 	if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))  in task_running_on_cpu()
11339 	p->se.avg.last_update_time = 0;  in task_move_group_fair()
11743 	return cfs_rq ? &cfs_rq->avg : NULL;  in sched_trace_cfs_rq_avg()