200 static unsigned long sugov_get_util(struct sugov_cpu *sg_cpu)  in sugov_get_util()  argument
202 	struct rq *rq = cpu_rq(sg_cpu->cpu);  in sugov_get_util()
205 	sg_cpu->max = max = arch_scale_cpu_capacity(NULL, sg_cpu->cpu);  in sugov_get_util()
206 	sg_cpu->bw_dl = cpu_bw_dl(rq);  in sugov_get_util()
263 	return min(max, util + sg_cpu->bw_dl);  in sugov_get_util()
277 static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time,  in sugov_iowait_reset()  argument
280 	s64 delta_ns = time - sg_cpu->last_update;  in sugov_iowait_reset()
286 	sg_cpu->iowait_boost = set_iowait_boost  in sugov_iowait_reset()
287 		? sg_cpu->sg_policy->policy->min : 0;  in sugov_iowait_reset()
288 	sg_cpu->iowait_boost_pending = set_iowait_boost;  in sugov_iowait_reset()
306 static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,  in sugov_iowait_boost()  argument
312 	if (sg_cpu->iowait_boost &&  in sugov_iowait_boost()
313 	    sugov_iowait_reset(sg_cpu, time, set_iowait_boost))  in sugov_iowait_boost()
321 	if (sg_cpu->iowait_boost_pending)  in sugov_iowait_boost()
323 	sg_cpu->iowait_boost_pending = true;  in sugov_iowait_boost()
326 	if (sg_cpu->iowait_boost) {  in sugov_iowait_boost()
327 		sg_cpu->iowait_boost <<= 1;  in sugov_iowait_boost()
328 		if (sg_cpu->iowait_boost > sg_cpu->iowait_boost_max)  in sugov_iowait_boost()
329 			sg_cpu->iowait_boost = sg_cpu->iowait_boost_max;  in sugov_iowait_boost()
334 	sg_cpu->iowait_boost = sg_cpu->sg_policy->policy->min;  in sugov_iowait_boost()
356 static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time,  in sugov_iowait_apply()  argument
362 	if (!sg_cpu->iowait_boost)  in sugov_iowait_apply()
366 	if (sugov_iowait_reset(sg_cpu, time, false))  in sugov_iowait_apply()
373 	if (sg_cpu->iowait_boost_pending) {  in sugov_iowait_apply()
374 		sg_cpu->iowait_boost_pending = false;  in sugov_iowait_apply()
380 		sg_cpu->iowait_boost >>= 1;  in sugov_iowait_apply()
381 		if (sg_cpu->iowait_boost < sg_cpu->sg_policy->policy->min) {  in sugov_iowait_apply()
382 			sg_cpu->iowait_boost = 0;  in sugov_iowait_apply()
391 	boost_util = sg_cpu->iowait_boost;  in sugov_iowait_apply()
392 	boost_max = sg_cpu->iowait_boost_max;  in sugov_iowait_apply()
400 static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)  in sugov_cpu_is_busy()  argument
402 	unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);  in sugov_cpu_is_busy()
403 	bool ret = idle_calls == sg_cpu->saved_idle_calls;  in sugov_cpu_is_busy()
405 	sg_cpu->saved_idle_calls = idle_calls;  in sugov_cpu_is_busy()
409 static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }  in sugov_cpu_is_busy()  argument
416 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu, struct sugov_policy *sg_policy)  in ignore_dl_rate_limit()  argument
418 	if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_dl)  in ignore_dl_rate_limit()
425 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);  in sugov_update_single()  local
426 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single()
431 	sugov_iowait_boost(sg_cpu, time, flags);  in sugov_update_single()
432 	sg_cpu->last_update = time;  in sugov_update_single()
434 	ignore_dl_rate_limit(sg_cpu, sg_policy);  in sugov_update_single()
439 	busy = sugov_cpu_is_busy(sg_cpu);  in sugov_update_single()
441 	util = sugov_get_util(sg_cpu);  in sugov_update_single()
442 	max = sg_cpu->max;  in sugov_update_single()
443 	sugov_iowait_apply(sg_cpu, time, &util, &max);  in sugov_update_single()
470 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)  in sugov_next_freq_shared()  argument
472 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()
497 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);  in sugov_update_shared()  local
498 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()
503 	sugov_iowait_boost(sg_cpu, time, flags);  in sugov_update_shared()
504 	sg_cpu->last_update = time;  in sugov_update_shared()
506 	ignore_dl_rate_limit(sg_cpu, sg_policy);  in sugov_update_shared()
509 		next_f = sugov_next_freq_shared(sg_cpu, time);  in sugov_update_shared()
815 		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);  in sugov_start()  local
817 		memset(sg_cpu, 0, sizeof(*sg_cpu));  in sugov_start()
818 		sg_cpu->cpu			= cpu;  in sugov_start()
819 		sg_cpu->sg_policy		= sg_policy;  in sugov_start()
820 		sg_cpu->iowait_boost_max	= policy->cpuinfo.max_freq;  in sugov_start()
824 		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);  in sugov_start()  local
826 		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,  in sugov_start()