156 static void sugov_get_util(struct sugov_cpu *sg_cpu)  in sugov_get_util()  argument
158 	unsigned long util = cpu_util_cfs_boost(sg_cpu->cpu);  in sugov_get_util()
159 	struct rq *rq = cpu_rq(sg_cpu->cpu);  in sugov_get_util()
161 	sg_cpu->bw_dl = cpu_bw_dl(rq);  in sugov_get_util()
162 	sg_cpu->util = effective_cpu_util(sg_cpu->cpu, util,  in sugov_get_util()
177 static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time,  in sugov_iowait_reset()  argument
180 	s64 delta_ns = time - sg_cpu->last_update;  in sugov_iowait_reset()
186 	sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0;  in sugov_iowait_reset()
187 	sg_cpu->iowait_boost_pending = set_iowait_boost;  in sugov_iowait_reset()
206 static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,  in sugov_iowait_boost()  argument
212 	if (sg_cpu->iowait_boost &&  in sugov_iowait_boost()
213 	    sugov_iowait_reset(sg_cpu, time, set_iowait_boost))  in sugov_iowait_boost()
221 	if (sg_cpu->iowait_boost_pending)  in sugov_iowait_boost()
223 	sg_cpu->iowait_boost_pending = true;  in sugov_iowait_boost()
226 	if (sg_cpu->iowait_boost) {  in sugov_iowait_boost()
227 		sg_cpu->iowait_boost =  in sugov_iowait_boost()
228 			min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE);  in sugov_iowait_boost()
233 	sg_cpu->iowait_boost = IOWAIT_BOOST_MIN;  in sugov_iowait_boost()
254 static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time,  in sugov_iowait_apply()  argument
260 	if (!sg_cpu->iowait_boost)  in sugov_iowait_apply()
264 	if (sugov_iowait_reset(sg_cpu, time, false))  in sugov_iowait_apply()
267 	if (!sg_cpu->iowait_boost_pending) {  in sugov_iowait_apply()
271 		sg_cpu->iowait_boost >>= 1;  in sugov_iowait_apply()
272 		if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) {  in sugov_iowait_apply()
273 			sg_cpu->iowait_boost = 0;  in sugov_iowait_apply()
278 	sg_cpu->iowait_boost_pending = false;  in sugov_iowait_apply()
284 	boost = (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT;  in sugov_iowait_apply()
285 	boost = uclamp_rq_util_with(cpu_rq(sg_cpu->cpu), boost, NULL);  in sugov_iowait_apply()
286 	if (sg_cpu->util < boost)  in sugov_iowait_apply()
287 		sg_cpu->util = boost;  in sugov_iowait_apply()
291 static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)  in sugov_cpu_is_busy()  argument
293 	unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);  in sugov_cpu_is_busy()
294 	bool ret = idle_calls == sg_cpu->saved_idle_calls;  in sugov_cpu_is_busy()
296 	sg_cpu->saved_idle_calls = idle_calls;  in sugov_cpu_is_busy()
300 static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }  in sugov_cpu_is_busy()  argument
307 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu)  in ignore_dl_rate_limit()  argument
309 	if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_dl)  in ignore_dl_rate_limit()
310 		sg_cpu->sg_policy->limits_changed = true;  in ignore_dl_rate_limit()
313 static inline bool sugov_update_single_common(struct sugov_cpu *sg_cpu,  in sugov_update_single_common()  argument
317 	sugov_iowait_boost(sg_cpu, time, flags);  in sugov_update_single_common()
318 	sg_cpu->last_update = time;  in sugov_update_single_common()
320 	ignore_dl_rate_limit(sg_cpu);  in sugov_update_single_common()
322 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))  in sugov_update_single_common()
325 	sugov_get_util(sg_cpu);  in sugov_update_single_common()
326 	sugov_iowait_apply(sg_cpu, time, max_cap);  in sugov_update_single_common()
334 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);  in sugov_update_single_freq()  local
335 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single_freq()
340 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_freq()
342 	if (!sugov_update_single_common(sg_cpu, time, max_cap, flags))  in sugov_update_single_freq()
345 	next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);  in sugov_update_single_freq()
352 	if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) &&  in sugov_update_single_freq()
353 	    sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq &&  in sugov_update_single_freq()
381 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);  in sugov_update_single_perf()  local
382 	unsigned long prev_util = sg_cpu->util;  in sugov_update_single_perf()
395 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_perf()
397 	if (!sugov_update_single_common(sg_cpu, time, max_cap, flags))  in sugov_update_single_perf()
406 	if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) &&  in sugov_update_single_perf()
407 	    sugov_cpu_is_busy(sg_cpu) && sg_cpu->util < prev_util)  in sugov_update_single_perf()
408 		sg_cpu->util = prev_util;  in sugov_update_single_perf()
410 	cpufreq_driver_adjust_perf(sg_cpu->cpu, map_util_perf(sg_cpu->bw_dl),  in sugov_update_single_perf()
411 				   map_util_perf(sg_cpu->util), max_cap);  in sugov_update_single_perf()
413 	sg_cpu->sg_policy->last_freq_update_time = time;  in sugov_update_single_perf()
416 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)  in sugov_next_freq_shared()  argument
418 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()
423 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_next_freq_shared()
440 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);  in sugov_update_shared()  local
441 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()
446 	sugov_iowait_boost(sg_cpu, time, flags);  in sugov_update_shared()
447 	sg_cpu->last_update = time;  in sugov_update_shared()
449 	ignore_dl_rate_limit(sg_cpu);  in sugov_update_shared()
452 		next_f = sugov_next_freq_shared(sg_cpu, time);  in sugov_update_shared()
772 		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);  in sugov_start()  local
774 		memset(sg_cpu, 0, sizeof(*sg_cpu));  in sugov_start()
775 		sg_cpu->cpu			= cpu;  in sugov_start()
776 		sg_cpu->sg_policy		= sg_policy;  in sugov_start()
787 		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);  in sugov_start()  local
789 		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);  in sugov_start()