49 	struct sugov_policy	*sg_policy;  member
70 static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)  in sugov_should_update_freq()  argument
89 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))  in sugov_should_update_freq()
92 	if (unlikely(sg_policy->limits_changed)) {  in sugov_should_update_freq()
93 		sg_policy->limits_changed = false;  in sugov_should_update_freq()
94 		sg_policy->need_freq_update = true;  in sugov_should_update_freq()
98 	delta_ns = time - sg_policy->last_freq_update_time;  in sugov_should_update_freq()
100 	return delta_ns >= sg_policy->freq_update_delay_ns;  in sugov_should_update_freq()
103 static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time,  in sugov_update_next_freq()  argument
106 	if (sg_policy->need_freq_update)  in sugov_update_next_freq()
107 		sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_update_next_freq()
108 	else if (sg_policy->next_freq == next_freq)  in sugov_update_next_freq()
111 	sg_policy->next_freq = next_freq;  in sugov_update_next_freq()
112 	sg_policy->last_freq_update_time = time;  in sugov_update_next_freq()
117 static void sugov_deferred_update(struct sugov_policy *sg_policy)  in sugov_deferred_update()  argument
119 	if (!sg_policy->work_in_progress) {  in sugov_deferred_update()
120 		sg_policy->work_in_progress = true;  in sugov_deferred_update()
121 		irq_work_queue(&sg_policy->irq_work);  in sugov_deferred_update()
147 static unsigned int get_next_freq(struct sugov_policy *sg_policy,  in get_next_freq()  argument
150 	struct cpufreq_policy *policy = sg_policy->policy;  in get_next_freq()
157 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)  in get_next_freq()
158 		return sg_policy->next_freq;  in get_next_freq()
160 	sg_policy->cached_raw_freq = freq;  in get_next_freq()
316 		sg_cpu->sg_policy->limits_changed = true;  in ignore_dl_rate_limit()
327 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))  in sugov_update_single_common()
340 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single_freq()  local
341 	unsigned int cached_freq = sg_policy->cached_raw_freq;  in sugov_update_single_freq()
347 	next_f = get_next_freq(sg_policy, sg_cpu->util, sg_cpu->max);  in sugov_update_single_freq()
352 	if (sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq) {  in sugov_update_single_freq()
353 		next_f = sg_policy->next_freq;  in sugov_update_single_freq()
356 		sg_policy->cached_raw_freq = cached_freq;  in sugov_update_single_freq()
359 	if (!sugov_update_next_freq(sg_policy, time, next_f))  in sugov_update_single_freq()
367 	if (sg_policy->policy->fast_switch_enabled) {  in sugov_update_single_freq()
368 		cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_single_freq()
370 		raw_spin_lock(&sg_policy->update_lock);  in sugov_update_single_freq()
371 		sugov_deferred_update(sg_policy);  in sugov_update_single_freq()
372 		raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_single_freq()
405 	sg_cpu->sg_policy->last_freq_update_time = time;  in sugov_update_single_perf()
410 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()  local
411 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_next_freq_shared()
430 	return get_next_freq(sg_policy, util, max);  in sugov_next_freq_shared()
437 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()  local
440 	raw_spin_lock(&sg_policy->update_lock);  in sugov_update_shared()
447 	if (sugov_should_update_freq(sg_policy, time)) {  in sugov_update_shared()
450 		if (!sugov_update_next_freq(sg_policy, time, next_f))  in sugov_update_shared()
453 		if (sg_policy->policy->fast_switch_enabled)  in sugov_update_shared()
454 			cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_shared()
456 			sugov_deferred_update(sg_policy);  in sugov_update_shared()
459 	raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_shared()
464 	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);  in sugov_work()  local
478 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);  in sugov_work()
479 	freq = sg_policy->next_freq;  in sugov_work()
480 	sg_policy->work_in_progress = false;  in sugov_work()
481 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);  in sugov_work()
483 	mutex_lock(&sg_policy->work_lock);  in sugov_work()
484 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);  in sugov_work()
485 	mutex_unlock(&sg_policy->work_lock);  in sugov_work()
490 	struct sugov_policy *sg_policy;  in sugov_irq_work()  local
492 	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);  in sugov_irq_work()
494 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);  in sugov_irq_work()
518 	struct sugov_policy *sg_policy;  in rate_limit_us_store()  local
526 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)  in rate_limit_us_store()
527 		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;  in rate_limit_us_store()
559 	struct sugov_policy *sg_policy;  in sugov_policy_alloc()  local
561 	sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);  in sugov_policy_alloc()
562 	if (!sg_policy)  in sugov_policy_alloc()
565 	sg_policy->policy = policy;  in sugov_policy_alloc()
566 	raw_spin_lock_init(&sg_policy->update_lock);  in sugov_policy_alloc()
567 	return sg_policy;  in sugov_policy_alloc()
570 static void sugov_policy_free(struct sugov_policy *sg_policy)  in sugov_policy_free()  argument
572 	kfree(sg_policy);  in sugov_policy_free()
575 static int sugov_kthread_create(struct sugov_policy *sg_policy)  in sugov_kthread_create()  argument
592 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_kthread_create()
599 	kthread_init_work(&sg_policy->work, sugov_work);  in sugov_kthread_create()
600 	kthread_init_worker(&sg_policy->worker);  in sugov_kthread_create()
601 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,  in sugov_kthread_create()
616 	sg_policy->thread = thread;  in sugov_kthread_create()
618 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);  in sugov_kthread_create()
619 	mutex_init(&sg_policy->work_lock);  in sugov_kthread_create()
626 static void sugov_kthread_stop(struct sugov_policy *sg_policy)  in sugov_kthread_stop()  argument
629 	if (sg_policy->policy->fast_switch_enabled)  in sugov_kthread_stop()
632 	kthread_flush_worker(&sg_policy->worker);  in sugov_kthread_stop()
633 	kthread_stop(sg_policy->thread);  in sugov_kthread_stop()
634 	mutex_destroy(&sg_policy->work_lock);  in sugov_kthread_stop()
637 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)  in sugov_tunables_alloc()  argument
643 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_tunables_alloc()
658 	struct sugov_policy *sg_policy;  in sugov_init()  local
668 	sg_policy = sugov_policy_alloc(policy);  in sugov_init()
669 	if (!sg_policy) {  in sugov_init()
674 	ret = sugov_kthread_create(sg_policy);  in sugov_init()
685 		policy->governor_data = sg_policy;  in sugov_init()
686 		sg_policy->tunables = global_tunables;  in sugov_init()
688 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);  in sugov_init()
692 	tunables = sugov_tunables_alloc(sg_policy);  in sugov_init()
700 	policy->governor_data = sg_policy;  in sugov_init()
701 	sg_policy->tunables = tunables;  in sugov_init()
719 	sugov_kthread_stop(sg_policy);  in sugov_init()
723 	sugov_policy_free(sg_policy);  in sugov_init()
734 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_exit()  local
735 	struct sugov_tunables *tunables = sg_policy->tunables;  in sugov_exit()
740 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_exit()
747 	sugov_kthread_stop(sg_policy);  in sugov_exit()
748 	sugov_policy_free(sg_policy);  in sugov_exit()
754 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_start()  local
758 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;  in sugov_start()
759 	sg_policy->last_freq_update_time	= 0;  in sugov_start()
760 	sg_policy->next_freq			= 0;  in sugov_start()
761 	sg_policy->work_in_progress		= false;  in sugov_start()
762 	sg_policy->limits_changed		= false;  in sugov_start()
763 	sg_policy->cached_raw_freq		= 0;  in sugov_start()
765 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_start()
772 		sg_cpu->sg_policy		= sg_policy;  in sugov_start()
792 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_stop()  local
801 		irq_work_sync(&sg_policy->irq_work);  in sugov_stop()
802 		kthread_cancel_work_sync(&sg_policy->work);  in sugov_stop()
808 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_limits()  local
811 		mutex_lock(&sg_policy->work_lock);  in sugov_limits()
813 		mutex_unlock(&sg_policy->work_lock);  in sugov_limits()
816 	sg_policy->limits_changed = true;  in sugov_limits()