42 	struct sugov_policy	*sg_policy;  member
63 static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)  in sugov_should_update_freq()  argument
82 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))  in sugov_should_update_freq()
85 	if (unlikely(sg_policy->limits_changed)) {  in sugov_should_update_freq()
86 		sg_policy->limits_changed = false;  in sugov_should_update_freq()
87 		sg_policy->need_freq_update = true;  in sugov_should_update_freq()
91 	delta_ns = time - sg_policy->last_freq_update_time;  in sugov_should_update_freq()
93 	return delta_ns >= sg_policy->freq_update_delay_ns;  in sugov_should_update_freq()
96 static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time,  in sugov_update_next_freq()  argument
99 	if (sg_policy->need_freq_update)  in sugov_update_next_freq()
100 		sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_update_next_freq()
101 	else if (sg_policy->next_freq == next_freq)  in sugov_update_next_freq()
104 	sg_policy->next_freq = next_freq;  in sugov_update_next_freq()
105 	sg_policy->last_freq_update_time = time;  in sugov_update_next_freq()
110 static void sugov_deferred_update(struct sugov_policy *sg_policy)  in sugov_deferred_update()  argument
112 	if (!sg_policy->work_in_progress) {  in sugov_deferred_update()
113 		sg_policy->work_in_progress = true;  in sugov_deferred_update()
114 		irq_work_queue(&sg_policy->irq_work);  in sugov_deferred_update()
140 static unsigned int get_next_freq(struct sugov_policy *sg_policy,  in get_next_freq()  argument
143 	struct cpufreq_policy *policy = sg_policy->policy;  in get_next_freq()
150 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)  in get_next_freq()
151 		return sg_policy->next_freq;  in get_next_freq()
153 	sg_policy->cached_raw_freq = freq;  in get_next_freq()
309 		sg_cpu->sg_policy->limits_changed = true;  in ignore_dl_rate_limit()
320 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))  in sugov_update_single_common()
333 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single_freq()  local
334 	unsigned int cached_freq = sg_policy->cached_raw_freq;  in sugov_update_single_freq()
340 	next_f = get_next_freq(sg_policy, sg_cpu->util, sg_cpu->max);  in sugov_update_single_freq()
348 	    sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq) {  in sugov_update_single_freq()
349 		next_f = sg_policy->next_freq;  in sugov_update_single_freq()
352 		sg_policy->cached_raw_freq = cached_freq;  in sugov_update_single_freq()
355 	if (!sugov_update_next_freq(sg_policy, time, next_f))  in sugov_update_single_freq()
363 	if (sg_policy->policy->fast_switch_enabled) {  in sugov_update_single_freq()
364 		cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_single_freq()
366 		raw_spin_lock(&sg_policy->update_lock);  in sugov_update_single_freq()
367 		sugov_deferred_update(sg_policy);  in sugov_update_single_freq()
368 		raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_single_freq()
404 	sg_cpu->sg_policy->last_freq_update_time = time;  in sugov_update_single_perf()
409 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()  local
410 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_next_freq_shared()
429 	return get_next_freq(sg_policy, util, max);  in sugov_next_freq_shared()
436 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()  local
439 	raw_spin_lock(&sg_policy->update_lock);  in sugov_update_shared()
446 	if (sugov_should_update_freq(sg_policy, time)) {  in sugov_update_shared()
449 		if (!sugov_update_next_freq(sg_policy, time, next_f))  in sugov_update_shared()
452 		if (sg_policy->policy->fast_switch_enabled)  in sugov_update_shared()
453 			cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_shared()
455 			sugov_deferred_update(sg_policy);  in sugov_update_shared()
458 	raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_shared()
463 	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);  in sugov_work()  local
477 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);  in sugov_work()
478 	freq = sg_policy->next_freq;  in sugov_work()
479 	sg_policy->work_in_progress = false;  in sugov_work()
480 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);  in sugov_work()
482 	mutex_lock(&sg_policy->work_lock);  in sugov_work()
483 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);  in sugov_work()
484 	mutex_unlock(&sg_policy->work_lock);  in sugov_work()
489 	struct sugov_policy *sg_policy;  in sugov_irq_work()  local
491 	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);  in sugov_irq_work()
493 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);  in sugov_irq_work()
517 	struct sugov_policy *sg_policy;  in rate_limit_us_store()  local
525 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)  in rate_limit_us_store()
526 		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;  in rate_limit_us_store()
558 	struct sugov_policy *sg_policy;  in sugov_policy_alloc()  local
560 	sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);  in sugov_policy_alloc()
561 	if (!sg_policy)  in sugov_policy_alloc()
564 	sg_policy->policy = policy;  in sugov_policy_alloc()
565 	raw_spin_lock_init(&sg_policy->update_lock);  in sugov_policy_alloc()
566 	return sg_policy;  in sugov_policy_alloc()
569 static void sugov_policy_free(struct sugov_policy *sg_policy)  in sugov_policy_free()  argument
571 	kfree(sg_policy);  in sugov_policy_free()
574 static int sugov_kthread_create(struct sugov_policy *sg_policy)  in sugov_kthread_create()  argument
591 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_kthread_create()
598 	kthread_init_work(&sg_policy->work, sugov_work);  in sugov_kthread_create()
599 	kthread_init_worker(&sg_policy->worker);  in sugov_kthread_create()
600 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,  in sugov_kthread_create()
615 	sg_policy->thread = thread;  in sugov_kthread_create()
617 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);  in sugov_kthread_create()
618 	mutex_init(&sg_policy->work_lock);  in sugov_kthread_create()
625 static void sugov_kthread_stop(struct sugov_policy *sg_policy)  in sugov_kthread_stop()  argument
628 	if (sg_policy->policy->fast_switch_enabled)  in sugov_kthread_stop()
631 	kthread_flush_worker(&sg_policy->worker);  in sugov_kthread_stop()
632 	kthread_stop(sg_policy->thread);  in sugov_kthread_stop()
633 	mutex_destroy(&sg_policy->work_lock);  in sugov_kthread_stop()
636 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)  in sugov_tunables_alloc()  argument
642 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_tunables_alloc()
657 	struct sugov_policy *sg_policy;  in sugov_init()  local
667 	sg_policy = sugov_policy_alloc(policy);  in sugov_init()
668 	if (!sg_policy) {  in sugov_init()
673 	ret = sugov_kthread_create(sg_policy);  in sugov_init()
684 		policy->governor_data = sg_policy;  in sugov_init()
685 		sg_policy->tunables = global_tunables;  in sugov_init()
687 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);  in sugov_init()
691 	tunables = sugov_tunables_alloc(sg_policy);  in sugov_init()
699 	policy->governor_data = sg_policy;  in sugov_init()
700 	sg_policy->tunables = tunables;  in sugov_init()
718 	sugov_kthread_stop(sg_policy);  in sugov_init()
722 	sugov_policy_free(sg_policy);  in sugov_init()
733 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_exit()  local
734 	struct sugov_tunables *tunables = sg_policy->tunables;  in sugov_exit()
739 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_exit()
746 	sugov_kthread_stop(sg_policy);  in sugov_exit()
747 	sugov_policy_free(sg_policy);  in sugov_exit()
753 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_start()  local
757 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;  in sugov_start()
758 	sg_policy->last_freq_update_time	= 0;  in sugov_start()
759 	sg_policy->next_freq			= 0;  in sugov_start()
760 	sg_policy->work_in_progress		= false;  in sugov_start()
761 	sg_policy->limits_changed		= false;  in sugov_start()
762 	sg_policy->cached_raw_freq		= 0;  in sugov_start()
764 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_start()
771 		sg_cpu->sg_policy		= sg_policy;  in sugov_start()
791 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_stop()  local
800 		irq_work_sync(&sg_policy->irq_work);  in sugov_stop()
801 		kthread_cancel_work_sync(&sg_policy->work);  in sugov_stop()
807 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_limits()  local
810 		mutex_lock(&sg_policy->work_lock);  in sugov_limits()
812 		mutex_unlock(&sg_policy->work_lock);  in sugov_limits()
815 	sg_policy->limits_changed = true;  in sugov_limits()