51 	struct policy_dbs_info *policy_dbs;  in store_sampling_rate()  local
65 	list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {  in store_sampling_rate()
66 		mutex_lock(&policy_dbs->update_mutex);  in store_sampling_rate()
80 		gov_update_sample_delay(policy_dbs, 0);  in store_sampling_rate()
81 		mutex_unlock(&policy_dbs->update_mutex);  in store_sampling_rate()
100 	struct policy_dbs_info *policy_dbs;  in gov_update_cpu_data()  local
102 	list_for_each_entry(policy_dbs, &dbs_data->attr_set.policy_list, list) {  in gov_update_cpu_data()
105 		for_each_cpu(j, policy_dbs->policy->cpus) {  in gov_update_cpu_data()
119 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in dbs_update()  local
120 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in dbs_update()
131 	sampling_rate = dbs_data->sampling_rate * policy_dbs->rate_mult;  in dbs_update()
229 	policy_dbs->idle_periods = idle_periods;  in dbs_update()
237 	struct policy_dbs_info *policy_dbs;  in dbs_work_handler()  local
241 	policy_dbs = container_of(work, struct policy_dbs_info, work);  in dbs_work_handler()
242 	policy = policy_dbs->policy;  in dbs_work_handler()
249 	mutex_lock(&policy_dbs->update_mutex);  in dbs_work_handler()
250 	gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy));  in dbs_work_handler()
251 	mutex_unlock(&policy_dbs->update_mutex);  in dbs_work_handler()
254 	atomic_set(&policy_dbs->work_count, 0);  in dbs_work_handler()
261 	policy_dbs->work_in_progress = false;  in dbs_work_handler()
266 	struct policy_dbs_info *policy_dbs;  in dbs_irq_work()  local
268 	policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work);  in dbs_irq_work()
269 	schedule_work_on(smp_processor_id(), &policy_dbs->work);  in dbs_irq_work()
276 	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;  in dbs_update_util_handler()  local
279 	if (!cpufreq_this_cpu_can_update(policy_dbs->policy))  in dbs_update_util_handler()
288 	if (policy_dbs->work_in_progress)  in dbs_update_util_handler()
296 	lst = READ_ONCE(policy_dbs->last_sample_time);  in dbs_update_util_handler()
298 	if ((s64)delta_ns < policy_dbs->sample_delay_ns)  in dbs_update_util_handler()
306 	if (policy_dbs->is_shared) {  in dbs_update_util_handler()
307 		if (!atomic_add_unless(&policy_dbs->work_count, 1, 1))  in dbs_update_util_handler()
314 		if (unlikely(lst != READ_ONCE(policy_dbs->last_sample_time))) {  in dbs_update_util_handler()
315 			atomic_set(&policy_dbs->work_count, 0);  in dbs_update_util_handler()
320 	policy_dbs->last_sample_time = time;  in dbs_update_util_handler()
321 	policy_dbs->work_in_progress = true;  in dbs_update_util_handler()
322 	irq_work_queue(&policy_dbs->irq_work);  in dbs_update_util_handler()
325 static void gov_set_update_util(struct policy_dbs_info *policy_dbs,  in gov_set_update_util()  argument
328 	struct cpufreq_policy *policy = policy_dbs->policy;  in gov_set_update_util()
331 	gov_update_sample_delay(policy_dbs, delay_us);  in gov_set_update_util()
332 	policy_dbs->last_sample_time = 0;  in gov_set_update_util()
355 	struct policy_dbs_info *policy_dbs;  in alloc_policy_dbs_info()  local
359 	policy_dbs = gov->alloc();  in alloc_policy_dbs_info()
360 	if (!policy_dbs)  in alloc_policy_dbs_info()
363 	policy_dbs->policy = policy;  in alloc_policy_dbs_info()
364 	mutex_init(&policy_dbs->update_mutex);  in alloc_policy_dbs_info()
365 	atomic_set(&policy_dbs->work_count, 0);  in alloc_policy_dbs_info()
366 	init_irq_work(&policy_dbs->irq_work, dbs_irq_work);  in alloc_policy_dbs_info()
367 	INIT_WORK(&policy_dbs->work, dbs_work_handler);  in alloc_policy_dbs_info()
373 		j_cdbs->policy_dbs = policy_dbs;  in alloc_policy_dbs_info()
375 	return policy_dbs;  in alloc_policy_dbs_info()
378 static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs,  in free_policy_dbs_info()  argument
383 	mutex_destroy(&policy_dbs->update_mutex);  in free_policy_dbs_info()
385 	for_each_cpu(j, policy_dbs->policy->related_cpus) {  in free_policy_dbs_info()
388 		j_cdbs->policy_dbs = NULL;  in free_policy_dbs_info()
391 	gov->free(policy_dbs);  in free_policy_dbs_info()
398 	struct policy_dbs_info *policy_dbs;  in cpufreq_dbs_governor_init()  local
405 	policy_dbs = alloc_policy_dbs_info(policy, gov);  in cpufreq_dbs_governor_init()
406 	if (!policy_dbs)  in cpufreq_dbs_governor_init()
418 		policy_dbs->dbs_data = dbs_data;  in cpufreq_dbs_governor_init()
419 		policy->governor_data = policy_dbs;  in cpufreq_dbs_governor_init()
421 		gov_attr_set_get(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_init()
431 	gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_init()
449 	policy_dbs->dbs_data = dbs_data;  in cpufreq_dbs_governor_init()
450 	policy->governor_data = policy_dbs;  in cpufreq_dbs_governor_init()
470 	free_policy_dbs_info(policy_dbs, gov);  in cpufreq_dbs_governor_init()
481 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_exit()  local
482 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in cpufreq_dbs_governor_exit()
488 	count = gov_attr_set_put(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_exit()
500 	free_policy_dbs_info(policy_dbs, gov);  in cpufreq_dbs_governor_exit()
509 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_start()  local
510 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in cpufreq_dbs_governor_start()
517 	policy_dbs->is_shared = policy_is_shared(policy);  in cpufreq_dbs_governor_start()
518 	policy_dbs->rate_mult = 1;  in cpufreq_dbs_governor_start()
539 	gov_set_update_util(policy_dbs, sampling_rate);  in cpufreq_dbs_governor_start()
546 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_stop()  local
548 	gov_clear_update_util(policy_dbs->policy);  in cpufreq_dbs_governor_stop()
549 	irq_work_sync(&policy_dbs->irq_work);  in cpufreq_dbs_governor_stop()
550 	cancel_work_sync(&policy_dbs->work);  in cpufreq_dbs_governor_stop()
551 	atomic_set(&policy_dbs->work_count, 0);  in cpufreq_dbs_governor_stop()
552 	policy_dbs->work_in_progress = false;  in cpufreq_dbs_governor_stop()
558 	struct policy_dbs_info *policy_dbs;  in cpufreq_dbs_governor_limits()  local
562 	policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_limits()
563 	if (!policy_dbs)  in cpufreq_dbs_governor_limits()
566 	mutex_lock(&policy_dbs->update_mutex);  in cpufreq_dbs_governor_limits()
568 	gov_update_sample_delay(policy_dbs, 0);  in cpufreq_dbs_governor_limits()
569 	mutex_unlock(&policy_dbs->update_mutex);  in cpufreq_dbs_governor_limits()