48 	struct policy_dbs_info *policy_dbs;  in store_sampling_rate()  local
62 	list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {  in store_sampling_rate()
63 		mutex_lock(&policy_dbs->update_mutex);  in store_sampling_rate()
77 		gov_update_sample_delay(policy_dbs, 0);  in store_sampling_rate()
78 		mutex_unlock(&policy_dbs->update_mutex);  in store_sampling_rate()
97 	struct policy_dbs_info *policy_dbs;  in gov_update_cpu_data()  local
99 	list_for_each_entry(policy_dbs, &dbs_data->attr_set.policy_list, list) {  in gov_update_cpu_data()
102 		for_each_cpu(j, policy_dbs->policy->cpus) {  in gov_update_cpu_data()
116 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in dbs_update()  local
117 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in dbs_update()
128 	sampling_rate = dbs_data->sampling_rate * policy_dbs->rate_mult;  in dbs_update()
226 	policy_dbs->idle_periods = idle_periods;  in dbs_update()
234 	struct policy_dbs_info *policy_dbs;  in dbs_work_handler()  local
238 	policy_dbs = container_of(work, struct policy_dbs_info, work);  in dbs_work_handler()
239 	policy = policy_dbs->policy;  in dbs_work_handler()
246 	mutex_lock(&policy_dbs->update_mutex);  in dbs_work_handler()
247 	gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy));  in dbs_work_handler()
248 	mutex_unlock(&policy_dbs->update_mutex);  in dbs_work_handler()
251 	atomic_set(&policy_dbs->work_count, 0);  in dbs_work_handler()
258 	policy_dbs->work_in_progress = false;  in dbs_work_handler()
263 	struct policy_dbs_info *policy_dbs;  in dbs_irq_work()  local
265 	policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work);  in dbs_irq_work()
266 	schedule_work_on(smp_processor_id(), &policy_dbs->work);  in dbs_irq_work()
273 	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;  in dbs_update_util_handler()  local
276 	if (!cpufreq_this_cpu_can_update(policy_dbs->policy))  in dbs_update_util_handler()
285 	if (policy_dbs->work_in_progress)  in dbs_update_util_handler()
293 	lst = READ_ONCE(policy_dbs->last_sample_time);  in dbs_update_util_handler()
295 	if ((s64)delta_ns < policy_dbs->sample_delay_ns)  in dbs_update_util_handler()
303 	if (policy_dbs->is_shared) {  in dbs_update_util_handler()
304 		if (!atomic_add_unless(&policy_dbs->work_count, 1, 1))  in dbs_update_util_handler()
311 		if (unlikely(lst != READ_ONCE(policy_dbs->last_sample_time))) {  in dbs_update_util_handler()
312 			atomic_set(&policy_dbs->work_count, 0);  in dbs_update_util_handler()
317 	policy_dbs->last_sample_time = time;  in dbs_update_util_handler()
318 	policy_dbs->work_in_progress = true;  in dbs_update_util_handler()
319 	irq_work_queue(&policy_dbs->irq_work);  in dbs_update_util_handler()
322 static void gov_set_update_util(struct policy_dbs_info *policy_dbs,  in gov_set_update_util()  argument
325 	struct cpufreq_policy *policy = policy_dbs->policy;  in gov_set_update_util()
328 	gov_update_sample_delay(policy_dbs, delay_us);  in gov_set_update_util()
329 	policy_dbs->last_sample_time = 0;  in gov_set_update_util()
352 	struct policy_dbs_info *policy_dbs;  in alloc_policy_dbs_info()  local
356 	policy_dbs = gov->alloc();  in alloc_policy_dbs_info()
357 	if (!policy_dbs)  in alloc_policy_dbs_info()
360 	policy_dbs->policy = policy;  in alloc_policy_dbs_info()
361 	mutex_init(&policy_dbs->update_mutex);  in alloc_policy_dbs_info()
362 	atomic_set(&policy_dbs->work_count, 0);  in alloc_policy_dbs_info()
363 	init_irq_work(&policy_dbs->irq_work, dbs_irq_work);  in alloc_policy_dbs_info()
364 	INIT_WORK(&policy_dbs->work, dbs_work_handler);  in alloc_policy_dbs_info()
370 		j_cdbs->policy_dbs = policy_dbs;  in alloc_policy_dbs_info()
372 	return policy_dbs;  in alloc_policy_dbs_info()
375 static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs,  in free_policy_dbs_info()  argument
380 	mutex_destroy(&policy_dbs->update_mutex);  in free_policy_dbs_info()
382 	for_each_cpu(j, policy_dbs->policy->related_cpus) {  in free_policy_dbs_info()
385 		j_cdbs->policy_dbs = NULL;  in free_policy_dbs_info()
388 	gov->free(policy_dbs);  in free_policy_dbs_info()
395 	struct policy_dbs_info *policy_dbs;  in cpufreq_dbs_governor_init()  local
402 	policy_dbs = alloc_policy_dbs_info(policy, gov);  in cpufreq_dbs_governor_init()
403 	if (!policy_dbs)  in cpufreq_dbs_governor_init()
415 		policy_dbs->dbs_data = dbs_data;  in cpufreq_dbs_governor_init()
416 		policy->governor_data = policy_dbs;  in cpufreq_dbs_governor_init()
418 		gov_attr_set_get(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_init()
428 	gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_init()
446 	policy_dbs->dbs_data = dbs_data;  in cpufreq_dbs_governor_init()
447 	policy->governor_data = policy_dbs;  in cpufreq_dbs_governor_init()
469 	free_policy_dbs_info(policy_dbs, gov);  in cpufreq_dbs_governor_init()
480 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_exit()  local
481 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in cpufreq_dbs_governor_exit()
487 	count = gov_attr_set_put(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_exit()
499 	free_policy_dbs_info(policy_dbs, gov);  in cpufreq_dbs_governor_exit()
508 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_start()  local
509 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in cpufreq_dbs_governor_start()
516 	policy_dbs->is_shared = policy_is_shared(policy);  in cpufreq_dbs_governor_start()
517 	policy_dbs->rate_mult = 1;  in cpufreq_dbs_governor_start()
538 	gov_set_update_util(policy_dbs, sampling_rate);  in cpufreq_dbs_governor_start()
545 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_stop()  local
547 	gov_clear_update_util(policy_dbs->policy);  in cpufreq_dbs_governor_stop()
548 	irq_work_sync(&policy_dbs->irq_work);  in cpufreq_dbs_governor_stop()
549 	cancel_work_sync(&policy_dbs->work);  in cpufreq_dbs_governor_stop()
550 	atomic_set(&policy_dbs->work_count, 0);  in cpufreq_dbs_governor_stop()
551 	policy_dbs->work_in_progress = false;  in cpufreq_dbs_governor_stop()
557 	struct policy_dbs_info *policy_dbs;  in cpufreq_dbs_governor_limits()  local
561 	policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_limits()
562 	if (!policy_dbs)  in cpufreq_dbs_governor_limits()
565 	mutex_lock(&policy_dbs->update_mutex);  in cpufreq_dbs_governor_limits()
567 	gov_update_sample_delay(policy_dbs, 0);  in cpufreq_dbs_governor_limits()
568 	mutex_unlock(&policy_dbs->update_mutex);  in cpufreq_dbs_governor_limits()