71 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
72 static int cpufreq_init_governor(struct cpufreq_policy *policy);
73 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
74 static int cpufreq_start_governor(struct cpufreq_policy *policy);
75 static void cpufreq_stop_governor(struct cpufreq_policy *policy);
76 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
105 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)  in get_governor_parent_kobj()  argument
108 		return &policy->kobj;  in get_governor_parent_kobj()
162 void cpufreq_generic_init(struct cpufreq_policy *policy,  in cpufreq_generic_init()  argument
166 	policy->freq_table = table;  in cpufreq_generic_init()
167 	policy->cpuinfo.transition_latency = transition_latency;  in cpufreq_generic_init()
173 	cpumask_setall(policy->cpus);  in cpufreq_generic_init()
179 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_cpu_get_raw()  local
181 	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;  in cpufreq_cpu_get_raw()
187 	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);  in cpufreq_generic_get()  local
189 	if (!policy || IS_ERR(policy->clk)) {  in cpufreq_generic_get()
191 		       __func__, policy ? "clk" : "policy", cpu);  in cpufreq_generic_get()
195 	return clk_get_rate(policy->clk) / 1000;  in cpufreq_generic_get()
212 	struct cpufreq_policy *policy = NULL;  in cpufreq_cpu_get()  local
223 		policy = cpufreq_cpu_get_raw(cpu);  in cpufreq_cpu_get()
224 		if (policy)  in cpufreq_cpu_get()
225 			kobject_get(&policy->kobj);  in cpufreq_cpu_get()
230 	return policy;  in cpufreq_cpu_get()
238 void cpufreq_cpu_put(struct cpufreq_policy *policy)  in cpufreq_cpu_put()  argument
240 	kobject_put(&policy->kobj);  in cpufreq_cpu_put()
248 void cpufreq_cpu_release(struct cpufreq_policy *policy)  in cpufreq_cpu_release()  argument
250 	if (WARN_ON(!policy))  in cpufreq_cpu_release()
253 	lockdep_assert_held(&policy->rwsem);  in cpufreq_cpu_release()
255 	up_write(&policy->rwsem);  in cpufreq_cpu_release()
257 	cpufreq_cpu_put(policy);  in cpufreq_cpu_release()
274 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_cpu_acquire()  local
276 	if (!policy)  in cpufreq_cpu_acquire()
279 	down_write(&policy->rwsem);  in cpufreq_cpu_acquire()
281 	if (policy_is_inactive(policy)) {  in cpufreq_cpu_acquire()
282 		cpufreq_cpu_release(policy);  in cpufreq_cpu_acquire()
286 	return policy;  in cpufreq_cpu_acquire()
335 static void cpufreq_notify_transition(struct cpufreq_policy *policy,  in cpufreq_notify_transition()  argument
346 	freqs->policy = policy;  in cpufreq_notify_transition()
358 		if (policy->cur && policy->cur != freqs->old) {  in cpufreq_notify_transition()
360 				 freqs->old, policy->cur);  in cpufreq_notify_transition()
361 			freqs->old = policy->cur;  in cpufreq_notify_transition()
373 			 cpumask_pr_args(policy->cpus));  in cpufreq_notify_transition()
375 		for_each_cpu(cpu, policy->cpus)  in cpufreq_notify_transition()
381 		cpufreq_stats_record_transition(policy, freqs->new);  in cpufreq_notify_transition()
382 		policy->cur = freqs->new;  in cpufreq_notify_transition()
387 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,  in cpufreq_notify_post_transition()  argument
390 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);  in cpufreq_notify_post_transition()
395 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);  in cpufreq_notify_post_transition()
396 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);  in cpufreq_notify_post_transition()
399 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,  in cpufreq_freq_transition_begin()  argument
412 				&& current == policy->transition_task);  in cpufreq_freq_transition_begin()
415 	wait_event(policy->transition_wait, !policy->transition_ongoing);  in cpufreq_freq_transition_begin()
417 	spin_lock(&policy->transition_lock);  in cpufreq_freq_transition_begin()
419 	if (unlikely(policy->transition_ongoing)) {  in cpufreq_freq_transition_begin()
420 		spin_unlock(&policy->transition_lock);  in cpufreq_freq_transition_begin()
424 	policy->transition_ongoing = true;  in cpufreq_freq_transition_begin()
425 	policy->transition_task = current;  in cpufreq_freq_transition_begin()
427 	spin_unlock(&policy->transition_lock);  in cpufreq_freq_transition_begin()
429 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);  in cpufreq_freq_transition_begin()
433 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,  in cpufreq_freq_transition_end()  argument
436 	if (WARN_ON(!policy->transition_ongoing))  in cpufreq_freq_transition_end()
439 	cpufreq_notify_post_transition(policy, freqs, transition_failed);  in cpufreq_freq_transition_end()
441 	policy->transition_ongoing = false;  in cpufreq_freq_transition_end()
442 	policy->transition_task = NULL;  in cpufreq_freq_transition_end()
444 	wake_up(&policy->transition_wait);  in cpufreq_freq_transition_end()
480 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)  in cpufreq_enable_fast_switch()  argument
482 	lockdep_assert_held(&policy->rwsem);  in cpufreq_enable_fast_switch()
484 	if (!policy->fast_switch_possible)  in cpufreq_enable_fast_switch()
490 		policy->fast_switch_enabled = true;  in cpufreq_enable_fast_switch()
493 			policy->cpu);  in cpufreq_enable_fast_switch()
504 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)  in cpufreq_disable_fast_switch()  argument
507 	if (policy->fast_switch_enabled) {  in cpufreq_disable_fast_switch()
508 		policy->fast_switch_enabled = false;  in cpufreq_disable_fast_switch()
526 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,  in cpufreq_driver_resolve_freq()  argument
529 	target_freq = clamp_val(target_freq, policy->min, policy->max);  in cpufreq_driver_resolve_freq()
530 	policy->cached_target_freq = target_freq;  in cpufreq_driver_resolve_freq()
535 		idx = cpufreq_frequency_table_target(policy, target_freq,  in cpufreq_driver_resolve_freq()
537 		policy->cached_resolved_idx = idx;  in cpufreq_driver_resolve_freq()
538 		return policy->freq_table[idx].frequency;  in cpufreq_driver_resolve_freq()
542 		return cpufreq_driver->resolve_freq(policy, target_freq);  in cpufreq_driver_resolve_freq()
548 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)  in cpufreq_policy_transition_delay_us()  argument
552 	if (policy->transition_delay_us)  in cpufreq_policy_transition_delay_us()
553 		return policy->transition_delay_us;  in cpufreq_policy_transition_delay_us()
555 	latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;  in cpufreq_policy_transition_delay_us()
617 				struct cpufreq_policy *policy)  in cpufreq_parse_policy()  argument
620 		policy->policy = CPUFREQ_POLICY_PERFORMANCE;  in cpufreq_parse_policy()
624 		policy->policy = CPUFREQ_POLICY_POWERSAVE;  in cpufreq_parse_policy()
634 				  struct cpufreq_policy *policy)  in cpufreq_parse_governor()  argument
660 		policy->governor = t;  in cpufreq_parse_governor()
677 (struct cpufreq_policy *policy, char *buf)		\
679 	return sprintf(buf, "%u\n", policy->object);	\
693 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)  in show_scaling_cur_freq()  argument
698 	freq = arch_freq_get_on_cpu(policy->cpu);  in show_scaling_cur_freq()
703 		ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));  in show_scaling_cur_freq()
705 		ret = sprintf(buf, "%u\n", policy->cur);  in show_scaling_cur_freq()
714 (struct cpufreq_policy *policy, const char *buf, size_t count)		\
723 	ret = freq_qos_update_request(policy->object##_freq_req, val);\
733 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,  in show_cpuinfo_cur_freq()  argument
736 	unsigned int cur_freq = __cpufreq_get(policy);  in show_cpuinfo_cur_freq()
747 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)  in show_scaling_governor()  argument
749 	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)  in show_scaling_governor()
751 	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)  in show_scaling_governor()
753 	else if (policy->governor)  in show_scaling_governor()
755 				policy->governor->name);  in show_scaling_governor()
762 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,  in store_scaling_governor()  argument
769 	memcpy(&new_policy, policy, sizeof(*policy));  in store_scaling_governor()
783 	ret = cpufreq_set_policy(policy, &new_policy);  in store_scaling_governor()
794 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)  in show_scaling_driver()  argument
802 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,  in show_scaling_available_governors()  argument
845 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)  in show_related_cpus()  argument
847 	return cpufreq_show_cpus(policy->related_cpus, buf);  in show_related_cpus()
853 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)  in show_affected_cpus()  argument
855 	return cpufreq_show_cpus(policy->cpus, buf);  in show_affected_cpus()
858 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,  in store_scaling_setspeed()  argument
864 	if (!policy->governor || !policy->governor->store_setspeed)  in store_scaling_setspeed()
871 	policy->governor->store_setspeed(policy, freq);  in store_scaling_setspeed()
876 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)  in show_scaling_setspeed()  argument
878 	if (!policy->governor || !policy->governor->show_setspeed)  in show_scaling_setspeed()
881 	return policy->governor->show_setspeed(policy, buf);  in show_scaling_setspeed()
887 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)  in show_bios_limit()  argument
891 	ret = cpufreq_driver->bios_limit(policy->cpu, &limit);  in show_bios_limit()
894 	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);  in show_bios_limit()
932 	struct cpufreq_policy *policy = to_policy(kobj);  in show()  local
936 	down_read(&policy->rwsem);  in show()
937 	ret = fattr->show(policy, buf);  in show()
938 	up_read(&policy->rwsem);  in show()
946 	struct cpufreq_policy *policy = to_policy(kobj);  in store()  local
957 	if (cpu_online(policy->cpu)) {  in store()
958 		down_write(&policy->rwsem);  in store()
959 		ret = fattr->store(policy, buf, count);  in store()
960 		up_write(&policy->rwsem);  in store()
970 	struct cpufreq_policy *policy = to_policy(kobj);  in cpufreq_sysfs_release()  local
972 	complete(&policy->kobj_unregister);  in cpufreq_sysfs_release()
986 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)  in add_cpu_dev_symlink()  argument
993 	if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))  in add_cpu_dev_symlink()
997 	if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))  in add_cpu_dev_symlink()
1001 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,  in remove_cpu_dev_symlink()  argument
1008 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)  in cpufreq_add_dev_interface()  argument
1016 		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));  in cpufreq_add_dev_interface()
1022 		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);  in cpufreq_add_dev_interface()
1027 	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);  in cpufreq_add_dev_interface()
1032 		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);  in cpufreq_add_dev_interface()
1045 static int cpufreq_init_policy(struct cpufreq_policy *policy)  in cpufreq_init_policy()  argument
1050 	memcpy(&new_policy, policy, sizeof(*policy));  in cpufreq_init_policy()
1059 		gov = find_governor(policy->last_governor);  in cpufreq_init_policy()
1062 				policy->governor->name, policy->cpu);  in cpufreq_init_policy()
1071 		if (policy->last_policy) {  in cpufreq_init_policy()
1072 			new_policy.policy = policy->last_policy;  in cpufreq_init_policy()
1080 	return cpufreq_set_policy(policy, &new_policy);  in cpufreq_init_policy()
1083 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)  in cpufreq_add_policy_cpu()  argument
1088 	if (cpumask_test_cpu(cpu, policy->cpus))  in cpufreq_add_policy_cpu()
1091 	down_write(&policy->rwsem);  in cpufreq_add_policy_cpu()
1093 		cpufreq_stop_governor(policy);  in cpufreq_add_policy_cpu()
1095 	cpumask_set_cpu(cpu, policy->cpus);  in cpufreq_add_policy_cpu()
1098 		ret = cpufreq_start_governor(policy);  in cpufreq_add_policy_cpu()
1102 	up_write(&policy->rwsem);  in cpufreq_add_policy_cpu()
1106 void refresh_frequency_limits(struct cpufreq_policy *policy)  in refresh_frequency_limits()  argument
1110 	if (!policy_is_inactive(policy)) {  in refresh_frequency_limits()
1111 		new_policy = *policy;  in refresh_frequency_limits()
1112 		pr_debug("updating policy for CPU %u\n", policy->cpu);  in refresh_frequency_limits()
1114 		cpufreq_set_policy(policy, &new_policy);  in refresh_frequency_limits()
1121 	struct cpufreq_policy *policy =  in handle_update()  local
1124 	pr_debug("handle_update for cpu %u called\n", policy->cpu);  in handle_update()
1125 	down_write(&policy->rwsem);  in handle_update()
1126 	refresh_frequency_limits(policy);  in handle_update()
1127 	up_write(&policy->rwsem);  in handle_update()
1133 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);  in cpufreq_notifier_min()  local
1135 	schedule_work(&policy->update);  in cpufreq_notifier_min()
1142 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);  in cpufreq_notifier_max()  local
1144 	schedule_work(&policy->update);  in cpufreq_notifier_max()
1148 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)  in cpufreq_policy_put_kobj()  argument
1153 	down_write(&policy->rwsem);  in cpufreq_policy_put_kobj()
1154 	cpufreq_stats_free_table(policy);  in cpufreq_policy_put_kobj()
1155 	kobj = &policy->kobj;  in cpufreq_policy_put_kobj()
1156 	cmp = &policy->kobj_unregister;  in cpufreq_policy_put_kobj()
1157 	up_write(&policy->rwsem);  in cpufreq_policy_put_kobj()
1172 	struct cpufreq_policy *policy;  in cpufreq_policy_alloc()  local
1179 	policy = kzalloc(sizeof(*policy), GFP_KERNEL);  in cpufreq_policy_alloc()
1180 	if (!policy)  in cpufreq_policy_alloc()
1183 	if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))  in cpufreq_policy_alloc()
1186 	if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))  in cpufreq_policy_alloc()
1189 	if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))  in cpufreq_policy_alloc()
1192 	ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,  in cpufreq_policy_alloc()
1201 		kobject_put(&policy->kobj);  in cpufreq_policy_alloc()
1205 	freq_constraints_init(&policy->constraints);  in cpufreq_policy_alloc()
1207 	policy->nb_min.notifier_call = cpufreq_notifier_min;  in cpufreq_policy_alloc()
1208 	policy->nb_max.notifier_call = cpufreq_notifier_max;  in cpufreq_policy_alloc()
1210 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,  in cpufreq_policy_alloc()
1211 				    &policy->nb_min);  in cpufreq_policy_alloc()
1214 			ret, cpumask_pr_args(policy->cpus));  in cpufreq_policy_alloc()
1218 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,  in cpufreq_policy_alloc()
1219 				    &policy->nb_max);  in cpufreq_policy_alloc()
1222 			ret, cpumask_pr_args(policy->cpus));  in cpufreq_policy_alloc()
1226 	INIT_LIST_HEAD(&policy->policy_list);  in cpufreq_policy_alloc()
1227 	init_rwsem(&policy->rwsem);  in cpufreq_policy_alloc()
1228 	spin_lock_init(&policy->transition_lock);  in cpufreq_policy_alloc()
1229 	init_waitqueue_head(&policy->transition_wait);  in cpufreq_policy_alloc()
1230 	init_completion(&policy->kobj_unregister);  in cpufreq_policy_alloc()
1231 	INIT_WORK(&policy->update, handle_update);  in cpufreq_policy_alloc()
1233 	policy->cpu = cpu;  in cpufreq_policy_alloc()
1234 	return policy;  in cpufreq_policy_alloc()
1237 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,  in cpufreq_policy_alloc()
1238 				 &policy->nb_min);  in cpufreq_policy_alloc()
1240 	cpufreq_policy_put_kobj(policy);  in cpufreq_policy_alloc()
1242 	free_cpumask_var(policy->real_cpus);  in cpufreq_policy_alloc()
1244 	free_cpumask_var(policy->related_cpus);  in cpufreq_policy_alloc()
1246 	free_cpumask_var(policy->cpus);  in cpufreq_policy_alloc()
1248 	kfree(policy);  in cpufreq_policy_alloc()
1253 static void cpufreq_policy_free(struct cpufreq_policy *policy)  in cpufreq_policy_free()  argument
1260 	list_del(&policy->policy_list);  in cpufreq_policy_free()
1262 	for_each_cpu(cpu, policy->related_cpus)  in cpufreq_policy_free()
1266 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,  in cpufreq_policy_free()
1267 				 &policy->nb_max);  in cpufreq_policy_free()
1268 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,  in cpufreq_policy_free()
1269 				 &policy->nb_min);  in cpufreq_policy_free()
1272 	cancel_work_sync(&policy->update);  in cpufreq_policy_free()
1274 	if (policy->max_freq_req) {  in cpufreq_policy_free()
1280 					     CPUFREQ_REMOVE_POLICY, policy);  in cpufreq_policy_free()
1281 		freq_qos_remove_request(policy->max_freq_req);  in cpufreq_policy_free()
1284 	freq_qos_remove_request(policy->min_freq_req);  in cpufreq_policy_free()
1285 	kfree(policy->min_freq_req);  in cpufreq_policy_free()
1287 	cpufreq_policy_put_kobj(policy);  in cpufreq_policy_free()
1288 	free_cpumask_var(policy->real_cpus);  in cpufreq_policy_free()
1289 	free_cpumask_var(policy->related_cpus);  in cpufreq_policy_free()
1290 	free_cpumask_var(policy->cpus);  in cpufreq_policy_free()
1291 	kfree(policy);  in cpufreq_policy_free()
1296 	struct cpufreq_policy *policy;  in cpufreq_online()  local
1305 	policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_online()
1306 	if (policy) {  in cpufreq_online()
1307 		WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));  in cpufreq_online()
1308 		if (!policy_is_inactive(policy))  in cpufreq_online()
1309 			return cpufreq_add_policy_cpu(policy, cpu);  in cpufreq_online()
1313 		down_write(&policy->rwsem);  in cpufreq_online()
1314 		policy->cpu = cpu;  in cpufreq_online()
1315 		policy->governor = NULL;  in cpufreq_online()
1316 		up_write(&policy->rwsem);  in cpufreq_online()
1319 		policy = cpufreq_policy_alloc(cpu);  in cpufreq_online()
1320 		if (!policy)  in cpufreq_online()
1325 		ret = cpufreq_driver->online(policy);  in cpufreq_online()
1333 		cpumask_copy(policy->cpus, policy->related_cpus);  in cpufreq_online()
1335 		cpumask_copy(policy->cpus, cpumask_of(cpu));  in cpufreq_online()
1341 		ret = cpufreq_driver->init(policy);  in cpufreq_online()
1348 		ret = cpufreq_table_validate_and_sort(policy);  in cpufreq_online()
1353 		cpumask_copy(policy->related_cpus, policy->cpus);  in cpufreq_online()
1356 	down_write(&policy->rwsem);  in cpufreq_online()
1361 	cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);  in cpufreq_online()
1364 		for_each_cpu(j, policy->related_cpus) {  in cpufreq_online()
1365 			per_cpu(cpufreq_cpu_data, j) = policy;  in cpufreq_online()
1366 			add_cpu_dev_symlink(policy, j);  in cpufreq_online()
1369 		policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),  in cpufreq_online()
1371 		if (!policy->min_freq_req)  in cpufreq_online()
1374 		ret = freq_qos_add_request(&policy->constraints,  in cpufreq_online()
1375 					   policy->min_freq_req, FREQ_QOS_MIN,  in cpufreq_online()
1376 					   policy->min);  in cpufreq_online()
1382 			kfree(policy->min_freq_req);  in cpufreq_online()
1383 			policy->min_freq_req = NULL;  in cpufreq_online()
1392 		policy->max_freq_req = policy->min_freq_req + 1;  in cpufreq_online()
1394 		ret = freq_qos_add_request(&policy->constraints,  in cpufreq_online()
1395 					   policy->max_freq_req, FREQ_QOS_MAX,  in cpufreq_online()
1396 					   policy->max);  in cpufreq_online()
1398 			policy->max_freq_req = NULL;  in cpufreq_online()
1403 				CPUFREQ_CREATE_POLICY, policy);  in cpufreq_online()
1407 		policy->cur = cpufreq_driver->get(policy->cpu);  in cpufreq_online()
1408 		if (!policy->cur) {  in cpufreq_online()
1435 		ret = cpufreq_frequency_table_get_index(policy, policy->cur);  in cpufreq_online()
1439 				__func__, policy->cpu, policy->cur);  in cpufreq_online()
1440 			ret = __cpufreq_driver_target(policy, policy->cur - 1,  in cpufreq_online()
1450 				__func__, policy->cpu, policy->cur);  in cpufreq_online()
1455 		ret = cpufreq_add_dev_interface(policy);  in cpufreq_online()
1459 		cpufreq_stats_create_table(policy);  in cpufreq_online()
1462 		list_add(&policy->policy_list, &cpufreq_policy_list);  in cpufreq_online()
1466 	ret = cpufreq_init_policy(policy);  in cpufreq_online()
1473 	up_write(&policy->rwsem);  in cpufreq_online()
1475 	kobject_uevent(&policy->kobj, KOBJ_ADD);  in cpufreq_online()
1479 		cpufreq_driver->ready(policy);  in cpufreq_online()
1482 		policy->cdev = of_cpufreq_cooling_register(policy);  in cpufreq_online()
1489 	for_each_cpu(j, policy->real_cpus)  in cpufreq_online()
1490 		remove_cpu_dev_symlink(policy, get_cpu_device(j));  in cpufreq_online()
1492 	up_write(&policy->rwsem);  in cpufreq_online()
1496 		cpufreq_driver->exit(policy);  in cpufreq_online()
1499 	cpufreq_policy_free(policy);  in cpufreq_online()
1510 	struct cpufreq_policy *policy;  in cpufreq_add_dev()  local
1523 	policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_add_dev()
1524 	if (policy)  in cpufreq_add_dev()
1525 		add_cpu_dev_symlink(policy, cpu);  in cpufreq_add_dev()
1532 	struct cpufreq_policy *policy;  in cpufreq_offline()  local
1537 	policy = cpufreq_cpu_get_raw(cpu);  in cpufreq_offline()
1538 	if (!policy) {  in cpufreq_offline()
1543 	down_write(&policy->rwsem);  in cpufreq_offline()
1545 		cpufreq_stop_governor(policy);  in cpufreq_offline()
1547 	cpumask_clear_cpu(cpu, policy->cpus);  in cpufreq_offline()
1549 	if (policy_is_inactive(policy)) {  in cpufreq_offline()
1551 			strncpy(policy->last_governor, policy->governor->name,  in cpufreq_offline()
1554 			policy->last_policy = policy->policy;  in cpufreq_offline()
1555 	} else if (cpu == policy->cpu) {  in cpufreq_offline()
1557 		policy->cpu = cpumask_any(policy->cpus);  in cpufreq_offline()
1561 	if (!policy_is_inactive(policy)) {  in cpufreq_offline()
1563 			ret = cpufreq_start_governor(policy);  in cpufreq_offline()
1572 		cpufreq_cooling_unregister(policy->cdev);  in cpufreq_offline()
1573 		policy->cdev = NULL;  in cpufreq_offline()
1577 		cpufreq_driver->stop_cpu(policy);  in cpufreq_offline()
1580 		cpufreq_exit_governor(policy);  in cpufreq_offline()
1587 		cpufreq_driver->offline(policy);  in cpufreq_offline()
1589 		cpufreq_driver->exit(policy);  in cpufreq_offline()
1590 		policy->freq_table = NULL;  in cpufreq_offline()
1594 	up_write(&policy->rwsem);  in cpufreq_offline()
1606 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_remove_dev()  local
1608 	if (!policy)  in cpufreq_remove_dev()
1614 	cpumask_clear_cpu(cpu, policy->real_cpus);  in cpufreq_remove_dev()
1615 	remove_cpu_dev_symlink(policy, dev);  in cpufreq_remove_dev()
1617 	if (cpumask_empty(policy->real_cpus)) {  in cpufreq_remove_dev()
1620 			cpufreq_driver->exit(policy);  in cpufreq_remove_dev()
1622 		cpufreq_policy_free(policy);  in cpufreq_remove_dev()
1635 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,  in cpufreq_out_of_sync()  argument
1641 		 policy->cur, new_freq);  in cpufreq_out_of_sync()
1643 	freqs.old = policy->cur;  in cpufreq_out_of_sync()
1646 	cpufreq_freq_transition_begin(policy, &freqs);  in cpufreq_out_of_sync()
1647 	cpufreq_freq_transition_end(policy, &freqs, 0);  in cpufreq_out_of_sync()
1650 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)  in cpufreq_verify_current_freq()  argument
1654 	new_freq = cpufreq_driver->get(policy->cpu);  in cpufreq_verify_current_freq()
1662 	if (policy->fast_switch_enabled || !has_target())  in cpufreq_verify_current_freq()
1665 	if (policy->cur != new_freq) {  in cpufreq_verify_current_freq()
1666 		cpufreq_out_of_sync(policy, new_freq);  in cpufreq_verify_current_freq()
1668 			schedule_work(&policy->update);  in cpufreq_verify_current_freq()
1683 	struct cpufreq_policy *policy;  in cpufreq_quick_get()  local
1697 	policy = cpufreq_cpu_get(cpu);  in cpufreq_quick_get()
1698 	if (policy) {  in cpufreq_quick_get()
1699 		ret_freq = policy->cur;  in cpufreq_quick_get()
1700 		cpufreq_cpu_put(policy);  in cpufreq_quick_get()
1715 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_quick_get_max()  local
1718 	if (policy) {  in cpufreq_quick_get_max()
1719 		ret_freq = policy->max;  in cpufreq_quick_get_max()
1720 		cpufreq_cpu_put(policy);  in cpufreq_quick_get_max()
1727 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)  in __cpufreq_get()  argument
1729 	if (unlikely(policy_is_inactive(policy)))  in __cpufreq_get()
1732 	return cpufreq_verify_current_freq(policy, true);  in __cpufreq_get()
1743 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_get()  local
1746 	if (policy) {  in cpufreq_get()
1747 		down_read(&policy->rwsem);  in cpufreq_get()
1749 			ret_freq = __cpufreq_get(policy);  in cpufreq_get()
1750 		up_read(&policy->rwsem);  in cpufreq_get()
1752 		cpufreq_cpu_put(policy);  in cpufreq_get()
1770 int cpufreq_generic_suspend(struct cpufreq_policy *policy)  in cpufreq_generic_suspend()  argument
1774 	if (!policy->suspend_freq) {  in cpufreq_generic_suspend()
1780 			policy->suspend_freq);  in cpufreq_generic_suspend()
1782 	ret = __cpufreq_driver_target(policy, policy->suspend_freq,  in cpufreq_generic_suspend()
1786 				__func__, policy->suspend_freq, ret);  in cpufreq_generic_suspend()
1802 	struct cpufreq_policy *policy;  in cpufreq_suspend()  local
1812 	for_each_active_policy(policy) {  in cpufreq_suspend()
1814 			down_write(&policy->rwsem);  in cpufreq_suspend()
1815 			cpufreq_stop_governor(policy);  in cpufreq_suspend()
1816 			up_write(&policy->rwsem);  in cpufreq_suspend()
1819 		if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))  in cpufreq_suspend()
1836 	struct cpufreq_policy *policy;  in cpufreq_resume()  local
1852 	for_each_active_policy(policy) {  in cpufreq_resume()
1853 		if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {  in cpufreq_resume()
1855 				policy);  in cpufreq_resume()
1857 			down_write(&policy->rwsem);  in cpufreq_resume()
1858 			ret = cpufreq_start_governor(policy);  in cpufreq_resume()
1859 			up_write(&policy->rwsem);  in cpufreq_resume()
1863 				       __func__, policy);  in cpufreq_resume()
2017 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,  in cpufreq_driver_fast_switch()  argument
2020 	target_freq = clamp_val(target_freq, policy->min, policy->max);  in cpufreq_driver_fast_switch()
2022 	return cpufreq_driver->fast_switch(policy, target_freq);  in cpufreq_driver_fast_switch()
2027 static int __target_intermediate(struct cpufreq_policy *policy,  in __target_intermediate()  argument
2032 	freqs->new = cpufreq_driver->get_intermediate(policy, index);  in __target_intermediate()
2039 		 __func__, policy->cpu, freqs->old, freqs->new);  in __target_intermediate()
2041 	cpufreq_freq_transition_begin(policy, freqs);  in __target_intermediate()
2042 	ret = cpufreq_driver->target_intermediate(policy, index);  in __target_intermediate()
2043 	cpufreq_freq_transition_end(policy, freqs, ret);  in __target_intermediate()
2052 static int __target_index(struct cpufreq_policy *policy, int index)  in __target_index()  argument
2054 	struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};  in __target_index()
2056 	unsigned int newfreq = policy->freq_table[index].frequency;  in __target_index()
2060 	if (newfreq == policy->cur)  in __target_index()
2067 			retval = __target_intermediate(policy, &freqs, index);  in __target_index()
2079 			 __func__, policy->cpu, freqs.old, freqs.new);  in __target_index()
2081 		cpufreq_freq_transition_begin(policy, &freqs);  in __target_index()
2084 	retval = cpufreq_driver->target_index(policy, index);  in __target_index()
2090 		cpufreq_freq_transition_end(policy, &freqs, retval);  in __target_index()
2100 			freqs.new = policy->restore_freq;  in __target_index()
2101 			cpufreq_freq_transition_begin(policy, &freqs);  in __target_index()
2102 			cpufreq_freq_transition_end(policy, &freqs, 0);  in __target_index()
2109 int __cpufreq_driver_target(struct cpufreq_policy *policy,  in __cpufreq_driver_target()  argument
2120 	target_freq = clamp_val(target_freq, policy->min, policy->max);  in __cpufreq_driver_target()
2123 		 policy->cpu, target_freq, relation, old_target_freq);  in __cpufreq_driver_target()
2131 	if (target_freq == policy->cur)  in __cpufreq_driver_target()
2135 	policy->restore_freq = policy->cur;  in __cpufreq_driver_target()
2138 		return cpufreq_driver->target(policy, target_freq, relation);  in __cpufreq_driver_target()
2143 	index = cpufreq_frequency_table_target(policy, target_freq, relation);  in __cpufreq_driver_target()
2145 	return __target_index(policy, index);  in __cpufreq_driver_target()
2149 int cpufreq_driver_target(struct cpufreq_policy *policy,  in cpufreq_driver_target()  argument
2155 	down_write(&policy->rwsem);  in cpufreq_driver_target()
2157 	ret = __cpufreq_driver_target(policy, target_freq, relation);  in cpufreq_driver_target()
2159 	up_write(&policy->rwsem);  in cpufreq_driver_target()
2170 static int cpufreq_init_governor(struct cpufreq_policy *policy)  in cpufreq_init_governor()  argument
2181 	if (!policy->governor)  in cpufreq_init_governor()
2185 	if (policy->governor->dynamic_switching &&  in cpufreq_init_governor()
2191 				policy->governor->name, gov->name);  in cpufreq_init_governor()
2192 			policy->governor = gov;  in cpufreq_init_governor()
2198 	if (!try_module_get(policy->governor->owner))  in cpufreq_init_governor()
2201 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);  in cpufreq_init_governor()
2203 	if (policy->governor->init) {  in cpufreq_init_governor()
2204 		ret = policy->governor->init(policy);  in cpufreq_init_governor()
2206 			module_put(policy->governor->owner);  in cpufreq_init_governor()
2214 static void cpufreq_exit_governor(struct cpufreq_policy *policy)  in cpufreq_exit_governor()  argument
2216 	if (cpufreq_suspended || !policy->governor)  in cpufreq_exit_governor()
2219 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);  in cpufreq_exit_governor()
2221 	if (policy->governor->exit)  in cpufreq_exit_governor()
2222 		policy->governor->exit(policy);  in cpufreq_exit_governor()
2224 	module_put(policy->governor->owner);  in cpufreq_exit_governor()
2227 static int cpufreq_start_governor(struct cpufreq_policy *policy)  in cpufreq_start_governor()  argument
2234 	if (!policy->governor)  in cpufreq_start_governor()
2237 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);  in cpufreq_start_governor()
2240 		cpufreq_verify_current_freq(policy, false);  in cpufreq_start_governor()
2242 	if (policy->governor->start) {  in cpufreq_start_governor()
2243 		ret = policy->governor->start(policy);  in cpufreq_start_governor()
2248 	if (policy->governor->limits)  in cpufreq_start_governor()
2249 		policy->governor->limits(policy);  in cpufreq_start_governor()
2254 static void cpufreq_stop_governor(struct cpufreq_policy *policy)  in cpufreq_stop_governor()  argument
2256 	if (cpufreq_suspended || !policy->governor)  in cpufreq_stop_governor()
2259 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);  in cpufreq_stop_governor()
2261 	if (policy->governor->stop)  in cpufreq_stop_governor()
2262 		policy->governor->stop(policy);  in cpufreq_stop_governor()
2265 static void cpufreq_governor_limits(struct cpufreq_policy *policy)  in cpufreq_governor_limits()  argument
2267 	if (cpufreq_suspended || !policy->governor)  in cpufreq_governor_limits()
2270 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);  in cpufreq_governor_limits()
2272 	if (policy->governor->limits)  in cpufreq_governor_limits()
2273 		policy->governor->limits(policy);  in cpufreq_governor_limits()
2301 	struct cpufreq_policy *policy;  in cpufreq_unregister_governor()  local
2312 	for_each_inactive_policy(policy) {  in cpufreq_unregister_governor()
2313 		if (!strcmp(policy->last_governor, governor->name)) {  in cpufreq_unregister_governor()
2314 			policy->governor = NULL;  in cpufreq_unregister_governor()
2315 			strcpy(policy->last_governor, "\0");  in cpufreq_unregister_governor()
2338 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)  in cpufreq_get_policy()  argument
2341 	if (!policy)  in cpufreq_get_policy()
2348 	memcpy(policy, cpu_policy, sizeof(*policy));  in cpufreq_get_policy()
2370 int cpufreq_set_policy(struct cpufreq_policy *policy,  in cpufreq_set_policy()  argument
2379 	memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));  in cpufreq_set_policy()
2385 	new_policy->min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);  in cpufreq_set_policy()
2386 	new_policy->max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);  in cpufreq_set_policy()
2393 	policy->min = new_policy->min;  in cpufreq_set_policy()
2394 	policy->max = new_policy->max;  in cpufreq_set_policy()
2395 	trace_cpu_frequency_limits(policy);  in cpufreq_set_policy()
2397 	policy->cached_target_freq = UINT_MAX;  in cpufreq_set_policy()
2400 		 policy->min, policy->max);  in cpufreq_set_policy()
2403 		policy->policy = new_policy->policy;  in cpufreq_set_policy()
2405 		return cpufreq_driver->setpolicy(policy);  in cpufreq_set_policy()
2408 	if (new_policy->governor == policy->governor) {  in cpufreq_set_policy()
2410 		cpufreq_governor_limits(policy);  in cpufreq_set_policy()
2417 	old_gov = policy->governor;  in cpufreq_set_policy()
2420 		cpufreq_stop_governor(policy);  in cpufreq_set_policy()
2421 		cpufreq_exit_governor(policy);  in cpufreq_set_policy()
2425 	policy->governor = new_policy->governor;  in cpufreq_set_policy()
2426 	ret = cpufreq_init_governor(policy);  in cpufreq_set_policy()
2428 		ret = cpufreq_start_governor(policy);  in cpufreq_set_policy()
2431 			sched_cpufreq_governor_change(policy, old_gov);  in cpufreq_set_policy()
2434 		cpufreq_exit_governor(policy);  in cpufreq_set_policy()
2438 	pr_debug("starting governor %s failed\n", policy->governor->name);  in cpufreq_set_policy()
2440 		policy->governor = old_gov;  in cpufreq_set_policy()
2441 		if (cpufreq_init_governor(policy))  in cpufreq_set_policy()
2442 			policy->governor = NULL;  in cpufreq_set_policy()
2444 			cpufreq_start_governor(policy);  in cpufreq_set_policy()
2461 	struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);  in cpufreq_update_policy()  local
2463 	if (!policy)  in cpufreq_update_policy()
2471 	    (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))  in cpufreq_update_policy()
2474 	refresh_frequency_limits(policy);  in cpufreq_update_policy()
2477 	cpufreq_cpu_release(policy);  in cpufreq_update_policy()
2502 	struct cpufreq_policy *policy;  in cpufreq_boost_set_sw()  local
2505 	for_each_active_policy(policy) {  in cpufreq_boost_set_sw()
2506 		if (!policy->freq_table)  in cpufreq_boost_set_sw()
2509 		ret = cpufreq_frequency_table_cpuinfo(policy,  in cpufreq_boost_set_sw()
2510 						      policy->freq_table);  in cpufreq_boost_set_sw()
2517 		ret = freq_qos_update_request(policy->max_freq_req, policy->max);  in cpufreq_boost_set_sw()