228 struct cpudata {  struct
266 static struct cpudata **all_cpu_data;  argument
291 	u64 (*get_val)(struct cpudata*, int pstate);
292 	void (*get_vid)(struct cpudata *);
410 	struct cpudata *cpu;  in intel_pstate_init_acpi_perf_limits()
480 	struct cpudata *cpu;  in intel_pstate_exit_perf_limits()
521 static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu)  in intel_pstate_hybrid_hwp_adjust()
555 	struct cpudata *cpu;  in update_turbo_state()
566 	struct cpudata *cpu = all_cpu_data[0];  in min_perf_pct_min()
573 static s16 intel_pstate_get_epb(struct cpudata *cpu_data)  in intel_pstate_get_epb()
588 static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)  in intel_pstate_get_epp()
666 static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)  in intel_pstate_get_energy_pref_index()
704 static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)  in intel_pstate_set_epp()
730 static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,  in intel_pstate_set_energy_pref_index()
785 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in store_energy_performance_preference()
851 	struct cpudata *cpu_data = all_cpu_data[policy->cpu];  in show_energy_performance_preference()
868 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in show_base_frequency()
895 static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)  in __intel_pstate_get_hwp_cap()
905 static void intel_pstate_get_hwp_cap(struct cpudata *cpu)  in intel_pstate_get_hwp_cap()
925 	struct cpudata *cpu_data = all_cpu_data[cpu];  in intel_pstate_hwp_set()
986 static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata);
988 static void intel_pstate_hwp_offline(struct cpudata *cpu)  in intel_pstate_hwp_offline()
1055 static void intel_pstate_hwp_enable(struct cpudata *cpudata);
1057 static void intel_pstate_hwp_reenable(struct cpudata *cpu)  in intel_pstate_hwp_reenable()
1065 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_suspend()
1079 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_resume()
1111 static void __intel_pstate_update_max_freq(struct cpudata *cpudata,  in __intel_pstate_update_max_freq()  argument
1115 			cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;  in __intel_pstate_update_max_freq()
1191 	struct cpudata *cpu;  in show_turbo_pct()
1217 	struct cpudata *cpu;  in show_num_pstates()
1288 		struct cpudata *cpu = all_cpu_data[0];  in store_no_turbo()
1313 		struct cpudata *cpu = all_cpu_data[i];  in update_qos_request()
1573 	struct cpudata *cpudata =  in intel_pstate_notify_work()  local
1574 		container_of(to_delayed_work(work), struct cpudata, hwp_notify_work);  in intel_pstate_notify_work()
1575 	struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpudata->cpu);  in intel_pstate_notify_work()
1578 		intel_pstate_get_hwp_cap(cpudata);  in intel_pstate_notify_work()
1579 		__intel_pstate_update_max_freq(cpudata, policy);  in intel_pstate_notify_work()
1584 	wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);  in intel_pstate_notify_work()
1593 	struct cpudata *cpudata;  in notify_hwp_interrupt()  local
1622 	cpudata = READ_ONCE(all_cpu_data[this_cpu]);  in notify_hwp_interrupt()
1623 	if (unlikely(!cpudata))  in notify_hwp_interrupt()
1626 	schedule_delayed_work(&cpudata->hwp_notify_work, msecs_to_jiffies(10));  in notify_hwp_interrupt()
1637 static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata)  in intel_pstate_disable_hwp_interrupt()  argument
1645 	wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);  in intel_pstate_disable_hwp_interrupt()
1648 	if (cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask))  in intel_pstate_disable_hwp_interrupt()
1649 		cancel_delayed_work(&cpudata->hwp_notify_work);  in intel_pstate_disable_hwp_interrupt()
1653 static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata)  in intel_pstate_enable_hwp_interrupt()  argument
1660 		INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work);  in intel_pstate_enable_hwp_interrupt()
1661 		cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask);  in intel_pstate_enable_hwp_interrupt()
1665 		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x01);  in intel_pstate_enable_hwp_interrupt()
1666 		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);  in intel_pstate_enable_hwp_interrupt()
1670 static void intel_pstate_update_epp_defaults(struct cpudata *cpudata)  in intel_pstate_update_epp_defaults()  argument
1672 	cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);  in intel_pstate_update_epp_defaults()
1687 	if (cpudata->epp_default < HWP_EPP_BALANCE_PERFORMANCE &&  in intel_pstate_update_epp_defaults()
1688 	    cpudata->epp_default > HWP_EPP_PERFORMANCE) {  in intel_pstate_update_epp_defaults()
1689 		epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default;  in intel_pstate_update_epp_defaults()
1697 	cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE];  in intel_pstate_update_epp_defaults()
1698 	intel_pstate_set_epp(cpudata, cpudata->epp_default);  in intel_pstate_update_epp_defaults()
1701 static void intel_pstate_hwp_enable(struct cpudata *cpudata)  in intel_pstate_hwp_enable()  argument
1705 		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);  in intel_pstate_hwp_enable()
1707 	wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);  in intel_pstate_hwp_enable()
1709 	intel_pstate_enable_hwp_interrupt(cpudata);  in intel_pstate_hwp_enable()
1711 	if (cpudata->epp_default >= 0)  in intel_pstate_hwp_enable()
1714 	intel_pstate_update_epp_defaults(cpudata);  in intel_pstate_hwp_enable()
1741 static u64 atom_get_val(struct cpudata *cpudata, int pstate)  in atom_get_val()  argument
1751 	vid_fp = cpudata->vid.min + mul_fp(  in atom_get_val()
1752 		int_tofp(pstate - cpudata->pstate.min_pstate),  in atom_get_val()
1753 		cpudata->vid.ratio);  in atom_get_val()
1755 	vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);  in atom_get_val()
1758 	if (pstate > cpudata->pstate.max_pstate)  in atom_get_val()
1759 		vid = cpudata->vid.turbo;  in atom_get_val()
1795 static void atom_get_vid(struct cpudata *cpudata)  in atom_get_vid()  argument
1800 	cpudata->vid.min = int_tofp((value >> 8) & 0x7f);  in atom_get_vid()
1801 	cpudata->vid.max = int_tofp((value >> 16) & 0x7f);  in atom_get_vid()
1802 	cpudata->vid.ratio = div_fp(  in atom_get_vid()
1803 		cpudata->vid.max - cpudata->vid.min,  in atom_get_vid()
1804 		int_tofp(cpudata->pstate.max_pstate -  in atom_get_vid()
1805 			cpudata->pstate.min_pstate));  in atom_get_vid()
1808 	cpudata->vid.turbo = value & 0x7f;  in atom_get_vid()
1913 static u64 core_get_val(struct cpudata *cpudata, int pstate)  in core_get_val()  argument
1961 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)  in intel_pstate_set_pstate()
1974 static void intel_pstate_set_min_pstate(struct cpudata *cpu)  in intel_pstate_set_min_pstate()
1979 static void intel_pstate_max_within_limits(struct cpudata *cpu)  in intel_pstate_max_within_limits()
1987 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)  in intel_pstate_get_cpu_pstates()
2035 static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)  in intel_pstate_hwp_boost_up()
2082 static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)  in intel_pstate_hwp_boost_down()
2098 static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,  in intel_pstate_update_util_hwp_local()
2130 	struct cpudata *cpu = container_of(data, struct cpudata, update_util);  in intel_pstate_update_util_hwp()
2138 static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)  in intel_pstate_calc_avg_perf()
2145 static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)  in intel_pstate_sample()
2187 static inline int32_t get_avg_frequency(struct cpudata *cpu)  in get_avg_frequency()
2192 static inline int32_t get_avg_pstate(struct cpudata *cpu)  in get_avg_pstate()
2198 static inline int32_t get_target_pstate(struct cpudata *cpu)  in get_target_pstate()
2233 static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)  in intel_pstate_prepare_request()
2241 static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)  in intel_pstate_update_pstate()
2250 static void intel_pstate_adjust_pstate(struct cpudata *cpu)  in intel_pstate_adjust_pstate()
2278 	struct cpudata *cpu = container_of(data, struct cpudata, update_util);  in intel_pstate_update_util()
2407 	struct cpudata *cpu;  in intel_pstate_init_cpu()
2452 	struct cpudata *cpu = all_cpu_data[cpu_num];  in intel_pstate_set_update_util_hook()
2471 	struct cpudata *cpu_data = all_cpu_data[cpu];  in intel_pstate_clear_update_util_hook()
2481 static int intel_pstate_get_max_freq(struct cpudata *cpu)  in intel_pstate_get_max_freq()
2487 static void intel_pstate_update_perf_limits(struct cpudata *cpu,  in intel_pstate_update_perf_limits()
2553 	struct cpudata *cpu;  in intel_pstate_set_policy()
2595 static void intel_pstate_adjust_policy_max(struct cpudata *cpu,  in intel_pstate_adjust_policy_max()
2607 static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,  in intel_pstate_verify_cpu_policy()
2634 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_cpu_offline()
2659 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_cpu_online()
2695 	struct cpudata *cpu;  in __intel_pstate_cpu_init()
2738 		struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_cpu_init()
2762 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_verify_policy()
2786 static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)  in intel_cpufreq_trace()
2808 static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max,  in intel_cpufreq_hwp_update()
2832 static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu,  in intel_cpufreq_perf_ctl_update()
2846 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_update_pstate()
2872 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_target()
2907 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_fast_switch()
2924 	struct cpudata *cpu = all_cpu_data[cpunum];  in intel_cpufreq_adjust_perf()
2964 	struct cpudata *cpu;  in intel_cpufreq_cpu_init()
3053 		struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_suspend()
3378 	static struct cpudata **_all_cpu_data;  in intel_pstate_init()