228 struct cpudata {  struct
266 static struct cpudata **all_cpu_data;  argument
291 	u64 (*get_val)(struct cpudata*, int pstate);
292 	void (*get_vid)(struct cpudata *);
438 	struct cpudata *cpu;  in intel_pstate_init_acpi_perf_limits()
494 	struct cpudata *cpu;  in intel_pstate_exit_perf_limits()
540 static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu)  in intel_pstate_hybrid_hwp_adjust()
574 	struct cpudata *cpu;  in update_turbo_state()
585 	struct cpudata *cpu = all_cpu_data[0];  in min_perf_pct_min()
592 static s16 intel_pstate_get_epb(struct cpudata *cpu_data)  in intel_pstate_get_epb()
607 static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)  in intel_pstate_get_epp()
685 static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)  in intel_pstate_get_energy_pref_index()
723 static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)  in intel_pstate_set_epp()
749 static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,  in intel_pstate_set_energy_pref_index()
804 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in store_energy_performance_preference()
872 	struct cpudata *cpu_data = all_cpu_data[policy->cpu];  in show_energy_performance_preference()
889 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in show_base_frequency()
916 static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)  in __intel_pstate_get_hwp_cap()
926 static void intel_pstate_get_hwp_cap(struct cpudata *cpu)  in intel_pstate_get_hwp_cap()
946 	struct cpudata *cpu_data = all_cpu_data[cpu];  in intel_pstate_hwp_set()
1007 static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata);
1009 static void intel_pstate_hwp_offline(struct cpudata *cpu)  in intel_pstate_hwp_offline()
1076 static void intel_pstate_hwp_enable(struct cpudata *cpudata);
1078 static void intel_pstate_hwp_reenable(struct cpudata *cpu)  in intel_pstate_hwp_reenable()
1086 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_suspend()
1100 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_resume()
1132 static void __intel_pstate_update_max_freq(struct cpudata *cpudata,  in __intel_pstate_update_max_freq()  argument
1136 			cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;  in __intel_pstate_update_max_freq()
1212 	struct cpudata *cpu;  in show_turbo_pct()
1238 	struct cpudata *cpu;  in show_num_pstates()
1309 		struct cpudata *cpu = all_cpu_data[0];  in store_no_turbo()
1334 		struct cpudata *cpu = all_cpu_data[i];  in update_qos_request()
1597 	struct cpudata *cpudata =  in intel_pstate_notify_work()  local
1598 		container_of(to_delayed_work(work), struct cpudata, hwp_notify_work);  in intel_pstate_notify_work()
1599 	struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpudata->cpu);  in intel_pstate_notify_work()
1602 		intel_pstate_get_hwp_cap(cpudata);  in intel_pstate_notify_work()
1603 		__intel_pstate_update_max_freq(cpudata, policy);  in intel_pstate_notify_work()
1608 	wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);  in intel_pstate_notify_work()
1617 	struct cpudata *cpudata;  in notify_hwp_interrupt()  local
1646 	cpudata = READ_ONCE(all_cpu_data[this_cpu]);  in notify_hwp_interrupt()
1647 	if (unlikely(!cpudata))  in notify_hwp_interrupt()
1650 	schedule_delayed_work(&cpudata->hwp_notify_work, msecs_to_jiffies(10));  in notify_hwp_interrupt()
1661 static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata)  in intel_pstate_disable_hwp_interrupt()  argument
1669 	wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);  in intel_pstate_disable_hwp_interrupt()
1672 	if (cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask))  in intel_pstate_disable_hwp_interrupt()
1673 		cancel_delayed_work(&cpudata->hwp_notify_work);  in intel_pstate_disable_hwp_interrupt()
1677 static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata)  in intel_pstate_enable_hwp_interrupt()  argument
1684 		INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work);  in intel_pstate_enable_hwp_interrupt()
1685 		cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask);  in intel_pstate_enable_hwp_interrupt()
1689 		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x01);  in intel_pstate_enable_hwp_interrupt()
1690 		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);  in intel_pstate_enable_hwp_interrupt()
1694 static void intel_pstate_update_epp_defaults(struct cpudata *cpudata)  in intel_pstate_update_epp_defaults()  argument
1696 	cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);  in intel_pstate_update_epp_defaults()
1711 	if (hwp_forced && cpudata->epp_default <= HWP_EPP_BALANCE_PERFORMANCE &&  in intel_pstate_update_epp_defaults()
1712 	    cpudata->epp_default > HWP_EPP_PERFORMANCE) {  in intel_pstate_update_epp_defaults()
1713 		epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default;  in intel_pstate_update_epp_defaults()
1721 	cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE];  in intel_pstate_update_epp_defaults()
1722 	intel_pstate_set_epp(cpudata, cpudata->epp_default);  in intel_pstate_update_epp_defaults()
1725 static void intel_pstate_hwp_enable(struct cpudata *cpudata)  in intel_pstate_hwp_enable()  argument
1729 		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);  in intel_pstate_hwp_enable()
1731 	wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);  in intel_pstate_hwp_enable()
1733 	intel_pstate_enable_hwp_interrupt(cpudata);  in intel_pstate_hwp_enable()
1735 	if (cpudata->epp_default >= 0)  in intel_pstate_hwp_enable()
1738 	intel_pstate_update_epp_defaults(cpudata);  in intel_pstate_hwp_enable()
1765 static u64 atom_get_val(struct cpudata *cpudata, int pstate)  in atom_get_val()  argument
1775 	vid_fp = cpudata->vid.min + mul_fp(  in atom_get_val()
1776 		int_tofp(pstate - cpudata->pstate.min_pstate),  in atom_get_val()
1777 		cpudata->vid.ratio);  in atom_get_val()
1779 	vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);  in atom_get_val()
1782 	if (pstate > cpudata->pstate.max_pstate)  in atom_get_val()
1783 		vid = cpudata->vid.turbo;  in atom_get_val()
1819 static void atom_get_vid(struct cpudata *cpudata)  in atom_get_vid()  argument
1824 	cpudata->vid.min = int_tofp((value >> 8) & 0x7f);  in atom_get_vid()
1825 	cpudata->vid.max = int_tofp((value >> 16) & 0x7f);  in atom_get_vid()
1826 	cpudata->vid.ratio = div_fp(  in atom_get_vid()
1827 		cpudata->vid.max - cpudata->vid.min,  in atom_get_vid()
1828 		int_tofp(cpudata->pstate.max_pstate -  in atom_get_vid()
1829 			cpudata->pstate.min_pstate));  in atom_get_vid()
1832 	cpudata->vid.turbo = value & 0x7f;  in atom_get_vid()
1932 static u64 core_get_val(struct cpudata *cpudata, int pstate)  in core_get_val()  argument
1992 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)  in intel_pstate_set_pstate()
2005 static void intel_pstate_set_min_pstate(struct cpudata *cpu)  in intel_pstate_set_min_pstate()
2010 static void intel_pstate_max_within_limits(struct cpudata *cpu)  in intel_pstate_max_within_limits()
2018 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)  in intel_pstate_get_cpu_pstates()
2066 static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)  in intel_pstate_hwp_boost_up()
2113 static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)  in intel_pstate_hwp_boost_down()
2129 static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,  in intel_pstate_update_util_hwp_local()
2161 	struct cpudata *cpu = container_of(data, struct cpudata, update_util);  in intel_pstate_update_util_hwp()
2169 static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)  in intel_pstate_calc_avg_perf()
2176 static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)  in intel_pstate_sample()
2218 static inline int32_t get_avg_frequency(struct cpudata *cpu)  in get_avg_frequency()
2223 static inline int32_t get_avg_pstate(struct cpudata *cpu)  in get_avg_pstate()
2229 static inline int32_t get_target_pstate(struct cpudata *cpu)  in get_target_pstate()
2264 static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)  in intel_pstate_prepare_request()
2272 static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)  in intel_pstate_update_pstate()
2281 static void intel_pstate_adjust_pstate(struct cpudata *cpu)  in intel_pstate_adjust_pstate()
2309 	struct cpudata *cpu = container_of(data, struct cpudata, update_util);  in intel_pstate_update_util()
2433 	struct cpudata *cpu;  in intel_pstate_init_cpu()
2475 	struct cpudata *cpu = all_cpu_data[cpu_num];  in intel_pstate_set_update_util_hook()
2494 	struct cpudata *cpu_data = all_cpu_data[cpu];  in intel_pstate_clear_update_util_hook()
2504 static int intel_pstate_get_max_freq(struct cpudata *cpu)  in intel_pstate_get_max_freq()
2510 static void intel_pstate_update_perf_limits(struct cpudata *cpu,  in intel_pstate_update_perf_limits()
2576 	struct cpudata *cpu;  in intel_pstate_set_policy()
2623 static void intel_pstate_adjust_policy_max(struct cpudata *cpu,  in intel_pstate_adjust_policy_max()
2635 static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,  in intel_pstate_verify_cpu_policy()
2662 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_cpu_offline()
2687 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_cpu_online()
2723 	struct cpudata *cpu;  in __intel_pstate_cpu_init()
2766 		struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_pstate_cpu_init()
2790 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_verify_policy()
2814 static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)  in intel_cpufreq_trace()
2836 static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max,  in intel_cpufreq_hwp_update()
2860 static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu,  in intel_cpufreq_perf_ctl_update()
2874 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_update_pstate()
2900 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_target()
2935 	struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_fast_switch()
2952 	struct cpudata *cpu = all_cpu_data[cpunum];  in intel_cpufreq_adjust_perf()
2992 	struct cpudata *cpu;  in intel_cpufreq_cpu_init()
3081 		struct cpudata *cpu = all_cpu_data[policy->cpu];  in intel_cpufreq_suspend()
3407 	static struct cpudata **_all_cpu_data;  in intel_pstate_init()