1951 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_disable_all()  local
1955 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))  in __intel_pmu_disable_all()
1969 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_enable_all()  local
1974 			x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);  in __intel_pmu_enable_all()
1976 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {  in __intel_pmu_enable_all()
1978 			cpuc->events[INTEL_PMC_IDX_FIXED_BTS];  in __intel_pmu_enable_all()
2008 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_nhm_workaround()  local
2041 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
2055 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
2073 static void intel_set_tfa(struct cpu_hw_events *cpuc, bool on)  in intel_set_tfa()  argument
2077 	if (cpuc->tfa_shadow != val) {  in intel_set_tfa()
2078 		cpuc->tfa_shadow = val;  in intel_set_tfa()
2083 static void intel_tfa_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)  in intel_tfa_commit_scheduling()  argument
2089 		intel_set_tfa(cpuc, true);  in intel_tfa_commit_scheduling()
2094 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_tfa_pmu_enable_all()  local
2100 	if (!test_bit(3, cpuc->active_mask))  in intel_tfa_pmu_enable_all()
2101 		intel_set_tfa(cpuc, false);  in intel_tfa_pmu_enable_all()
2152 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_disable_event()  local
2160 	cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
2161 	cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
2162 	cpuc->intel_cp_status &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
2234 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_enable_event()  local
2245 		cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);  in intel_pmu_enable_event()
2247 		cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx);  in intel_pmu_enable_event()
2250 		cpuc->intel_cp_status |= (1ull << hwc->idx);  in intel_pmu_enable_event()
2333 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in handle_pmi_common()  local
2369 		status &= ~cpuc->pebs_enabled;  in handle_pmi_common()
2371 		status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);  in handle_pmi_common()
2399 	status |= cpuc->intel_cp_status;  in handle_pmi_common()
2402 		struct perf_event *event = cpuc->events[bit];  in handle_pmi_common()
2406 		if (!test_bit(bit, cpuc->active_mask))  in handle_pmi_common()
2415 			data.br_stack = &cpuc->lbr_stack;  in handle_pmi_common()
2446 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_handle_irq_v4()  local
2450 	int pmu_enabled = cpuc->enabled;  in intel_pmu_handle_irq_v4()
2454 	cpuc->enabled = 0;  in intel_pmu_handle_irq_v4()
2455 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {  in intel_pmu_handle_irq_v4()
2509 	cpuc->enabled = pmu_enabled;  in intel_pmu_handle_irq_v4()
2519 	struct cpu_hw_events *cpuc;  in intel_pmu_handle_irq()  local
2525 	cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_handle_irq()
2531 	pmu_enabled = cpuc->enabled;  in intel_pmu_handle_irq()
2539 	cpuc->enabled = 0;  in intel_pmu_handle_irq()
2574 	cpuc->enabled = pmu_enabled;  in intel_pmu_handle_irq()
2640 __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,  in __intel_shared_reg_get_constraints()  argument
2654 	if (reg->alloc && !cpuc->is_fake)  in __intel_shared_reg_get_constraints()
2658 	era = &cpuc->shared_regs->regs[idx];  in __intel_shared_reg_get_constraints()
2677 		if (!cpuc->is_fake) {  in __intel_shared_reg_get_constraints()
2715 __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,  in __intel_shared_reg_put_constraints()  argument
2728 	if (!reg->alloc || cpuc->is_fake)  in __intel_shared_reg_put_constraints()
2731 	era = &cpuc->shared_regs->regs[reg->idx];  in __intel_shared_reg_put_constraints()
2741 intel_shared_regs_constraints(struct cpu_hw_events *cpuc,  in intel_shared_regs_constraints()  argument
2749 		c = __intel_shared_reg_get_constraints(cpuc, event, xreg);  in intel_shared_regs_constraints()
2755 		d = __intel_shared_reg_get_constraints(cpuc, event, breg);  in intel_shared_regs_constraints()
2757 			__intel_shared_reg_put_constraints(cpuc, xreg);  in intel_shared_regs_constraints()
2765 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in x86_get_event_constraints()  argument
2783 __intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in __intel_get_event_constraints()  argument
2792 	c = intel_shared_regs_constraints(cpuc, event);  in __intel_get_event_constraints()
2800 	return x86_get_event_constraints(cpuc, idx, event);  in __intel_get_event_constraints()
2804 intel_start_scheduling(struct cpu_hw_events *cpuc)  in intel_start_scheduling()  argument
2806 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_start_scheduling()
2808 	int tid = cpuc->excl_thread_id;  in intel_start_scheduling()
2813 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_start_scheduling()
2833 static void intel_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)  in intel_commit_scheduling()  argument
2835 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_commit_scheduling()
2836 	struct event_constraint *c = cpuc->event_constraint[idx];  in intel_commit_scheduling()
2838 	int tid = cpuc->excl_thread_id;  in intel_commit_scheduling()
2840 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_commit_scheduling()
2860 intel_stop_scheduling(struct cpu_hw_events *cpuc)  in intel_stop_scheduling()  argument
2862 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_stop_scheduling()
2864 	int tid = cpuc->excl_thread_id;  in intel_stop_scheduling()
2869 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_stop_scheduling()
2887 dyn_constraint(struct cpu_hw_events *cpuc, struct event_constraint *c, int idx)  in dyn_constraint()  argument
2889 	WARN_ON_ONCE(!cpuc->constraint_list);  in dyn_constraint()
2897 		cx = &cpuc->constraint_list[idx];  in dyn_constraint()
2916 intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,  in intel_get_excl_constraints()  argument
2919 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_get_excl_constraints()
2921 	int tid = cpuc->excl_thread_id;  in intel_get_excl_constraints()
2928 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_get_excl_constraints()
2945 	c = dyn_constraint(cpuc, c, idx);  in intel_get_excl_constraints()
2966 		if (!cpuc->n_excl++)  in intel_get_excl_constraints()
3016 intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in intel_get_event_constraints()  argument
3021 	c1 = cpuc->event_constraint[idx];  in intel_get_event_constraints()
3028 	c2 = __intel_get_event_constraints(cpuc, idx, event);  in intel_get_event_constraints()
3036 	if (cpuc->excl_cntrs)  in intel_get_event_constraints()
3037 		return intel_get_excl_constraints(cpuc, event, idx, c2);  in intel_get_event_constraints()
3042 static void intel_put_excl_constraints(struct cpu_hw_events *cpuc,  in intel_put_excl_constraints()  argument
3046 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_put_excl_constraints()
3047 	int tid = cpuc->excl_thread_id;  in intel_put_excl_constraints()
3053 	if (cpuc->is_fake)  in intel_put_excl_constraints()
3061 		if (!--cpuc->n_excl)  in intel_put_excl_constraints()
3088 intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,  in intel_put_shared_regs_event_constraints()  argument
3095 		__intel_shared_reg_put_constraints(cpuc, reg);  in intel_put_shared_regs_event_constraints()
3099 		__intel_shared_reg_put_constraints(cpuc, reg);  in intel_put_shared_regs_event_constraints()
3102 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,  in intel_put_event_constraints()  argument
3105 	intel_put_shared_regs_event_constraints(cpuc, event);  in intel_put_event_constraints()
3112 	if (cpuc->excl_cntrs)  in intel_put_event_constraints()
3113 		intel_put_excl_constraints(cpuc, event);  in intel_put_event_constraints()
3337 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_guest_get_msrs()  local
3338 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;  in intel_guest_get_msrs()
3341 	arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;  in intel_guest_get_msrs()
3342 	arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;  in intel_guest_get_msrs()
3344 		arr[0].guest &= ~cpuc->pebs_enabled;  in intel_guest_get_msrs()
3346 		arr[0].guest &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);  in intel_guest_get_msrs()
3359 		arr[1].host = cpuc->pebs_enabled;  in intel_guest_get_msrs()
3369 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in core_guest_get_msrs()  local
3370 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;  in core_guest_get_msrs()
3374 		struct perf_event *event = cpuc->events[idx];  in core_guest_get_msrs()
3379 		if (!test_bit(idx, cpuc->active_mask))  in core_guest_get_msrs()
3403 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in core_pmu_enable_all()  local
3407 		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;  in core_pmu_enable_all()
3409 		if (!test_bit(idx, cpuc->active_mask) ||  in core_pmu_enable_all()
3410 				cpuc->events[idx]->attr.exclude_host)  in core_pmu_enable_all()
3467 hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in hsw_get_event_constraints()  argument
3472 	c = intel_get_event_constraints(cpuc, idx, event);  in hsw_get_event_constraints()
3485 icl_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in icl_get_event_constraints()  argument
3496 	return hsw_get_event_constraints(cpuc, idx, event);  in icl_get_event_constraints()
3500 glp_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in glp_get_event_constraints()  argument
3509 	c = intel_get_event_constraints(cpuc, idx, event);  in glp_get_event_constraints()
3515 tnt_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in tnt_get_event_constraints()  argument
3532 	c = intel_get_event_constraints(cpuc, idx, event);  in tnt_get_event_constraints()
3540 tfa_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in tfa_get_event_constraints()  argument
3543 	struct event_constraint *c = hsw_get_event_constraints(cpuc, idx, event);  in tfa_get_event_constraints()
3549 		c = dyn_constraint(cpuc, c, idx);  in tfa_get_event_constraints()
3648 int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)  in intel_cpuc_prepare()  argument
3650 	cpuc->pebs_record_size = x86_pmu.pebs_record_size;  in intel_cpuc_prepare()
3653 		cpuc->shared_regs = allocate_shared_regs(cpu);  in intel_cpuc_prepare()
3654 		if (!cpuc->shared_regs)  in intel_cpuc_prepare()
3661 		cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));  in intel_cpuc_prepare()
3662 		if (!cpuc->constraint_list)  in intel_cpuc_prepare()
3667 		cpuc->excl_cntrs = allocate_excl_cntrs(cpu);  in intel_cpuc_prepare()
3668 		if (!cpuc->excl_cntrs)  in intel_cpuc_prepare()
3671 		cpuc->excl_thread_id = 0;  in intel_cpuc_prepare()
3677 	kfree(cpuc->constraint_list);  in intel_cpuc_prepare()
3678 	cpuc->constraint_list = NULL;  in intel_cpuc_prepare()
3681 	kfree(cpuc->shared_regs);  in intel_cpuc_prepare()
3682 	cpuc->shared_regs = NULL;  in intel_cpuc_prepare()
3708 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in intel_pmu_cpu_starting()  local
3718 	cpuc->lbr_sel = NULL;  in intel_pmu_cpu_starting()
3721 		WARN_ON_ONCE(cpuc->tfa_shadow);  in intel_pmu_cpu_starting()
3722 		cpuc->tfa_shadow = ~0ULL;  in intel_pmu_cpu_starting()
3723 		intel_set_tfa(cpuc, false);  in intel_pmu_cpu_starting()
3732 	if (!cpuc->shared_regs)  in intel_pmu_cpu_starting()
3741 				cpuc->kfree_on_online[0] = cpuc->shared_regs;  in intel_pmu_cpu_starting()
3742 				cpuc->shared_regs = pc;  in intel_pmu_cpu_starting()
3746 		cpuc->shared_regs->core_id = core_id;  in intel_pmu_cpu_starting()
3747 		cpuc->shared_regs->refcnt++;  in intel_pmu_cpu_starting()
3751 		cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];  in intel_pmu_cpu_starting()
3761 				cpuc->kfree_on_online[1] = cpuc->excl_cntrs;  in intel_pmu_cpu_starting()
3762 				cpuc->excl_cntrs = c;  in intel_pmu_cpu_starting()
3764 					cpuc->excl_thread_id = 1;  in intel_pmu_cpu_starting()
3768 		cpuc->excl_cntrs->core_id = core_id;  in intel_pmu_cpu_starting()
3769 		cpuc->excl_cntrs->refcnt++;  in intel_pmu_cpu_starting()
3773 static void free_excl_cntrs(struct cpu_hw_events *cpuc)  in free_excl_cntrs()  argument
3777 	c = cpuc->excl_cntrs;  in free_excl_cntrs()
3781 		cpuc->excl_cntrs = NULL;  in free_excl_cntrs()
3784 	kfree(cpuc->constraint_list);  in free_excl_cntrs()
3785 	cpuc->constraint_list = NULL;  in free_excl_cntrs()
3796 void intel_cpuc_finish(struct cpu_hw_events *cpuc)  in intel_cpuc_finish()  argument
3800 	pc = cpuc->shared_regs;  in intel_cpuc_finish()
3804 		cpuc->shared_regs = NULL;  in intel_cpuc_finish()
3807 	free_excl_cntrs(cpuc);  in intel_cpuc_finish()
4342 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in update_tfa_sched()  local
4348 	if (test_bit(3, cpuc->active_mask))  in update_tfa_sched()