26 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc);
27 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc);
29 static struct kvm_vcpu *kvm_pmc_to_vcpu(const struct kvm_pmc *pmc)  in kvm_pmc_to_vcpu()  argument
31 	return container_of(pmc, struct kvm_vcpu, arch.pmu.pmc[pmc->idx]);  in kvm_pmc_to_vcpu()
36 	return &vcpu->arch.pmu.pmc[cnt_idx];  in kvm_vcpu_idx_to_pmc()
65  * @pmc: counter context
67 static bool kvm_pmc_is_64bit(struct kvm_pmc *pmc)  in kvm_pmc_is_64bit()  argument
69 	return (pmc->idx == ARMV8_PMU_CYCLE_IDX ||  in kvm_pmc_is_64bit()
70 		kvm_pmu_is_3p5(kvm_pmc_to_vcpu(pmc)));  in kvm_pmc_is_64bit()
73 static bool kvm_pmc_has_64bit_overflow(struct kvm_pmc *pmc)  in kvm_pmc_has_64bit_overflow()  argument
75 	u64 val = __vcpu_sys_reg(kvm_pmc_to_vcpu(pmc), PMCR_EL0);  in kvm_pmc_has_64bit_overflow()
77 	return (pmc->idx < ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LP)) ||  in kvm_pmc_has_64bit_overflow()
78 	       (pmc->idx == ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LC));  in kvm_pmc_has_64bit_overflow()
81 static bool kvm_pmu_counter_can_chain(struct kvm_pmc *pmc)  in kvm_pmu_counter_can_chain()  argument
83 	return (!(pmc->idx & 1) && (pmc->idx + 1) < ARMV8_PMU_CYCLE_IDX &&  in kvm_pmu_counter_can_chain()
84 		!kvm_pmc_has_64bit_overflow(pmc));  in kvm_pmu_counter_can_chain()
97 static u64 kvm_pmu_get_pmc_value(struct kvm_pmc *pmc)  in kvm_pmu_get_pmc_value()  argument
99 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_get_pmc_value()
102 	reg = counter_index_to_reg(pmc->idx);  in kvm_pmu_get_pmc_value()
109 	if (pmc->perf_event)  in kvm_pmu_get_pmc_value()
110 		counter += perf_event_read_value(pmc->perf_event, &enabled,  in kvm_pmu_get_pmc_value()
113 	if (!kvm_pmc_is_64bit(pmc))  in kvm_pmu_get_pmc_value()
132 static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force)  in kvm_pmu_set_pmc_value()  argument
134 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_set_pmc_value()
137 	kvm_pmu_release_perf_event(pmc);  in kvm_pmu_set_pmc_value()
139 	reg = counter_index_to_reg(pmc->idx);  in kvm_pmu_set_pmc_value()
141 	if (vcpu_mode_is_32bit(vcpu) && pmc->idx != ARMV8_PMU_CYCLE_IDX &&  in kvm_pmu_set_pmc_value()
156 	kvm_pmu_create_perf_event(pmc);  in kvm_pmu_set_pmc_value()
175  * @pmc: The PMU counter pointer
177 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)  in kvm_pmu_release_perf_event()  argument
179 	if (pmc->perf_event) {  in kvm_pmu_release_perf_event()
180 		perf_event_disable(pmc->perf_event);  in kvm_pmu_release_perf_event()
181 		perf_event_release_kernel(pmc->perf_event);  in kvm_pmu_release_perf_event()
182 		pmc->perf_event = NULL;  in kvm_pmu_release_perf_event()
188  * @pmc: The PMU counter pointer
192 static void kvm_pmu_stop_counter(struct kvm_pmc *pmc)  in kvm_pmu_stop_counter()  argument
194 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_stop_counter()
197 	if (!pmc->perf_event)  in kvm_pmu_stop_counter()
200 	val = kvm_pmu_get_pmc_value(pmc);  in kvm_pmu_stop_counter()
202 	reg = counter_index_to_reg(pmc->idx);  in kvm_pmu_stop_counter()
206 	kvm_pmu_release_perf_event(pmc);  in kvm_pmu_stop_counter()
220 		pmu->pmc[i].idx = i;  in kvm_pmu_vcpu_init()
279 		struct kvm_pmc *pmc;  in kvm_pmu_enable_counter_mask()  local
284 		pmc = kvm_vcpu_idx_to_pmc(vcpu, i);  in kvm_pmu_enable_counter_mask()
286 		if (!pmc->perf_event) {  in kvm_pmu_enable_counter_mask()
287 			kvm_pmu_create_perf_event(pmc);  in kvm_pmu_enable_counter_mask()
289 			perf_event_enable(pmc->perf_event);  in kvm_pmu_enable_counter_mask()
290 			if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)  in kvm_pmu_enable_counter_mask()
311 		struct kvm_pmc *pmc;  in kvm_pmu_disable_counter_mask()  local
316 		pmc = kvm_vcpu_idx_to_pmc(vcpu, i);  in kvm_pmu_disable_counter_mask()
318 		if (pmc->perf_event)  in kvm_pmu_disable_counter_mask()
319 			perf_event_disable(pmc->perf_event);  in kvm_pmu_disable_counter_mask()
436 		struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i);  in kvm_pmu_counter_increment()  local
447 		if (!kvm_pmc_is_64bit(pmc))  in kvm_pmu_counter_increment()
452 		if (kvm_pmc_has_64bit_overflow(pmc) ? reg : lower_32_bits(reg))  in kvm_pmu_counter_increment()
458 		if (kvm_pmu_counter_can_chain(pmc))  in kvm_pmu_counter_increment()
465 static u64 compute_period(struct kvm_pmc *pmc, u64 counter)  in compute_period()  argument
469 	if (kvm_pmc_is_64bit(pmc) && kvm_pmc_has_64bit_overflow(pmc))  in compute_period()
484 	struct kvm_pmc *pmc = perf_event->overflow_handler_context;  in kvm_pmu_perf_overflow()  local
486 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_perf_overflow()
487 	int idx = pmc->idx;  in kvm_pmu_perf_overflow()
496 	period = compute_period(pmc, local64_read(&perf_event->count));  in kvm_pmu_perf_overflow()
504 	if (kvm_pmu_counter_can_chain(pmc))  in kvm_pmu_perf_overflow()
569 static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc)  in kvm_pmu_counter_is_enabled()  argument
571 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_counter_is_enabled()
573 	       (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(pmc->idx));  in kvm_pmu_counter_is_enabled()
578  * @pmc: Counter context
580 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc)  in kvm_pmu_create_perf_event()  argument
582 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_create_perf_event()
588 	reg = counter_index_to_evtreg(pmc->idx);  in kvm_pmu_create_perf_event()
591 	kvm_pmu_stop_counter(pmc);  in kvm_pmu_create_perf_event()
592 	if (pmc->idx == ARMV8_PMU_CYCLE_IDX)  in kvm_pmu_create_perf_event()
617 	attr.disabled = !kvm_pmu_counter_is_enabled(pmc);  in kvm_pmu_create_perf_event()
629 	if (kvm_pmc_is_64bit(pmc))  in kvm_pmu_create_perf_event()
632 	attr.sample_period = compute_period(pmc, kvm_pmu_get_pmc_value(pmc));  in kvm_pmu_create_perf_event()
635 						 kvm_pmu_perf_overflow, pmc);  in kvm_pmu_create_perf_event()
643 	pmc->perf_event = event;  in kvm_pmu_create_perf_event()
652  * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
659 	struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, select_idx);  in kvm_pmu_set_counter_event_type()  local
669 	reg = counter_index_to_evtreg(pmc->idx);  in kvm_pmu_set_counter_event_type()
673 	kvm_pmu_create_perf_event(pmc);  in kvm_pmu_set_counter_event_type()