32 static struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)  in kvm_pmc_to_vcpu()  argument
37 	pmc -= pmc->idx;  in kvm_pmc_to_vcpu()
38 	pmu = container_of(pmc, struct kvm_pmu, pmc[0]);  in kvm_pmc_to_vcpu()
47 static bool kvm_pmu_pmc_is_chained(struct kvm_pmc *pmc)  in kvm_pmu_pmc_is_chained()  argument
49 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_pmc_is_chained()
51 	return test_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);  in kvm_pmu_pmc_is_chained()
70 static struct kvm_pmc *kvm_pmu_get_canonical_pmc(struct kvm_pmc *pmc)  in kvm_pmu_get_canonical_pmc()  argument
72 	if (kvm_pmu_pmc_is_chained(pmc) &&  in kvm_pmu_get_canonical_pmc()
73 	    kvm_pmu_idx_is_high_counter(pmc->idx))  in kvm_pmu_get_canonical_pmc()
74 		return pmc - 1;  in kvm_pmu_get_canonical_pmc()
76 	return pmc;  in kvm_pmu_get_canonical_pmc()
105 					  struct kvm_pmc *pmc)  in kvm_pmu_get_pair_counter_value()  argument
109 	if (kvm_pmu_pmc_is_chained(pmc)) {  in kvm_pmu_get_pair_counter_value()
110 		pmc = kvm_pmu_get_canonical_pmc(pmc);  in kvm_pmu_get_pair_counter_value()
111 		reg = PMEVCNTR0_EL0 + pmc->idx;  in kvm_pmu_get_pair_counter_value()
118 		reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)  in kvm_pmu_get_pair_counter_value()
119 		      ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;  in kvm_pmu_get_pair_counter_value()
127 	if (pmc->perf_event)  in kvm_pmu_get_pair_counter_value()
128 		counter += perf_event_read_value(pmc->perf_event, &enabled,  in kvm_pmu_get_pair_counter_value()
143 	struct kvm_pmc *pmc = &pmu->pmc[select_idx];  in kvm_pmu_get_counter_value()  local
145 	counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);  in kvm_pmu_get_counter_value()
147 	if (kvm_pmu_pmc_is_chained(pmc) &&  in kvm_pmu_get_counter_value()
178 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)  in kvm_pmu_release_perf_event()  argument
180 	pmc = kvm_pmu_get_canonical_pmc(pmc);  in kvm_pmu_release_perf_event()
181 	if (pmc->perf_event) {  in kvm_pmu_release_perf_event()
182 		perf_event_disable(pmc->perf_event);  in kvm_pmu_release_perf_event()
183 		perf_event_release_kernel(pmc->perf_event);  in kvm_pmu_release_perf_event()
184 		pmc->perf_event = NULL;  in kvm_pmu_release_perf_event()
194 static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)  in kvm_pmu_stop_counter()  argument
198 	pmc = kvm_pmu_get_canonical_pmc(pmc);  in kvm_pmu_stop_counter()
199 	if (!pmc->perf_event)  in kvm_pmu_stop_counter()
202 	counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);  in kvm_pmu_stop_counter()
204 	if (pmc->idx == ARMV8_PMU_CYCLE_IDX) {  in kvm_pmu_stop_counter()
208 		reg = PMEVCNTR0_EL0 + pmc->idx;  in kvm_pmu_stop_counter()
214 	if (kvm_pmu_pmc_is_chained(pmc))  in kvm_pmu_stop_counter()
217 	kvm_pmu_release_perf_event(pmc);  in kvm_pmu_stop_counter()
231 		pmu->pmc[i].idx = i;  in kvm_pmu_vcpu_init()
245 		kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);  in kvm_pmu_vcpu_reset()
261 		kvm_pmu_release_perf_event(&pmu->pmc[i]);  in kvm_pmu_vcpu_destroy()
286 	struct kvm_pmc *pmc;  in kvm_pmu_enable_counter_mask()  local
295 		pmc = &pmu->pmc[i];  in kvm_pmu_enable_counter_mask()
301 		if (kvm_pmu_pmc_is_chained(pmc) &&  in kvm_pmu_enable_counter_mask()
308 		if (pmc->perf_event) {  in kvm_pmu_enable_counter_mask()
309 			perf_event_enable(pmc->perf_event);  in kvm_pmu_enable_counter_mask()
310 			if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)  in kvm_pmu_enable_counter_mask()
327 	struct kvm_pmc *pmc;  in kvm_pmu_disable_counter_mask()  local
336 		pmc = &pmu->pmc[i];  in kvm_pmu_disable_counter_mask()
342 		if (kvm_pmu_pmc_is_chained(pmc) &&  in kvm_pmu_disable_counter_mask()
349 		if (pmc->perf_event)  in kvm_pmu_disable_counter_mask()
350 			perf_event_disable(pmc->perf_event);  in kvm_pmu_disable_counter_mask()
445 	struct kvm_pmc *pmc = perf_event->overflow_handler_context;  in kvm_pmu_perf_overflow()  local
447 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_perf_overflow()
448 	int idx = pmc->idx;  in kvm_pmu_perf_overflow()
459 	if (!kvm_pmu_idx_is_64bit(vcpu, pmc->idx))  in kvm_pmu_perf_overflow()
547 	struct kvm_pmc *pmc;  in kvm_pmu_create_perf_event()  local
557 	pmc = kvm_pmu_get_canonical_pmc(&pmu->pmc[select_idx]);  in kvm_pmu_create_perf_event()
559 	reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)  in kvm_pmu_create_perf_event()
560 	      ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + pmc->idx;  in kvm_pmu_create_perf_event()
563 	kvm_pmu_stop_counter(vcpu, pmc);  in kvm_pmu_create_perf_event()
568 	    pmc->idx != ARMV8_PMU_CYCLE_IDX)  in kvm_pmu_create_perf_event()
575 	attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, pmc->idx);  in kvm_pmu_create_perf_event()
580 	attr.config = (pmc->idx == ARMV8_PMU_CYCLE_IDX) ?  in kvm_pmu_create_perf_event()
583 	counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);  in kvm_pmu_create_perf_event()
585 	if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {  in kvm_pmu_create_perf_event()
592 		if (kvm_pmu_counter_is_enabled(vcpu, pmc->idx + 1))  in kvm_pmu_create_perf_event()
597 							 pmc + 1);  in kvm_pmu_create_perf_event()
600 		if (kvm_pmu_idx_is_64bit(vcpu, pmc->idx))  in kvm_pmu_create_perf_event()
606 						 kvm_pmu_perf_overflow, pmc);  in kvm_pmu_create_perf_event()
615 	pmc->perf_event = event;  in kvm_pmu_create_perf_event()
629 	struct kvm_pmc *pmc = &pmu->pmc[select_idx];  in kvm_pmu_update_pmc_chained()  local
631 	if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {  in kvm_pmu_update_pmc_chained()
636 		if (!kvm_pmu_pmc_is_chained(pmc))  in kvm_pmu_update_pmc_chained()
637 			kvm_pmu_stop_counter(vcpu, pmc);  in kvm_pmu_update_pmc_chained()
639 		set_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);  in kvm_pmu_update_pmc_chained()
641 		clear_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);  in kvm_pmu_update_pmc_chained()