17 static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx);
26 static bool kvm_pmu_idx_is_64bit(struct kvm_vcpu *vcpu, u64 select_idx)  in kvm_pmu_idx_is_64bit()  argument
29 		__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_LC);  in kvm_pmu_idx_is_64bit()
49 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_pmc_is_chained()  local
51 	return test_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);  in kvm_pmu_pmc_is_chained()
84 static bool kvm_pmu_idx_has_chain_evtype(struct kvm_vcpu *vcpu, u64 select_idx)  in kvm_pmu_idx_has_chain_evtype()  argument
94 	eventsel = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_EVENT;  in kvm_pmu_idx_has_chain_evtype()
104 static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu,  in kvm_pmu_get_pair_counter_value()  argument
113 		counter = __vcpu_sys_reg(vcpu, reg);  in kvm_pmu_get_pair_counter_value()
114 		counter_high = __vcpu_sys_reg(vcpu, reg + 1);  in kvm_pmu_get_pair_counter_value()
120 		counter = __vcpu_sys_reg(vcpu, reg);  in kvm_pmu_get_pair_counter_value()
139 u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)  in kvm_pmu_get_counter_value()  argument
142 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_get_counter_value()
145 	counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);  in kvm_pmu_get_counter_value()
162 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)  in kvm_pmu_set_counter_value()  argument
168 	__vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);  in kvm_pmu_set_counter_value()
171 	kvm_pmu_create_perf_event(vcpu, select_idx);  in kvm_pmu_set_counter_value()
194 static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)  in kvm_pmu_stop_counter()  argument
202 	counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);  in kvm_pmu_stop_counter()
212 	__vcpu_sys_reg(vcpu, reg) = val;  in kvm_pmu_stop_counter()
215 		__vcpu_sys_reg(vcpu, reg + 1) = upper_32_bits(counter);  in kvm_pmu_stop_counter()
225 void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu)  in kvm_pmu_vcpu_init()  argument
228 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_vcpu_init()
239 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)  in kvm_pmu_vcpu_reset()  argument
242 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_vcpu_reset()
245 		kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);  in kvm_pmu_vcpu_reset()
247 	bitmap_zero(vcpu->arch.pmu.chained, ARMV8_PMU_MAX_COUNTER_PAIRS);  in kvm_pmu_vcpu_reset()
255 void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_pmu_vcpu_destroy()  argument
258 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_vcpu_destroy()
264 u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)  in kvm_pmu_valid_counter_mask()  argument
266 	u64 val = __vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;  in kvm_pmu_valid_counter_mask()
282 void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)  in kvm_pmu_enable_counter_mask()  argument
285 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_enable_counter_mask()
288 	if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)  in kvm_pmu_enable_counter_mask()
303 			kvm_pmu_create_perf_event(vcpu, i);  in kvm_pmu_enable_counter_mask()
323 void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)  in kvm_pmu_disable_counter_mask()  argument
326 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_disable_counter_mask()
344 			kvm_pmu_create_perf_event(vcpu, i);  in kvm_pmu_disable_counter_mask()
354 static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)  in kvm_pmu_overflow_status()  argument
358 	if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) {  in kvm_pmu_overflow_status()
359 		reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);  in kvm_pmu_overflow_status()
360 		reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);  in kvm_pmu_overflow_status()
361 		reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1);  in kvm_pmu_overflow_status()
362 		reg &= kvm_pmu_valid_counter_mask(vcpu);  in kvm_pmu_overflow_status()
368 static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)  in kvm_pmu_update_state()  argument
370 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_update_state()
373 	if (!kvm_arm_pmu_v3_ready(vcpu))  in kvm_pmu_update_state()
376 	overflow = !!kvm_pmu_overflow_status(vcpu);  in kvm_pmu_update_state()
382 	if (likely(irqchip_in_kernel(vcpu->kvm))) {  in kvm_pmu_update_state()
383 		int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,  in kvm_pmu_update_state()
389 bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)  in kvm_pmu_should_notify_user()  argument
391 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_should_notify_user()
392 	struct kvm_sync_regs *sregs = &vcpu->run->s.regs;  in kvm_pmu_should_notify_user()
395 	if (likely(irqchip_in_kernel(vcpu->kvm)))  in kvm_pmu_should_notify_user()
404 void kvm_pmu_update_run(struct kvm_vcpu *vcpu)  in kvm_pmu_update_run()  argument
406 	struct kvm_sync_regs *regs = &vcpu->run->s.regs;  in kvm_pmu_update_run()
410 	if (vcpu->arch.pmu.irq_level)  in kvm_pmu_update_run()
421 void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)  in kvm_pmu_flush_hwstate()  argument
423 	kvm_pmu_update_state(vcpu);  in kvm_pmu_flush_hwstate()
433 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)  in kvm_pmu_sync_hwstate()  argument
435 	kvm_pmu_update_state(vcpu);  in kvm_pmu_sync_hwstate()
447 	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);  in kvm_pmu_perf_overflow()  local
459 	if (!kvm_pmu_idx_is_64bit(vcpu, pmc->idx))  in kvm_pmu_perf_overflow()
466 	__vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);  in kvm_pmu_perf_overflow()
468 	if (kvm_pmu_overflow_status(vcpu)) {  in kvm_pmu_perf_overflow()
469 		kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);  in kvm_pmu_perf_overflow()
470 		kvm_vcpu_kick(vcpu);  in kvm_pmu_perf_overflow()
481 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)  in kvm_pmu_software_increment()  argument
489 	enable = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);  in kvm_pmu_software_increment()
493 		type = __vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)  in kvm_pmu_software_increment()
497 			reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;  in kvm_pmu_software_increment()
499 			__vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;  in kvm_pmu_software_increment()
501 				__vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);  in kvm_pmu_software_increment()
511 void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)  in kvm_pmu_handle_pmcr()  argument
516 	mask = kvm_pmu_valid_counter_mask(vcpu);  in kvm_pmu_handle_pmcr()
518 		kvm_pmu_enable_counter_mask(vcpu,  in kvm_pmu_handle_pmcr()
519 		       __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);  in kvm_pmu_handle_pmcr()
521 		kvm_pmu_disable_counter_mask(vcpu, mask);  in kvm_pmu_handle_pmcr()
525 		kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);  in kvm_pmu_handle_pmcr()
529 			kvm_pmu_set_counter_value(vcpu, i, 0);  in kvm_pmu_handle_pmcr()
533 static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)  in kvm_pmu_counter_is_enabled()  argument
535 	return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&  in kvm_pmu_counter_is_enabled()
536 	       (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));  in kvm_pmu_counter_is_enabled()
544 static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx)  in kvm_pmu_create_perf_event()  argument
546 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_create_perf_event()
561 	data = __vcpu_sys_reg(vcpu, reg);  in kvm_pmu_create_perf_event()
563 	kvm_pmu_stop_counter(vcpu, pmc);  in kvm_pmu_create_perf_event()
575 	attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, pmc->idx);  in kvm_pmu_create_perf_event()
583 	counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);  in kvm_pmu_create_perf_event()
585 	if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {  in kvm_pmu_create_perf_event()
592 		if (kvm_pmu_counter_is_enabled(vcpu, pmc->idx + 1))  in kvm_pmu_create_perf_event()
600 		if (kvm_pmu_idx_is_64bit(vcpu, pmc->idx))  in kvm_pmu_create_perf_event()
626 static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx)  in kvm_pmu_update_pmc_chained()  argument
628 	struct kvm_pmu *pmu = &vcpu->arch.pmu;  in kvm_pmu_update_pmc_chained()
631 	if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {  in kvm_pmu_update_pmc_chained()
637 			kvm_pmu_stop_counter(vcpu, pmc);  in kvm_pmu_update_pmc_chained()
639 		set_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);  in kvm_pmu_update_pmc_chained()
641 		clear_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);  in kvm_pmu_update_pmc_chained()
655 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,  in kvm_pmu_set_counter_event_type()  argument
663 	__vcpu_sys_reg(vcpu, reg) = event_type;  in kvm_pmu_set_counter_event_type()
665 	kvm_pmu_update_pmc_chained(vcpu, select_idx);  in kvm_pmu_set_counter_event_type()
666 	kvm_pmu_create_perf_event(vcpu, select_idx);  in kvm_pmu_set_counter_event_type()
679 int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu)  in kvm_arm_pmu_v3_enable()  argument
681 	if (!vcpu->arch.pmu.created)  in kvm_arm_pmu_v3_enable()
689 	if (irqchip_in_kernel(vcpu->kvm)) {  in kvm_arm_pmu_v3_enable()
690 		int irq = vcpu->arch.pmu.irq_num;  in kvm_arm_pmu_v3_enable()
691 		if (!kvm_arm_pmu_irq_initialized(vcpu))  in kvm_arm_pmu_v3_enable()
700 		if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq))  in kvm_arm_pmu_v3_enable()
702 	} else if (kvm_arm_pmu_irq_initialized(vcpu)) {  in kvm_arm_pmu_v3_enable()
706 	kvm_pmu_vcpu_reset(vcpu);  in kvm_arm_pmu_v3_enable()
707 	vcpu->arch.pmu.ready = true;  in kvm_arm_pmu_v3_enable()
712 static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)  in kvm_arm_pmu_v3_init()  argument
717 	if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))  in kvm_arm_pmu_v3_init()
720 	if (vcpu->arch.pmu.created)  in kvm_arm_pmu_v3_init()
723 	if (irqchip_in_kernel(vcpu->kvm)) {  in kvm_arm_pmu_v3_init()
731 		if (!vgic_initialized(vcpu->kvm))  in kvm_arm_pmu_v3_init()
734 		if (!kvm_arm_pmu_irq_initialized(vcpu))  in kvm_arm_pmu_v3_init()
737 		ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num,  in kvm_arm_pmu_v3_init()
738 					 &vcpu->arch.pmu);  in kvm_arm_pmu_v3_init()
743 	vcpu->arch.pmu.created = true;  in kvm_arm_pmu_v3_init()
755 	struct kvm_vcpu *vcpu;  in pmu_irq_is_valid()  local
757 	kvm_for_each_vcpu(i, vcpu, kvm) {  in pmu_irq_is_valid()
758 		if (!kvm_arm_pmu_irq_initialized(vcpu))  in pmu_irq_is_valid()
762 			if (vcpu->arch.pmu.irq_num != irq)  in pmu_irq_is_valid()
765 			if (vcpu->arch.pmu.irq_num == irq)  in pmu_irq_is_valid()
773 int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)  in kvm_arm_pmu_v3_set_attr()  argument
780 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_arm_pmu_v3_set_attr()
783 		if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))  in kvm_arm_pmu_v3_set_attr()
793 		if (!pmu_irq_is_valid(vcpu->kvm, irq))  in kvm_arm_pmu_v3_set_attr()
796 		if (kvm_arm_pmu_irq_initialized(vcpu))  in kvm_arm_pmu_v3_set_attr()
800 		vcpu->arch.pmu.irq_num = irq;  in kvm_arm_pmu_v3_set_attr()
804 		return kvm_arm_pmu_v3_init(vcpu);  in kvm_arm_pmu_v3_set_attr()
810 int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)  in kvm_arm_pmu_v3_get_attr()  argument
817 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_arm_pmu_v3_get_attr()
820 		if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))  in kvm_arm_pmu_v3_get_attr()
823 		if (!kvm_arm_pmu_irq_initialized(vcpu))  in kvm_arm_pmu_v3_get_attr()
826 		irq = vcpu->arch.pmu.irq_num;  in kvm_arm_pmu_v3_get_attr()
834 int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)  in kvm_arm_pmu_v3_has_attr()  argument
840 		    test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))  in kvm_arm_pmu_v3_has_attr()