1 // SPDX-License-Identifier: GPL-2.0-only
3 #include <linux/irqchip/arm-gic-v3.h>
20 	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;  in vgic_v3_set_underflow()
22 	cpuif->vgic_hcr |= ICH_HCR_UIE;  in vgic_v3_set_underflow()
33 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;  in vgic_v3_fold_lr_state()
34 	struct vgic_v3_cpu_if *cpuif = &vgic_cpu->vgic_v3;  in vgic_v3_fold_lr_state()
35 	u32 model = vcpu->kvm->arch.vgic.vgic_model;  in vgic_v3_fold_lr_state()
40 	cpuif->vgic_hcr &= ~ICH_HCR_UIE;  in vgic_v3_fold_lr_state()
42 	for (lr = 0; lr < cpuif->used_lrs; lr++) {  in vgic_v3_fold_lr_state()
43 		u64 val = cpuif->vgic_lr[lr];  in vgic_v3_fold_lr_state()
58 		/* Notify fds when the guest EOI'ed a level-triggered IRQ */  in vgic_v3_fold_lr_state()
59 		if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))  in vgic_v3_fold_lr_state()
60 			kvm_notify_acked_irq(vcpu->kvm, 0,  in vgic_v3_fold_lr_state()
61 					     intid - VGIC_NR_PRIVATE_IRQS);  in vgic_v3_fold_lr_state()
63 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid);  in vgic_v3_fold_lr_state()
67 		raw_spin_lock(&irq->irq_lock);  in vgic_v3_fold_lr_state()
70 		irq->active = !!(val & ICH_LR_ACTIVE_BIT);  in vgic_v3_fold_lr_state()
72 		if (irq->active && is_v2_sgi)  in vgic_v3_fold_lr_state()
73 			irq->active_source = cpuid;  in vgic_v3_fold_lr_state()
76 		if (irq->config == VGIC_CONFIG_EDGE &&  in vgic_v3_fold_lr_state()
78 			irq->pending_latch = true;  in vgic_v3_fold_lr_state()
81 				irq->source |= (1 << cpuid);  in vgic_v3_fold_lr_state()
87 		if (irq->config == VGIC_CONFIG_LEVEL && !(val & ICH_LR_STATE))  in vgic_v3_fold_lr_state()
88 			irq->pending_latch = false;  in vgic_v3_fold_lr_state()
91 		 * Level-triggered mapped IRQs are special because we only  in vgic_v3_fold_lr_state()
104 			irq->line_level = vgic_get_phys_line_level(irq);  in vgic_v3_fold_lr_state()
106 			if (!irq->line_level)  in vgic_v3_fold_lr_state()
110 		raw_spin_unlock(&irq->irq_lock);  in vgic_v3_fold_lr_state()
111 		vgic_put_irq(vcpu->kvm, irq);  in vgic_v3_fold_lr_state()
114 	cpuif->used_lrs = 0;  in vgic_v3_fold_lr_state()
120 	u32 model = vcpu->kvm->arch.vgic.vgic_model;  in vgic_v3_populate_lr()
121 	u64 val = irq->intid;  in vgic_v3_populate_lr()
124 	is_v2_sgi = (vgic_irq_is_sgi(irq->intid) &&  in vgic_v3_populate_lr()
127 	if (irq->active) {  in vgic_v3_populate_lr()
130 			val |= irq->active_source << GICH_LR_PHYSID_CPUID_SHIFT;  in vgic_v3_populate_lr()
137 	if (irq->hw) {  in vgic_v3_populate_lr()
139 		val |= ((u64)irq->hwintid) << ICH_LR_PHYS_ID_SHIFT;  in vgic_v3_populate_lr()
145 		if (irq->active)  in vgic_v3_populate_lr()
148 		if (irq->config == VGIC_CONFIG_LEVEL) {  in vgic_v3_populate_lr()
155 			if (irq->active)  in vgic_v3_populate_lr()
163 		if (irq->config == VGIC_CONFIG_EDGE)  in vgic_v3_populate_lr()
164 			irq->pending_latch = false;  in vgic_v3_populate_lr()
166 		if (vgic_irq_is_sgi(irq->intid) &&  in vgic_v3_populate_lr()
168 			u32 src = ffs(irq->source);  in vgic_v3_populate_lr()
171 					   irq->intid))  in vgic_v3_populate_lr()
174 			val |= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT;  in vgic_v3_populate_lr()
175 			irq->source &= ~(1 << (src - 1));  in vgic_v3_populate_lr()
176 			if (irq->source) {  in vgic_v3_populate_lr()
177 				irq->pending_latch = true;  in vgic_v3_populate_lr()
184 	 * Level-triggered mapped IRQs are special because we only observe  in vgic_v3_populate_lr()
190 		irq->line_level = false;  in vgic_v3_populate_lr()
192 	if (irq->group)  in vgic_v3_populate_lr()
195 	val |= (u64)irq->priority << ICH_LR_PRIORITY_SHIFT;  in vgic_v3_populate_lr()
197 	vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = val;  in vgic_v3_populate_lr()
202 	vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = 0;  in vgic_v3_clear_lr()
207 	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;  in vgic_v3_set_vmcr()
208 	u32 model = vcpu->kvm->arch.vgic.vgic_model;  in vgic_v3_set_vmcr()
212 		vmcr = (vmcrp->ackctl << ICH_VMCR_ACK_CTL_SHIFT) &  in vgic_v3_set_vmcr()
214 		vmcr |= (vmcrp->fiqen << ICH_VMCR_FIQ_EN_SHIFT) &  in vgic_v3_set_vmcr()
224 	vmcr |= (vmcrp->cbpr << ICH_VMCR_CBPR_SHIFT) & ICH_VMCR_CBPR_MASK;  in vgic_v3_set_vmcr()
225 	vmcr |= (vmcrp->eoim << ICH_VMCR_EOIM_SHIFT) & ICH_VMCR_EOIM_MASK;  in vgic_v3_set_vmcr()
226 	vmcr |= (vmcrp->abpr << ICH_VMCR_BPR1_SHIFT) & ICH_VMCR_BPR1_MASK;  in vgic_v3_set_vmcr()
227 	vmcr |= (vmcrp->bpr << ICH_VMCR_BPR0_SHIFT) & ICH_VMCR_BPR0_MASK;  in vgic_v3_set_vmcr()
228 	vmcr |= (vmcrp->pmr << ICH_VMCR_PMR_SHIFT) & ICH_VMCR_PMR_MASK;  in vgic_v3_set_vmcr()
229 	vmcr |= (vmcrp->grpen0 << ICH_VMCR_ENG0_SHIFT) & ICH_VMCR_ENG0_MASK;  in vgic_v3_set_vmcr()
230 	vmcr |= (vmcrp->grpen1 << ICH_VMCR_ENG1_SHIFT) & ICH_VMCR_ENG1_MASK;  in vgic_v3_set_vmcr()
232 	cpu_if->vgic_vmcr = vmcr;  in vgic_v3_set_vmcr()
237 	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;  in vgic_v3_get_vmcr()
238 	u32 model = vcpu->kvm->arch.vgic.vgic_model;  in vgic_v3_get_vmcr()
241 	vmcr = cpu_if->vgic_vmcr;  in vgic_v3_get_vmcr()
244 		vmcrp->ackctl = (vmcr & ICH_VMCR_ACK_CTL_MASK) >>  in vgic_v3_get_vmcr()
246 		vmcrp->fiqen = (vmcr & ICH_VMCR_FIQ_EN_MASK) >>  in vgic_v3_get_vmcr()
253 		vmcrp->fiqen = 1;  in vgic_v3_get_vmcr()
254 		vmcrp->ackctl = 0;  in vgic_v3_get_vmcr()
257 	vmcrp->cbpr = (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;  in vgic_v3_get_vmcr()
258 	vmcrp->eoim = (vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT;  in vgic_v3_get_vmcr()
259 	vmcrp->abpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;  in vgic_v3_get_vmcr()
260 	vmcrp->bpr  = (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;  in vgic_v3_get_vmcr()
261 	vmcrp->pmr  = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;  in vgic_v3_get_vmcr()
262 	vmcrp->grpen0 = (vmcr & ICH_VMCR_ENG0_MASK) >> ICH_VMCR_ENG0_SHIFT;  in vgic_v3_get_vmcr()
263 	vmcrp->grpen1 = (vmcr & ICH_VMCR_ENG1_MASK) >> ICH_VMCR_ENG1_SHIFT;  in vgic_v3_get_vmcr()
273 	struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;  in vgic_v3_enable()
280 	vgic_v3->vgic_vmcr = 0;  in vgic_v3_enable()
283 	 * If we are emulating a GICv3, we do it in an non-GICv2-compatible  in vgic_v3_enable()
288 	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {  in vgic_v3_enable()
289 		vgic_v3->vgic_sre = (ICC_SRE_EL1_DIB |  in vgic_v3_enable()
292 		vcpu->arch.vgic_cpu.pendbaser = INITIAL_PENDBASER_VALUE;  in vgic_v3_enable()
294 		vgic_v3->vgic_sre = 0;  in vgic_v3_enable()
297 	vcpu->arch.vgic_cpu.num_id_bits = (kvm_vgic_global_state.ich_vtr_el2 &  in vgic_v3_enable()
300 	vcpu->arch.vgic_cpu.num_pri_bits = ((kvm_vgic_global_state.ich_vtr_el2 &  in vgic_v3_enable()
305 	vgic_v3->vgic_hcr = ICH_HCR_EN;  in vgic_v3_enable()
307 		vgic_v3->vgic_hcr |= ICH_HCR_TALL0;  in vgic_v3_enable()
309 		vgic_v3->vgic_hcr |= ICH_HCR_TALL1;  in vgic_v3_enable()
311 		vgic_v3->vgic_hcr |= ICH_HCR_TC;  in vgic_v3_enable()
325 	vcpu = irq->target_vcpu;  in vgic_v3_lpi_sync_pending_status()
329 	pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);  in vgic_v3_lpi_sync_pending_status()
331 	byte_offset = irq->intid / BITS_PER_BYTE;  in vgic_v3_lpi_sync_pending_status()
332 	bit_nr = irq->intid % BITS_PER_BYTE;  in vgic_v3_lpi_sync_pending_status()
341 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_v3_lpi_sync_pending_status()
342 	if (irq->target_vcpu != vcpu) {  in vgic_v3_lpi_sync_pending_status()
343 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_v3_lpi_sync_pending_status()
346 	irq->pending_latch = status;  in vgic_v3_lpi_sync_pending_status()
347 	vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_v3_lpi_sync_pending_status()
360  * vgic_v3_save_pending_tables - Save the pending tables into guest RAM
365 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_v3_save_pending_tables()
371 	list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {  in vgic_v3_save_pending_tables()
377 		vcpu = irq->target_vcpu;  in vgic_v3_save_pending_tables()
381 		pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);  in vgic_v3_save_pending_tables()
383 		byte_offset = irq->intid / BITS_PER_BYTE;  in vgic_v3_save_pending_tables()
384 		bit_nr = irq->intid % BITS_PER_BYTE;  in vgic_v3_save_pending_tables()
395 		if (stored == irq->pending_latch)  in vgic_v3_save_pending_tables()
398 		if (irq->pending_latch)  in vgic_v3_save_pending_tables()
411  * vgic_v3_rdist_overlap - check if a region overlaps with any
412  * existing redistributor region
422 	struct vgic_dist *d = &kvm->arch.vgic;  in vgic_v3_rdist_overlap()
425 	list_for_each_entry(rdreg, &d->rd_regions, list) {  in vgic_v3_rdist_overlap()
426 		if ((base + size > rdreg->base) &&  in vgic_v3_rdist_overlap()
427 			(base < rdreg->base + vgic_v3_rd_region_size(kvm, rdreg)))  in vgic_v3_rdist_overlap()
439 	struct vgic_dist *d = &kvm->arch.vgic;  in vgic_v3_check_base()
442 	if (!IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) &&  in vgic_v3_check_base()
443 	    d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE < d->vgic_dist_base)  in vgic_v3_check_base()
446 	list_for_each_entry(rdreg, &d->rd_regions, list) {  in vgic_v3_check_base()
447 		if (rdreg->base + vgic_v3_rd_region_size(kvm, rdreg) <  in vgic_v3_check_base()
448 			rdreg->base)  in vgic_v3_check_base()
452 	if (IS_VGIC_ADDR_UNDEF(d->vgic_dist_base))  in vgic_v3_check_base()
455 	return !vgic_v3_rdist_overlap(kvm, d->vgic_dist_base,  in vgic_v3_check_base()
460  * vgic_v3_rdist_free_slot - Look up registered rdist regions and identify one
463  * @rd_regions: redistributor region list head
465  * A redistributor regions maps n redistributors, n = region size / (2 x 64kB).
466  * Stride between redistributors is 0 and regions are filled in the index order.
485 	struct list_head *rd_regions = &kvm->arch.vgic.rd_regions;  in vgic_v3_rdist_region_from_index()
489 		if (rdreg->index == index)  in vgic_v3_rdist_region_from_index()
498 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_v3_map_resources()
507 		struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;  in vgic_v3_map_resources()
509 		if (IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr)) {  in vgic_v3_map_resources()
510 			kvm_debug("vcpu %d redistributor base not set\n", c);  in vgic_v3_map_resources()
511 			ret = -ENXIO;  in vgic_v3_map_resources()
516 	if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base)) {  in vgic_v3_map_resources()
518 		ret = -ENXIO;  in vgic_v3_map_resources()
524 		ret = -EINVAL;  in vgic_v3_map_resources()
533 		ret = -EBUSY;  in vgic_v3_map_resources()
537 	ret = vgic_register_dist_iodev(kvm, dist->vgic_dist_base, VGIC_V3);  in vgic_v3_map_resources()
545 	dist->ready = true;  in vgic_v3_map_resources()
557 early_param("kvm-arm.vgic_v3_group0_trap", early_group0_trap_cfg);
563 early_param("kvm-arm.vgic_v3_group1_trap", early_group1_trap_cfg);
569 early_param("kvm-arm.vgic_v3_common_trap", early_common_trap_cfg);
575 early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable);
578  * vgic_v3_probe - probe for a VGICv3 compatible interrupt controller
598 	if (info->has_v4) {  in vgic_v3_probe()
600 		kvm_vgic_global_state.has_gicv4_1 = info->has_v4_1 && gicv4_enable;  in vgic_v3_probe()
606 	if (!info->vcpu.start) {  in vgic_v3_probe()
609 	} else if (!PAGE_ALIGNED(info->vcpu.start)) {  in vgic_v3_probe()
611 			(unsigned long long)info->vcpu.start);  in vgic_v3_probe()
614 		kvm_vgic_global_state.vcpu_base = info->vcpu.start;  in vgic_v3_probe()
621 		kvm_info("vgic-v2@%llx\n", info->vcpu.start);  in vgic_v3_probe()
655 	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;  in vgic_v3_load()
662 	if (likely(cpu_if->vgic_sre))  in vgic_v3_load()
663 		kvm_call_hyp(__vgic_v3_write_vmcr, cpu_if->vgic_vmcr);  in vgic_v3_load()
675 	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;  in vgic_v3_vmcr_sync()
677 	if (likely(cpu_if->vgic_sre))  in vgic_v3_vmcr_sync()
678 		cpu_if->vgic_vmcr = kvm_call_hyp_ret(__vgic_v3_read_vmcr);  in vgic_v3_vmcr_sync()
683 	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;  in vgic_v3_put()