1 // SPDX-License-Identifier: GPL-2.0-only
17 #include "vgic-mmio.h"
28 	return -1UL;  in vgic_mmio_read_rao()
53 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_group()
55 		if (irq->group)  in vgic_mmio_read_group()
58 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_group()
66 	WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group));  in vgic_update_vsgi()
77 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_group()
79 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_group()
80 		irq->group = !!(val & BIT(i));  in vgic_mmio_write_group()
81 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_group()
83 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_group()
85 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_group()
88 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_group()
105 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_enable()
107 		if (irq->enabled)  in vgic_mmio_read_enable()
110 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_enable()
125 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_senable()
127 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_senable()
128 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_senable()
129 			if (!irq->enabled) {  in vgic_mmio_write_senable()
130 				struct irq_data *data;  in vgic_mmio_write_senable()  local
132 				irq->enabled = true;  in vgic_mmio_write_senable()
133 				data = &irq_to_desc(irq->host_irq)->irq_data;  in vgic_mmio_write_senable()
134 				while (irqd_irq_disabled(data))  in vgic_mmio_write_senable()
135 					enable_irq(irq->host_irq);  in vgic_mmio_write_senable()
138 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_senable()
139 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_senable()
143 			bool was_high = irq->line_level;  in vgic_mmio_write_senable()
150 			irq->line_level = vgic_get_phys_line_level(irq);  in vgic_mmio_write_senable()
155 			if (!irq->active && was_high && !irq->line_level)  in vgic_mmio_write_senable()
158 		irq->enabled = true;  in vgic_mmio_write_senable()
159 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_senable()
161 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_senable()
174 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_cenable()
176 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_cenable()
177 		if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled)  in vgic_mmio_write_cenable()
178 			disable_irq_nosync(irq->host_irq);  in vgic_mmio_write_cenable()
180 		irq->enabled = false;  in vgic_mmio_write_cenable()
182 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cenable()
183 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cenable()
196 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_senable()
198 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_senable()
199 		irq->enabled = true;  in vgic_uaccess_write_senable()
200 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_uaccess_write_senable()
202 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_senable()
217 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_cenable()
219 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_cenable()
220 		irq->enabled = false;  in vgic_uaccess_write_cenable()
221 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_uaccess_write_cenable()
223 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_cenable()
239 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __read_pending()
249 		 * Refer to Documentation/virt/kvm/devices/arm-vgic-v3.rst  in __read_pending()
252 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in __read_pending()
253 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in __read_pending()
257 			err = irq_get_irqchip_state(irq->host_irq,  in __read_pending()
260 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in __read_pending()
264 			switch (vcpu->kvm->arch.vgic.vgic_model) {  in __read_pending()
267 					val = irq->pending_latch;  in __read_pending()
278 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in __read_pending()
280 		vgic_put_irq(vcpu->kvm, irq);  in __read_pending()
300 	return (vgic_irq_is_sgi(irq->intid) &&  in is_vgic_v2_sgi()
301 		vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2);  in is_vgic_v2_sgi()
313 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_spending()
317 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
321 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_spending()
323 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_spending()
326 			err = irq_set_irqchip_state(irq->host_irq,  in vgic_mmio_write_spending()
329 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in vgic_mmio_write_spending()
331 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_spending()
332 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
337 		irq->pending_latch = true;  in vgic_mmio_write_spending()
338 		if (irq->hw)  in vgic_mmio_write_spending()
341 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_spending()
342 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
355 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_spending()
357 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_spending()
358 		irq->pending_latch = true;  in vgic_uaccess_write_spending()
366 			irq->source |= BIT(vcpu->vcpu_id);  in vgic_uaccess_write_spending()
368 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_uaccess_write_spending()
370 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_spending()
376 /* Must be called with irq->irq_lock held */
379 	irq->pending_latch = false;  in vgic_hw_irq_cpending()
384 	 * CPENDR for HW interrupts, so we clear the active state on  in vgic_hw_irq_cpending()
385 	 * the physical side if the virtual interrupt is not active.  in vgic_hw_irq_cpending()
393 	if (!irq->active)  in vgic_hw_irq_cpending()
406 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_cpending()
410 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
414 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
416 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_cpending()
419 			err = irq_set_irqchip_state(irq->host_irq,  in vgic_mmio_write_cpending()
422 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in vgic_mmio_write_cpending()
424 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
425 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
430 		if (irq->hw)  in vgic_mmio_write_cpending()
433 			irq->pending_latch = false;  in vgic_mmio_write_cpending()
435 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
436 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
449 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_cpending()
451 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_cpending()
458 			irq->source = 0;  in vgic_uaccess_write_cpending()
460 		irq->pending_latch = false;  in vgic_uaccess_write_cpending()
462 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_uaccess_write_cpending()
464 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_cpending()
471  * If we are fiddling with an IRQ's active state, we have to make sure the IRQ
473  * active state can be overwritten when the VCPU's state is synced coming back
477  * non-owning CPU, we have to stop all the VCPUs because interrupts can be
484  * active state, which guarantees that the VCPU is not running.
488 	if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&  in vgic_access_active_prepare()
491 		kvm_arm_halt_guest(vcpu->kvm);  in vgic_access_active_prepare()
497 	if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&  in vgic_access_active_finish()
500 		kvm_arm_resume_guest(vcpu->kvm);  in vgic_access_active_finish()
512 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_read_active()
518 		if (irq->active)  in __vgic_mmio_read_active()
521 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_read_active()
533 	mutex_lock(&vcpu->kvm->arch.config_lock);  in vgic_mmio_read_active()
539 	mutex_unlock(&vcpu->kvm->arch.config_lock);  in vgic_mmio_read_active()
550 /* Must be called with irq->irq_lock held */
552 				      bool active, bool is_uaccess)  in vgic_hw_irq_change_active()  argument
557 	irq->active = active;  in vgic_hw_irq_change_active()
558 	vgic_irq_set_phys_active(irq, active);  in vgic_hw_irq_change_active()
562 				    bool active)  in vgic_mmio_change_active()  argument
567 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_change_active()
569 	if (irq->hw && !vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_change_active()
570 		vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu);  in vgic_mmio_change_active()
571 	} else if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_change_active()
573 		 * GICv4.1 VSGI feature doesn't track an active state,  in vgic_mmio_change_active()
577 		irq->active = false;  in vgic_mmio_change_active()
579 		u32 model = vcpu->kvm->arch.vgic.vgic_model;  in vgic_mmio_change_active()
582 		irq->active = active;  in vgic_mmio_change_active()
587 		 * the active state is stored somewhere, but at the same time  in vgic_mmio_change_active()
595 		active_source = (requester_vcpu) ? requester_vcpu->vcpu_id : 0;  in vgic_mmio_change_active()
598 		    active && vgic_irq_is_sgi(irq->intid))  in vgic_mmio_change_active()
599 			irq->active_source = active_source;  in vgic_mmio_change_active()
602 	if (irq->active)  in vgic_mmio_change_active()
603 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_change_active()
605 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_change_active()
616 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_write_cactive()
618 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_write_cactive()
628 	mutex_lock(&vcpu->kvm->arch.config_lock);  in vgic_mmio_write_cactive()
634 	mutex_unlock(&vcpu->kvm->arch.config_lock);  in vgic_mmio_write_cactive()
653 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_write_sactive()
655 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_write_sactive()
665 	mutex_lock(&vcpu->kvm->arch.config_lock);  in vgic_mmio_write_sactive()
671 	mutex_unlock(&vcpu->kvm->arch.config_lock);  in vgic_mmio_write_sactive()
690 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_priority()
692 		val |= (u64)irq->priority << (i * 8);  in vgic_mmio_read_priority()
694 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_priority()
703  * need to make this VCPU exit and re-evaluate the priorities, potentially
716 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_priority()
718 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_priority()
720 		irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);  in vgic_mmio_write_priority()
721 		if (irq->hw && vgic_irq_is_sgi(irq->intid))  in vgic_mmio_write_priority()
723 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_priority()
725 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_priority()
737 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_config()
739 		if (irq->config == VGIC_CONFIG_EDGE)  in vgic_mmio_read_config()
742 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_config()
763 		 * make them read-only here.  in vgic_mmio_write_config()
768 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_config()
769 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_config()
772 			irq->config = VGIC_CONFIG_EDGE;  in vgic_mmio_write_config()
774 			irq->config = VGIC_CONFIG_LEVEL;  in vgic_mmio_write_config()
776 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_config()
777 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_config()
785 	int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;  in vgic_read_irq_line_level_info()
793 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_read_irq_line_level_info()
794 		if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level)  in vgic_read_irq_line_level_info()
797 		vgic_put_irq(vcpu->kvm, irq);  in vgic_read_irq_line_level_info()
807 	int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;  in vgic_write_irq_line_level_info()
817 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_write_irq_line_level_info()
825 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_write_irq_line_level_info()
826 		irq->line_level = new_level;  in vgic_write_irq_line_level_info()
828 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_write_irq_line_level_info()
830 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_write_irq_line_level_info()
832 		vgic_put_irq(vcpu->kvm, irq);  in vgic_write_irq_line_level_info()
841 	if (offset < region->reg_offset)  in match_region()
842 		return -1;  in match_region()
844 	if (offset >= region->reg_offset + region->len)  in match_region()
880  * We convert this value to the CPUs native format to deal with it as a data
885 	unsigned long data = kvm_mmio_read_buf(val, len);  in vgic_data_mmio_bus_to_host()  local
889 		return data;  in vgic_data_mmio_bus_to_host()
891 		return le16_to_cpu(data);  in vgic_data_mmio_bus_to_host()
893 		return le32_to_cpu(data);  in vgic_data_mmio_bus_to_host()
895 		return le64_to_cpu(data);  in vgic_data_mmio_bus_to_host()
905  * We convert the data value from the CPUs native format to LE so that the
909 				unsigned long data)  in vgic_data_host_to_mmio_bus()  argument
915 		data = cpu_to_le16(data);  in vgic_data_host_to_mmio_bus()
918 		data = cpu_to_le32(data);  in vgic_data_host_to_mmio_bus()
921 		data = cpu_to_le64(data);  in vgic_data_host_to_mmio_bus()
924 	kvm_mmio_write_buf(buf, len, data);  in vgic_data_host_to_mmio_bus()
937 	int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;  in check_region()
953 	if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {  in check_region()
954 		if (!region->bits_per_irq)  in check_region()
957 		/* Do we access a non-allocated IRQ? */  in check_region()
958 		return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;  in check_region()
970 	region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,  in vgic_get_mmio_region()
971 				       addr - iodev->base_addr);  in vgic_get_mmio_region()
972 	if (!region || !check_region(vcpu->kvm, region, addr, len))  in vgic_get_mmio_region()
990 	r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;  in vgic_uaccess_read()
991 	if (region->uaccess_read)  in vgic_uaccess_read()
992 		*val = region->uaccess_read(r_vcpu, addr, sizeof(u32));  in vgic_uaccess_read()
994 		*val = region->read(r_vcpu, addr, sizeof(u32));  in vgic_uaccess_read()
1009 	r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;  in vgic_uaccess_write()
1010 	if (region->uaccess_write)  in vgic_uaccess_write()
1011 		return region->uaccess_write(r_vcpu, addr, sizeof(u32), *val);  in vgic_uaccess_write()
1013 	region->write(r_vcpu, addr, sizeof(u32), *val);  in vgic_uaccess_write()
1034 	unsigned long data = 0;  in dispatch_mmio_read()  local
1042 	switch (iodev->iodev_type) {  in dispatch_mmio_read()
1044 		data = region->read(vcpu, addr, len);  in dispatch_mmio_read()
1047 		data = region->read(vcpu, addr, len);  in dispatch_mmio_read()
1050 		data = region->read(iodev->redist_vcpu, addr, len);  in dispatch_mmio_read()
1053 		data = region->its_read(vcpu->kvm, iodev->its, addr, len);  in dispatch_mmio_read()
1057 	vgic_data_host_to_mmio_bus(val, len, data);  in dispatch_mmio_read()
1066 	unsigned long data = vgic_data_mmio_bus_to_host(val, len);  in dispatch_mmio_write()  local
1072 	switch (iodev->iodev_type) {  in dispatch_mmio_write()
1074 		region->write(vcpu, addr, len, data);  in dispatch_mmio_write()
1077 		region->write(vcpu, addr, len, data);  in dispatch_mmio_write()
1080 		region->write(iodev->redist_vcpu, addr, len, data);  in dispatch_mmio_write()
1083 		region->its_write(vcpu->kvm, iodev->its, addr, len, data);  in dispatch_mmio_write()
1098 	struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;  in vgic_register_dist_iodev()
1112 	io_device->base_addr = dist_base_address;  in vgic_register_dist_iodev()
1113 	io_device->iodev_type = IODEV_DIST;  in vgic_register_dist_iodev()
1114 	io_device->redist_vcpu = NULL;  in vgic_register_dist_iodev()
1117 				       len, &io_device->dev);  in vgic_register_dist_iodev()