1 // SPDX-License-Identifier: GPL-2.0-only
17 #include "vgic-mmio.h"
28 	return -1UL;  in vgic_mmio_read_rao()
53 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_group()
55 		if (irq->group)  in vgic_mmio_read_group()
58 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_group()
66 	WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group));  in vgic_update_vsgi()
77 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_group()
79 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_group()
80 		irq->group = !!(val & BIT(i));  in vgic_mmio_write_group()
81 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_group()
83 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_group()
85 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_group()
88 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_group()
105 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_enable()
107 		if (irq->enabled)  in vgic_mmio_read_enable()
110 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_enable()
125 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_senable()
127 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_senable()
128 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_senable()
129 			if (!irq->enabled) {  in vgic_mmio_write_senable()
130 				struct irq_data *data;  in vgic_mmio_write_senable()  local
132 				irq->enabled = true;  in vgic_mmio_write_senable()
133 				data = &irq_to_desc(irq->host_irq)->irq_data;  in vgic_mmio_write_senable()
134 				while (irqd_irq_disabled(data))  in vgic_mmio_write_senable()
135 					enable_irq(irq->host_irq);  in vgic_mmio_write_senable()
138 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_senable()
139 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_senable()
143 			bool was_high = irq->line_level;  in vgic_mmio_write_senable()
150 			irq->line_level = vgic_get_phys_line_level(irq);  in vgic_mmio_write_senable()
155 			if (!irq->active && was_high && !irq->line_level)  in vgic_mmio_write_senable()
158 		irq->enabled = true;  in vgic_mmio_write_senable()
159 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_senable()
161 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_senable()
174 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_cenable()
176 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_cenable()
177 		if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled)  in vgic_mmio_write_cenable()
178 			disable_irq_nosync(irq->host_irq);  in vgic_mmio_write_cenable()
180 		irq->enabled = false;  in vgic_mmio_write_cenable()
182 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cenable()
183 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cenable()
196 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_senable()
198 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_senable()
199 		irq->enabled = true;  in vgic_uaccess_write_senable()
200 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_uaccess_write_senable()
202 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_senable()
217 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_cenable()
219 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_cenable()
220 		irq->enabled = false;  in vgic_uaccess_write_cenable()
221 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_uaccess_write_cenable()
223 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_cenable()
239 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __read_pending()
249 		 * Refer to Documentation/virt/kvm/devices/arm-vgic-v3.rst  in __read_pending()
252 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in __read_pending()
253 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in __read_pending()
257 			err = irq_get_irqchip_state(irq->host_irq,  in __read_pending()
260 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in __read_pending()
264 			switch (vcpu->kvm->arch.vgic.vgic_model) {  in __read_pending()
267 					val = irq->pending_latch;  in __read_pending()
278 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in __read_pending()
280 		vgic_put_irq(vcpu->kvm, irq);  in __read_pending()
300 	return (vgic_irq_is_sgi(irq->intid) &&  in is_vgic_v2_sgi()
301 		vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2);  in is_vgic_v2_sgi()
313 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_spending()
317 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
321 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_spending()
323 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_spending()
326 			err = irq_set_irqchip_state(irq->host_irq,  in vgic_mmio_write_spending()
329 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in vgic_mmio_write_spending()
331 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_spending()
332 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
337 		irq->pending_latch = true;  in vgic_mmio_write_spending()
338 		if (irq->hw)  in vgic_mmio_write_spending()
341 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_spending()
342 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
355 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_spending()
357 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_spending()
358 		irq->pending_latch = true;  in vgic_uaccess_write_spending()
366 			irq->source |= BIT(vcpu->vcpu_id);  in vgic_uaccess_write_spending()
368 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_uaccess_write_spending()
370 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_spending()
376 /* Must be called with irq->irq_lock held */
379 	irq->pending_latch = false;  in vgic_hw_irq_cpending()
384 	 * CPENDR for HW interrupts, so we clear the active state on  in vgic_hw_irq_cpending()
385 	 * the physical side if the virtual interrupt is not active.  in vgic_hw_irq_cpending()
393 	if (!irq->active)  in vgic_hw_irq_cpending()
406 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_cpending()
410 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
414 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
416 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_cpending()
419 			err = irq_set_irqchip_state(irq->host_irq,  in vgic_mmio_write_cpending()
422 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in vgic_mmio_write_cpending()
424 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
425 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
430 		if (irq->hw)  in vgic_mmio_write_cpending()
433 			irq->pending_latch = false;  in vgic_mmio_write_cpending()
435 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
436 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
449 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_cpending()
451 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_cpending()
458 			irq->source = 0;  in vgic_uaccess_write_cpending()
460 		irq->pending_latch = false;  in vgic_uaccess_write_cpending()
462 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_uaccess_write_cpending()
464 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_cpending()
471  * If we are fiddling with an IRQ's active state, we have to make sure the IRQ
473  * active state can be overwritten when the VCPU's state is synced coming back
483  * active state, which guarantees that the VCPU is not running.
487 	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 ||  in vgic_access_active_prepare()
489 		kvm_arm_halt_guest(vcpu->kvm);  in vgic_access_active_prepare()
495 	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 ||  in vgic_access_active_finish()
497 		kvm_arm_resume_guest(vcpu->kvm);  in vgic_access_active_finish()
509 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_read_active()
515 		if (irq->active)  in __vgic_mmio_read_active()
518 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_read_active()
530 	mutex_lock(&vcpu->kvm->lock);  in vgic_mmio_read_active()
536 	mutex_unlock(&vcpu->kvm->lock);  in vgic_mmio_read_active()
547 /* Must be called with irq->irq_lock held */
549 				      bool active, bool is_uaccess)  in vgic_hw_irq_change_active()  argument
554 	irq->active = active;  in vgic_hw_irq_change_active()
555 	vgic_irq_set_phys_active(irq, active);  in vgic_hw_irq_change_active()
559 				    bool active)  in vgic_mmio_change_active()  argument
564 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_change_active()
566 	if (irq->hw && !vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_change_active()
567 		vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu);  in vgic_mmio_change_active()
568 	} else if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_change_active()
570 		 * GICv4.1 VSGI feature doesn't track an active state,  in vgic_mmio_change_active()
574 		irq->active = false;  in vgic_mmio_change_active()
576 		u32 model = vcpu->kvm->arch.vgic.vgic_model;  in vgic_mmio_change_active()
579 		irq->active = active;  in vgic_mmio_change_active()
584 		 * the active state is stored somewhere, but at the same time  in vgic_mmio_change_active()
592 		active_source = (requester_vcpu) ? requester_vcpu->vcpu_id : 0;  in vgic_mmio_change_active()
595 		    active && vgic_irq_is_sgi(irq->intid))  in vgic_mmio_change_active()
596 			irq->active_source = active_source;  in vgic_mmio_change_active()
599 	if (irq->active)  in vgic_mmio_change_active()
600 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_change_active()
602 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_change_active()
613 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_write_cactive()
615 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_write_cactive()
625 	mutex_lock(&vcpu->kvm->lock);  in vgic_mmio_write_cactive()
631 	mutex_unlock(&vcpu->kvm->lock);  in vgic_mmio_write_cactive()
650 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_write_sactive()
652 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_write_sactive()
662 	mutex_lock(&vcpu->kvm->lock);  in vgic_mmio_write_sactive()
668 	mutex_unlock(&vcpu->kvm->lock);  in vgic_mmio_write_sactive()
687 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_priority()
689 		val |= (u64)irq->priority << (i * 8);  in vgic_mmio_read_priority()
691 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_priority()
700  * need to make this VCPU exit and re-evaluate the priorities, potentially
713 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_priority()
715 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_priority()
717 		irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);  in vgic_mmio_write_priority()
718 		if (irq->hw && vgic_irq_is_sgi(irq->intid))  in vgic_mmio_write_priority()
720 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_priority()
722 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_priority()
734 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_config()
736 		if (irq->config == VGIC_CONFIG_EDGE)  in vgic_mmio_read_config()
739 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_config()
760 		 * make them read-only here.  in vgic_mmio_write_config()
765 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_config()
766 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_config()
769 			irq->config = VGIC_CONFIG_EDGE;  in vgic_mmio_write_config()
771 			irq->config = VGIC_CONFIG_LEVEL;  in vgic_mmio_write_config()
773 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_config()
774 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_config()
782 	int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;  in vgic_read_irq_line_level_info()
790 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_read_irq_line_level_info()
791 		if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level)  in vgic_read_irq_line_level_info()
794 		vgic_put_irq(vcpu->kvm, irq);  in vgic_read_irq_line_level_info()
804 	int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;  in vgic_write_irq_line_level_info()
814 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_write_irq_line_level_info()
822 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_write_irq_line_level_info()
823 		irq->line_level = new_level;  in vgic_write_irq_line_level_info()
825 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_write_irq_line_level_info()
827 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_write_irq_line_level_info()
829 		vgic_put_irq(vcpu->kvm, irq);  in vgic_write_irq_line_level_info()
838 	if (offset < region->reg_offset)  in match_region()
839 		return -1;  in match_region()
841 	if (offset >= region->reg_offset + region->len)  in match_region()
877  * We convert this value to the CPUs native format to deal with it as a data
882 	unsigned long data = kvm_mmio_read_buf(val, len);  in vgic_data_mmio_bus_to_host()  local
886 		return data;  in vgic_data_mmio_bus_to_host()
888 		return le16_to_cpu(data);  in vgic_data_mmio_bus_to_host()
890 		return le32_to_cpu(data);  in vgic_data_mmio_bus_to_host()
892 		return le64_to_cpu(data);  in vgic_data_mmio_bus_to_host()
902  * We convert the data value from the CPUs native format to LE so that the
906 				unsigned long data)  in vgic_data_host_to_mmio_bus()  argument
912 		data = cpu_to_le16(data);  in vgic_data_host_to_mmio_bus()
915 		data = cpu_to_le32(data);  in vgic_data_host_to_mmio_bus()
918 		data = cpu_to_le64(data);  in vgic_data_host_to_mmio_bus()
921 	kvm_mmio_write_buf(buf, len, data);  in vgic_data_host_to_mmio_bus()
934 	int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;  in check_region()
950 	if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {  in check_region()
951 		if (!region->bits_per_irq)  in check_region()
954 		/* Do we access a non-allocated IRQ? */  in check_region()
955 		return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;  in check_region()
967 	region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,  in vgic_get_mmio_region()
968 				       addr - iodev->base_addr);  in vgic_get_mmio_region()
969 	if (!region || !check_region(vcpu->kvm, region, addr, len))  in vgic_get_mmio_region()
987 	r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;  in vgic_uaccess_read()
988 	if (region->uaccess_read)  in vgic_uaccess_read()
989 		*val = region->uaccess_read(r_vcpu, addr, sizeof(u32));  in vgic_uaccess_read()
991 		*val = region->read(r_vcpu, addr, sizeof(u32));  in vgic_uaccess_read()
1006 	r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;  in vgic_uaccess_write()
1007 	if (region->uaccess_write)  in vgic_uaccess_write()
1008 		return region->uaccess_write(r_vcpu, addr, sizeof(u32), *val);  in vgic_uaccess_write()
1010 	region->write(r_vcpu, addr, sizeof(u32), *val);  in vgic_uaccess_write()
1031 	unsigned long data = 0;  in dispatch_mmio_read()  local
1039 	switch (iodev->iodev_type) {  in dispatch_mmio_read()
1041 		data = region->read(vcpu, addr, len);  in dispatch_mmio_read()
1044 		data = region->read(vcpu, addr, len);  in dispatch_mmio_read()
1047 		data = region->read(iodev->redist_vcpu, addr, len);  in dispatch_mmio_read()
1050 		data = region->its_read(vcpu->kvm, iodev->its, addr, len);  in dispatch_mmio_read()
1054 	vgic_data_host_to_mmio_bus(val, len, data);  in dispatch_mmio_read()
1063 	unsigned long data = vgic_data_mmio_bus_to_host(val, len);  in dispatch_mmio_write()  local
1069 	switch (iodev->iodev_type) {  in dispatch_mmio_write()
1071 		region->write(vcpu, addr, len, data);  in dispatch_mmio_write()
1074 		region->write(vcpu, addr, len, data);  in dispatch_mmio_write()
1077 		region->write(iodev->redist_vcpu, addr, len, data);  in dispatch_mmio_write()
1080 		region->its_write(vcpu->kvm, iodev->its, addr, len, data);  in dispatch_mmio_write()
1095 	struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;  in vgic_register_dist_iodev()
1110 	io_device->base_addr = dist_base_address;  in vgic_register_dist_iodev()
1111 	io_device->iodev_type = IODEV_DIST;  in vgic_register_dist_iodev()
1112 	io_device->redist_vcpu = NULL;  in vgic_register_dist_iodev()
1114 	mutex_lock(&kvm->slots_lock);  in vgic_register_dist_iodev()
1116 				      len, &io_device->dev);  in vgic_register_dist_iodev()
1117 	mutex_unlock(&kvm->slots_lock);  in vgic_register_dist_iodev()