195 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)  in valid_vpe()  argument
197 	if (valid_col(its->collections + vpe->col_idx))  in valid_vpe()
198 		return vpe;  in valid_vpe()
256 			struct its_vpe *vpe;  member
260 			struct its_vpe *vpe;  member
266 			struct its_vpe *vpe;  member
274 			struct its_vpe *vpe;  member
281 			struct its_vpe *vpe;  member
571 	its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);  in its_build_vinvall_cmd()
575 	return valid_vpe(its, desc->its_vinvall_cmd.vpe);  in its_build_vinvall_cmd()
585 	vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));  in its_build_vmapp_cmd()
589 	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);  in its_build_vmapp_cmd()
597 	return valid_vpe(its, desc->its_vmapp_cmd.vpe);  in its_build_vmapp_cmd()
607 		db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;  in its_build_vmapti_cmd()
613 	its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);  in its_build_vmapti_cmd()
620 	return valid_vpe(its, desc->its_vmapti_cmd.vpe);  in its_build_vmapti_cmd()
630 		db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;  in its_build_vmovi_cmd()
636 	its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);  in its_build_vmovi_cmd()
643 	return valid_vpe(its, desc->its_vmovi_cmd.vpe);  in its_build_vmovi_cmd()
656 	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);  in its_build_vmovp_cmd()
661 	return valid_vpe(its, desc->its_vmovp_cmd.vpe);  in its_build_vmovp_cmd()
931 	desc.its_vmapti_cmd.vpe = map->vpe;  in its_send_vmapti()
945 	desc.its_vmovi_cmd.vpe = map->vpe;  in its_send_vmovi()
954 			   struct its_vpe *vpe, bool valid)  in its_send_vmapp()  argument
958 	desc.its_vmapp_cmd.vpe = vpe;  in its_send_vmapp()
960 	desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];  in its_send_vmapp()
965 static void its_send_vmovp(struct its_vpe *vpe)  in its_send_vmovp()  argument
970 	int col_id = vpe->col_idx;  in its_send_vmovp()
972 	desc.its_vmovp_cmd.vpe = vpe;  in its_send_vmovp()
1000 		if (!vpe->its_vm->vlpi_count[its->list_nr])  in its_send_vmovp()
1010 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)  in its_send_vinvall()  argument
1014 	desc.its_vinvall_cmd.vpe = vpe;  in its_send_vinvall()
1212 			struct its_vpe *vpe = vm->vpes[i];  in its_map_vm()  local
1213 			struct irq_data *d = irq_get_irq_data(vpe->irq);  in its_map_vm()
1216 			vpe->col_idx = cpumask_first(cpu_online_mask);  in its_map_vm()
1217 			its_send_vmapp(its, vpe, true);  in its_map_vm()
1218 			its_send_vinvall(its, vpe);  in its_map_vm()
1219 			irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));  in its_map_vm()
2485 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)  in its_vpe_db_proxy_unmap_locked()  argument
2488 	if (vpe->vpe_proxy_event == -1)  in its_vpe_db_proxy_unmap_locked()
2491 	its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);  in its_vpe_db_proxy_unmap_locked()
2492 	vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;  in its_vpe_db_proxy_unmap_locked()
2502 		vpe_proxy.next_victim = vpe->vpe_proxy_event;  in its_vpe_db_proxy_unmap_locked()
2504 	vpe->vpe_proxy_event = -1;  in its_vpe_db_proxy_unmap_locked()
2507 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)  in its_vpe_db_proxy_unmap()  argument
2513 		its_vpe_db_proxy_unmap_locked(vpe);  in its_vpe_db_proxy_unmap()
2518 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)  in its_vpe_db_proxy_map_locked()  argument
2521 	if (vpe->vpe_proxy_event != -1)  in its_vpe_db_proxy_map_locked()
2529 	vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;  in its_vpe_db_proxy_map_locked()
2530 	vpe->vpe_proxy_event = vpe_proxy.next_victim;  in its_vpe_db_proxy_map_locked()
2533 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;  in its_vpe_db_proxy_map_locked()
2534 	its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);  in its_vpe_db_proxy_map_locked()
2537 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)  in its_vpe_db_proxy_move()  argument
2546 		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);  in its_vpe_db_proxy_move()
2555 	its_vpe_db_proxy_map_locked(vpe);  in its_vpe_db_proxy_move()
2558 	its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);  in its_vpe_db_proxy_move()
2559 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;  in its_vpe_db_proxy_move()
2568 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);  in its_vpe_set_affinity()  local
2577 	if (vpe->col_idx != cpu) {  in its_vpe_set_affinity()
2578 		int from = vpe->col_idx;  in its_vpe_set_affinity()
2580 		vpe->col_idx = cpu;  in its_vpe_set_affinity()
2581 		its_send_vmovp(vpe);  in its_vpe_set_affinity()
2582 		its_vpe_db_proxy_move(vpe, from, cpu);  in its_vpe_set_affinity()
2590 static void its_vpe_schedule(struct its_vpe *vpe)  in its_vpe_schedule()  argument
2596 	val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &  in its_vpe_schedule()
2603 	val  = virt_to_phys(page_address(vpe->vpt_page)) &  in its_vpe_schedule()
2617 	val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;  in its_vpe_schedule()
2622 static void its_vpe_deschedule(struct its_vpe *vpe)  in its_vpe_deschedule()  argument
2646 		vpe->idai = false;  in its_vpe_deschedule()
2647 		vpe->pending_last = true;  in its_vpe_deschedule()
2649 		vpe->idai = !!(val & GICR_VPENDBASER_IDAI);  in its_vpe_deschedule()
2650 		vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);  in its_vpe_deschedule()
2654 static void its_vpe_invall(struct its_vpe *vpe)  in its_vpe_invall()  argument
2662 		if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])  in its_vpe_invall()
2669 		its_send_vinvall(its, vpe);  in its_vpe_invall()
2676 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);  in its_vpe_set_vcpu_affinity()  local
2681 		its_vpe_schedule(vpe);  in its_vpe_set_vcpu_affinity()
2685 		its_vpe_deschedule(vpe);  in its_vpe_set_vcpu_affinity()
2689 		its_vpe_invall(vpe);  in its_vpe_set_vcpu_affinity()
2697 static void its_vpe_send_cmd(struct its_vpe *vpe,  in its_vpe_send_cmd()  argument
2704 	its_vpe_db_proxy_map_locked(vpe);  in its_vpe_send_cmd()
2705 	cmd(vpe_proxy.dev, vpe->vpe_proxy_event);  in its_vpe_send_cmd()
2712 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);  in its_vpe_send_inv()  local
2717 		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;  in its_vpe_send_inv()
2718 		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR);  in its_vpe_send_inv()
2722 		its_vpe_send_cmd(vpe, its_send_inv);  in its_vpe_send_inv()
2749 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);  in its_vpe_set_irqchip_state()  local
2757 		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;  in its_vpe_set_irqchip_state()
2759 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);  in its_vpe_set_irqchip_state()
2761 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);  in its_vpe_set_irqchip_state()
2767 			its_vpe_send_cmd(vpe, its_send_int);  in its_vpe_set_irqchip_state()
2769 			its_vpe_send_cmd(vpe, its_send_clear);  in its_vpe_set_irqchip_state()
2795 static int its_vpe_init(struct its_vpe *vpe)  in its_vpe_init()  argument
2814 		its_free_pending_table(vpe->vpt_page);  in its_vpe_init()
2818 	vpe->vpe_id = vpe_id;  in its_vpe_init()
2819 	vpe->vpt_page = vpt_page;  in its_vpe_init()
2820 	vpe->vpe_proxy_event = -1;  in its_vpe_init()
2825 static void its_vpe_teardown(struct its_vpe *vpe)  in its_vpe_teardown()  argument
2827 	its_vpe_db_proxy_unmap(vpe);  in its_vpe_teardown()
2828 	its_vpe_id_free(vpe->vpe_id);  in its_vpe_teardown()
2829 	its_free_pending_table(vpe->vpt_page);  in its_vpe_teardown()
2844 		struct its_vpe *vpe = irq_data_get_irq_chip_data(data);  in its_vpe_irq_domain_free()  local
2846 		BUG_ON(vm != vpe->its_vm);  in its_vpe_irq_domain_free()
2849 		its_vpe_teardown(vpe);  in its_vpe_irq_domain_free()
2917 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);  in its_vpe_irq_domain_activate()  local
2925 	vpe->col_idx = cpumask_first(cpu_online_mask);  in its_vpe_irq_domain_activate()
2931 		its_send_vmapp(its, vpe, true);  in its_vpe_irq_domain_activate()
2932 		its_send_vinvall(its, vpe);  in its_vpe_irq_domain_activate()
2935 	irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));  in its_vpe_irq_domain_activate()
2943 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);  in its_vpe_irq_domain_deactivate()  local
2957 		its_send_vmapp(its, vpe, false);  in its_vpe_irq_domain_deactivate()