201 static notrace u8 xive_esb_read(struct xive_irq_data *xd, u32 offset)  in xive_esb_read()  argument
206 	if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)  in xive_esb_read()
209 	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)  in xive_esb_read()
210 		val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0);  in xive_esb_read()
212 		val = in_be64(xd->eoi_mmio + offset);  in xive_esb_read()
217 static void xive_esb_write(struct xive_irq_data *xd, u32 offset, u64 data)  in xive_esb_write()  argument
220 	if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)  in xive_esb_write()
223 	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)  in xive_esb_write()
224 		xive_ops->esb_rw(xd->hw_irq, offset, data, 1);  in xive_esb_write()
226 		out_be64(xd->eoi_mmio + offset, data);  in xive_esb_write()
283 		struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xmon_xive_get_irq_config()  local
284 		u64 val = xive_esb_read(xd, XIVE_ESB_GET);  in xmon_xive_get_irq_config()
354 static void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd)  in xive_do_source_eoi()  argument
356 	xd->stale_p = false;  in xive_do_source_eoi()
358 	if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)  in xive_do_source_eoi()
359 		xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0);  in xive_do_source_eoi()
360 	else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) {  in xive_do_source_eoi()
387 		if (xd->flags & XIVE_IRQ_FLAG_LSI)  in xive_do_source_eoi()
388 			xive_esb_read(xd, XIVE_ESB_LOAD_EOI);  in xive_do_source_eoi()
390 			eoi_val = xive_esb_read(xd, XIVE_ESB_SET_PQ_00);  in xive_do_source_eoi()
394 			if ((eoi_val & XIVE_ESB_VAL_Q) && xd->trig_mmio)  in xive_do_source_eoi()
395 				out_be64(xd->trig_mmio, 0);  in xive_do_source_eoi()
403 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_eoi()  local
414 	    !(xd->flags & XIVE_IRQ_NO_EOI))  in xive_irq_eoi()
415 		xive_do_source_eoi(irqd_to_hwirq(d), xd);  in xive_irq_eoi()
417 		xd->stale_p = true;  in xive_irq_eoi()
423 	xd->saved_p = false;  in xive_irq_eoi()
434 static void xive_do_source_set_mask(struct xive_irq_data *xd,  in xive_do_source_set_mask()  argument
448 		val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01);  in xive_do_source_set_mask()
449 		if (!xd->stale_p && !!(val & XIVE_ESB_VAL_P))  in xive_do_source_set_mask()
450 			xd->saved_p = true;  in xive_do_source_set_mask()
451 		xd->stale_p = false;  in xive_do_source_set_mask()
452 	} else if (xd->saved_p) {  in xive_do_source_set_mask()
453 		xive_esb_read(xd, XIVE_ESB_SET_PQ_10);  in xive_do_source_set_mask()
454 		xd->saved_p = false;  in xive_do_source_set_mask()
456 		xive_esb_read(xd, XIVE_ESB_SET_PQ_00);  in xive_do_source_set_mask()
457 		xd->stale_p = false;  in xive_do_source_set_mask()
561 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_pick_irq_target()  local
569 	if (xd->src_chip != XIVE_INVALID_CHIP_ID &&  in xive_pick_irq_target()
574 			if (xc->chip_id == xd->src_chip)  in xive_pick_irq_target()
594 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_startup()  local
598 	xd->saved_p = false;  in xive_irq_startup()
599 	xd->stale_p = false;  in xive_irq_startup()
628 	xd->target = target;  in xive_irq_startup()
641 	xive_do_source_set_mask(xd, false);  in xive_irq_startup()
649 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_shutdown()  local
655 	if (WARN_ON(xd->target == XIVE_INVALID_TARGET))  in xive_irq_shutdown()
659 	xive_do_source_set_mask(xd, true);  in xive_irq_shutdown()
666 				get_hard_smp_processor_id(xd->target),  in xive_irq_shutdown()
669 	xive_dec_target_count(xd->target);  in xive_irq_shutdown()
670 	xd->target = XIVE_INVALID_TARGET;  in xive_irq_shutdown()
675 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_unmask()  local
677 	pr_devel("xive_irq_unmask: irq %d data @%p\n", d->irq, xd);  in xive_irq_unmask()
685 	if (xd->flags & XIVE_IRQ_FLAG_MASK_FW) {  in xive_irq_unmask()
688 					get_hard_smp_processor_id(xd->target),  in xive_irq_unmask()
693 	xive_do_source_set_mask(xd, false);  in xive_irq_unmask()
698 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_mask()  local
700 	pr_devel("xive_irq_mask: irq %d data @%p\n", d->irq, xd);  in xive_irq_mask()
708 	if (xd->flags & XIVE_IRQ_FLAG_MASK_FW) {  in xive_irq_mask()
711 					get_hard_smp_processor_id(xd->target),  in xive_irq_mask()
716 	xive_do_source_set_mask(xd, true);  in xive_irq_mask()
723 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_affinity()  local
742 	if (xd->target != XIVE_INVALID_TARGET &&  in xive_irq_set_affinity()
743 	    cpu_online(xd->target) &&  in xive_irq_set_affinity()
744 	    cpumask_test_cpu(xd->target, cpumask))  in xive_irq_set_affinity()
758 	old_target = xd->target;  in xive_irq_set_affinity()
774 	xd->target = target;  in xive_irq_set_affinity()
785 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_type()  local
812 	    !!(xd->flags & XIVE_IRQ_FLAG_LSI)) {  in xive_irq_set_type()
816 			(xd->flags & XIVE_IRQ_FLAG_LSI) ? "Level" : "Edge");  in xive_irq_set_type()
824 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_retrigger()  local
827 	if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))  in xive_irq_retrigger()
834 	xive_esb_read(xd, XIVE_ESB_SET_PQ_11);  in xive_irq_retrigger()
842 	xive_do_source_eoi(0, xd);  in xive_irq_retrigger()
853 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_vcpu_affinity()  local
862 	if (xd->flags & XIVE_IRQ_FLAG_MASK_FW)  in xive_irq_set_vcpu_affinity()
873 		pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_10);  in xive_irq_set_vcpu_affinity()
874 		if (!xd->stale_p) {  in xive_irq_set_vcpu_affinity()
875 			xd->saved_p = !!(pq & XIVE_ESB_VAL_P);  in xive_irq_set_vcpu_affinity()
876 			xd->stale_p = !xd->saved_p;  in xive_irq_set_vcpu_affinity()
880 		if (xd->target == XIVE_INVALID_TARGET) {  in xive_irq_set_vcpu_affinity()
885 			WARN_ON(xd->saved_p);  in xive_irq_set_vcpu_affinity()
905 		if (xd->saved_p) {  in xive_irq_set_vcpu_affinity()
906 			xive_esb_read(xd, XIVE_ESB_SET_PQ_11);  in xive_irq_set_vcpu_affinity()
924 		if (xd->target == XIVE_INVALID_TARGET) {  in xive_irq_set_vcpu_affinity()
925 			xive_do_source_set_mask(xd, true);  in xive_irq_set_vcpu_affinity()
946 					     get_hard_smp_processor_id(xd->target),  in xive_irq_set_vcpu_affinity()
963 		if (!xd->saved_p)  in xive_irq_set_vcpu_affinity()
964 			xive_do_source_eoi(hw_irq, xd);  in xive_irq_set_vcpu_affinity()
974 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(data);  in xive_get_irqchip_state()  local
978 		*state = !xd->stale_p &&  in xive_get_irqchip_state()
979 			 (xd->saved_p ||  in xive_get_irqchip_state()
980 			  !!(xive_esb_read(xd, XIVE_ESB_GET) & XIVE_ESB_VAL_P));  in xive_get_irqchip_state()
1007 void xive_cleanup_irq_data(struct xive_irq_data *xd)  in xive_cleanup_irq_data()  argument
1009 	if (xd->eoi_mmio) {  in xive_cleanup_irq_data()
1010 		iounmap(xd->eoi_mmio);  in xive_cleanup_irq_data()
1011 		if (xd->eoi_mmio == xd->trig_mmio)  in xive_cleanup_irq_data()
1012 			xd->trig_mmio = NULL;  in xive_cleanup_irq_data()
1013 		xd->eoi_mmio = NULL;  in xive_cleanup_irq_data()
1015 	if (xd->trig_mmio) {  in xive_cleanup_irq_data()
1016 		iounmap(xd->trig_mmio);  in xive_cleanup_irq_data()
1017 		xd->trig_mmio = NULL;  in xive_cleanup_irq_data()
1024 	struct xive_irq_data *xd;  in xive_irq_alloc_data()  local
1027 	xd = kzalloc(sizeof(struct xive_irq_data), GFP_KERNEL);  in xive_irq_alloc_data()
1028 	if (!xd)  in xive_irq_alloc_data()
1030 	rc = xive_ops->populate_irq_data(hw, xd);  in xive_irq_alloc_data()
1032 		kfree(xd);  in xive_irq_alloc_data()
1035 	xd->target = XIVE_INVALID_TARGET;  in xive_irq_alloc_data()
1036 	irq_set_handler_data(virq, xd);  in xive_irq_alloc_data()
1043 	struct xive_irq_data *xd = irq_get_handler_data(virq);  in xive_irq_free_data()  local
1045 	if (!xd)  in xive_irq_free_data()
1048 	xive_cleanup_irq_data(xd);  in xive_irq_free_data()
1049 	kfree(xd);  in xive_irq_free_data()
1057 	struct xive_irq_data *xd;  in xive_cause_ipi()  local
1064 	xd = &xc->ipi_data;  in xive_cause_ipi()
1065 	if (WARN_ON(!xd->trig_mmio))  in xive_cause_ipi()
1067 	out_be64(xd->trig_mmio, 0);  in xive_cause_ipi()
1389 		struct xive_irq_data *xd;  in xive_flush_cpu_queue()  local
1409 		xd = irq_desc_get_handler_data(desc);  in xive_flush_cpu_queue()
1414 		xd->saved_p = false;  in xive_flush_cpu_queue()
1420 		if (xd->flags & XIVE_IRQ_FLAG_LSI)  in xive_flush_cpu_queue()
1421 			xive_do_source_eoi(irqd_to_hwirq(d), xd);  in xive_flush_cpu_queue()