43 #define DBG_VERBOSE(fmt...)	do { } while(0)
92 static bool xive_is_store_eoi(struct xive_irq_data *xd)  in xive_is_store_eoi()  argument
94 	return xd->flags & XIVE_IRQ_FLAG_STORE_EOI && xive_store_eoi;  in xive_is_store_eoi()
99  * or 0 if there is no new entry.
108 		return 0;  in xive_read_eq()
113 		return 0;  in xive_read_eq()
121 		if (q->idx == 0)  in xive_read_eq()
125 	return cur & 0x7fffffff;  in xive_read_eq()
135  * (0xff if none) and return what was found (0 if none).
153 	u32 irq = 0;  in xive_scan_interrupts()
154 	u8 prio = 0;  in xive_scan_interrupts()
157 	while (xc->pending_prio != 0) {  in xive_scan_interrupts()
191 			int p = atomic_xchg(&q->pending_count, 0);  in xive_scan_interrupts()
199 	/* If nothing was found, set CPPR to 0xff */  in xive_scan_interrupts()
200 	if (irq == 0)  in xive_scan_interrupts()
201 		prio = 0xff;  in xive_scan_interrupts()
217 static notrace u8 xive_esb_read(struct xive_irq_data *xd, u32 offset)  in xive_esb_read()  argument
221 	if (offset == XIVE_ESB_SET_PQ_10 && xive_is_store_eoi(xd))  in xive_esb_read()
224 	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)  in xive_esb_read()
225 		val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0);  in xive_esb_read()
227 		val = in_be64(xd->eoi_mmio + offset);  in xive_esb_read()
232 static void xive_esb_write(struct xive_irq_data *xd, u32 offset, u64 data)  in xive_esb_write()  argument
234 	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)  in xive_esb_write()
235 		xive_ops->esb_rw(xd->hw_irq, offset, data, 1);  in xive_esb_write()
237 		out_be64(xd->eoi_mmio + offset, data);  in xive_esb_write()
241 static void xive_irq_data_dump(struct xive_irq_data *xd, char *buffer, size_t size)  in xive_irq_data_dump()  argument
243 	u64 val = xive_esb_read(xd, XIVE_ESB_GET);  in xive_irq_data_dump()
245 	snprintf(buffer, size, "flags=%c%c%c PQ=%c%c 0x%016llx 0x%016llx",  in xive_irq_data_dump()
246 		 xive_is_store_eoi(xd) ? 'S' : ' ',  in xive_irq_data_dump()
247 		 xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ',  in xive_irq_data_dump()
248 		 xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ',  in xive_irq_data_dump()
251 		 xd->trig_page, xd->eoi_page);  in xive_irq_data_dump()
283 			xmon_printf("IPI=0x%08x %s", xc->hw_ipi, buffer);  in xmon_xive_do_dump()
307 		xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);  in xmon_xive_get_irq_config()
311 	xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",  in xmon_xive_get_irq_config()
326 	return 0;  in xmon_xive_get_irq_config()
370 	DBG_VERBOSE("get_irq: got irq 0x%x, new pending=0x%02x\n",  in xive_get_irq()
375 		return 0;  in xive_get_irq()
391 	if (xive_scan_interrupts(xc, true) != 0) {  in xive_do_queue_eoi()
392 		DBG_VERBOSE("eoi: pending=0x%02x\n", xc->pending_prio);  in xive_do_queue_eoi()
401 static void xive_do_source_eoi(struct xive_irq_data *xd)  in xive_do_source_eoi()  argument
405 	xd->stale_p = false;  in xive_do_source_eoi()
408 	if (xive_is_store_eoi(xd)) {  in xive_do_source_eoi()
409 		xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0);  in xive_do_source_eoi()
418 	if (xd->flags & XIVE_IRQ_FLAG_LSI) {  in xive_do_source_eoi()
419 		xive_esb_read(xd, XIVE_ESB_LOAD_EOI);  in xive_do_source_eoi()
429 	eoi_val = xive_esb_read(xd, XIVE_ESB_SET_PQ_00);  in xive_do_source_eoi()
433 	if ((eoi_val & XIVE_ESB_VAL_Q) && xd->trig_mmio)  in xive_do_source_eoi()
434 		out_be64(xd->trig_mmio, 0);  in xive_do_source_eoi()
440 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_eoi()  local
443 	DBG_VERBOSE("eoi_irq: irq=%d [0x%lx] pending=%02x\n",  in xive_irq_eoi()
451 	    !(xd->flags & XIVE_IRQ_FLAG_NO_EOI))  in xive_irq_eoi()
452 		xive_do_source_eoi(xd);  in xive_irq_eoi()
454 		xd->stale_p = true;  in xive_irq_eoi()
460 	xd->saved_p = false;  in xive_irq_eoi()
469 static void xive_do_source_set_mask(struct xive_irq_data *xd,  in xive_do_source_set_mask()  argument
474 	pr_debug("%s: HW 0x%x %smask\n", __func__, xd->hw_irq, mask ? "" : "un");  in xive_do_source_set_mask()
485 		val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01);  in xive_do_source_set_mask()
486 		if (!xd->stale_p && !!(val & XIVE_ESB_VAL_P))  in xive_do_source_set_mask()
487 			xd->saved_p = true;  in xive_do_source_set_mask()
488 		xd->stale_p = false;  in xive_do_source_set_mask()
489 	} else if (xd->saved_p) {  in xive_do_source_set_mask()
490 		xive_esb_read(xd, XIVE_ESB_SET_PQ_10);  in xive_do_source_set_mask()
491 		xd->saved_p = false;  in xive_do_source_set_mask()
493 		xive_esb_read(xd, XIVE_ESB_SET_PQ_00);  in xive_do_source_set_mask()
494 		xd->stale_p = false;  in xive_do_source_set_mask()
532 	if (WARN_ON(cpu < 0 || !xc)) {  in xive_dec_target_count()
559 	for (i = 0; i < first && cpu < nr_cpu_ids; i++)  in xive_find_target_in_mask()
598 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_pick_irq_target()  local
606 	if (xd->src_chip != XIVE_INVALID_CHIP_ID &&  in xive_pick_irq_target()
611 			if (xc->chip_id == xd->src_chip)  in xive_pick_irq_target()
620 		if (cpu >= 0)  in xive_pick_irq_target()
631 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_startup()  local
635 	xd->saved_p = false;  in xive_irq_startup()
636 	xd->stale_p = false;  in xive_irq_startup()
638 	pr_debug("%s: irq %d [0x%x] data @%p\n", __func__, d->irq, hw_irq, d);  in xive_irq_startup()
655 	xd->target = target;  in xive_irq_startup()
668 	xive_do_source_set_mask(xd, false);  in xive_irq_startup()
670 	return 0;  in xive_irq_startup()
676 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_shutdown()  local
679 	pr_debug("%s: irq %d [0x%x] data @%p\n", __func__, d->irq, hw_irq, d);  in xive_irq_shutdown()
681 	if (WARN_ON(xd->target == XIVE_INVALID_TARGET))  in xive_irq_shutdown()
685 	xive_do_source_set_mask(xd, true);  in xive_irq_shutdown()
692 				get_hard_smp_processor_id(xd->target),  in xive_irq_shutdown()
693 				0xff, XIVE_BAD_IRQ);  in xive_irq_shutdown()
695 	xive_dec_target_count(xd->target);  in xive_irq_shutdown()
696 	xd->target = XIVE_INVALID_TARGET;  in xive_irq_shutdown()
701 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_unmask()  local
703 	pr_debug("%s: irq %d data @%p\n", __func__, d->irq, xd);  in xive_irq_unmask()
705 	xive_do_source_set_mask(xd, false);  in xive_irq_unmask()
710 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_mask()  local
712 	pr_debug("%s: irq %d data @%p\n", __func__, d->irq, xd);  in xive_irq_mask()
714 	xive_do_source_set_mask(xd, true);  in xive_irq_mask()
721 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_affinity()  local
724 	int rc = 0;  in xive_irq_set_affinity()
726 	pr_debug("%s: irq %d/0x%x\n", __func__, d->irq, hw_irq);  in xive_irq_set_affinity()
736 	if (xd->target != XIVE_INVALID_TARGET &&  in xive_irq_set_affinity()
737 	    cpu_online(xd->target) &&  in xive_irq_set_affinity()
738 	    cpumask_test_cpu(xd->target, cpumask))  in xive_irq_set_affinity()
752 	old_target = xd->target;  in xive_irq_set_affinity()
762 	if (rc < 0) {  in xive_irq_set_affinity()
767 	pr_debug("  target: 0x%x\n", target);  in xive_irq_set_affinity()
768 	xd->target = target;  in xive_irq_set_affinity()
779 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_type()  local
806 	    !!(xd->flags & XIVE_IRQ_FLAG_LSI)) {  in xive_irq_set_type()
807 		pr_warn("Interrupt %d (HW 0x%x) type mismatch, Linux says %s, FW says %s\n",  in xive_irq_set_type()
810 			(xd->flags & XIVE_IRQ_FLAG_LSI) ? "Level" : "Edge");  in xive_irq_set_type()
818 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_retrigger()  local
821 	if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))  in xive_irq_retrigger()
822 		return 0;  in xive_irq_retrigger()
828 	xive_esb_read(xd, XIVE_ESB_SET_PQ_11);  in xive_irq_retrigger()
829 	xive_do_source_eoi(xd);  in xive_irq_retrigger()
840 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_vcpu_affinity()  local
853 		pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_10);  in xive_irq_set_vcpu_affinity()
854 		if (!xd->stale_p) {  in xive_irq_set_vcpu_affinity()
855 			xd->saved_p = !!(pq & XIVE_ESB_VAL_P);  in xive_irq_set_vcpu_affinity()
856 			xd->stale_p = !xd->saved_p;  in xive_irq_set_vcpu_affinity()
860 		if (xd->target == XIVE_INVALID_TARGET) {  in xive_irq_set_vcpu_affinity()
865 			WARN_ON(xd->saved_p);  in xive_irq_set_vcpu_affinity()
867 			return 0;  in xive_irq_set_vcpu_affinity()
885 		if (xd->saved_p) {  in xive_irq_set_vcpu_affinity()
886 			xive_esb_read(xd, XIVE_ESB_SET_PQ_11);  in xive_irq_set_vcpu_affinity()
904 		if (xd->target == XIVE_INVALID_TARGET) {  in xive_irq_set_vcpu_affinity()
905 			xive_do_source_set_mask(xd, true);  in xive_irq_set_vcpu_affinity()
906 			return 0;  in xive_irq_set_vcpu_affinity()
926 					     get_hard_smp_processor_id(xd->target),  in xive_irq_set_vcpu_affinity()
943 		if (!xd->saved_p)  in xive_irq_set_vcpu_affinity()
944 			xive_do_source_eoi(xd);  in xive_irq_set_vcpu_affinity()
947 	return 0;  in xive_irq_set_vcpu_affinity()
954 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(data);  in xive_get_irqchip_state()  local
959 		pq = xive_esb_read(xd, XIVE_ESB_GET);  in xive_get_irqchip_state()
968 		*state = (pq != XIVE_ESB_INVALID) && !xd->stale_p &&  in xive_get_irqchip_state()
969 			(xd->saved_p || (!!(pq & XIVE_ESB_VAL_P) &&  in xive_get_irqchip_state()
971 		return 0;  in xive_get_irqchip_state()
997 void xive_cleanup_irq_data(struct xive_irq_data *xd)  in xive_cleanup_irq_data()  argument
999 	pr_debug("%s for HW 0x%x\n", __func__, xd->hw_irq);  in xive_cleanup_irq_data()
1001 	if (xd->eoi_mmio) {  in xive_cleanup_irq_data()
1002 		iounmap(xd->eoi_mmio);  in xive_cleanup_irq_data()
1003 		if (xd->eoi_mmio == xd->trig_mmio)  in xive_cleanup_irq_data()
1004 			xd->trig_mmio = NULL;  in xive_cleanup_irq_data()
1005 		xd->eoi_mmio = NULL;  in xive_cleanup_irq_data()
1007 	if (xd->trig_mmio) {  in xive_cleanup_irq_data()
1008 		iounmap(xd->trig_mmio);  in xive_cleanup_irq_data()
1009 		xd->trig_mmio = NULL;  in xive_cleanup_irq_data()
1016 	struct xive_irq_data *xd;  in xive_irq_alloc_data()  local
1019 	xd = kzalloc(sizeof(struct xive_irq_data), GFP_KERNEL);  in xive_irq_alloc_data()
1020 	if (!xd)  in xive_irq_alloc_data()
1022 	rc = xive_ops->populate_irq_data(hw, xd);  in xive_irq_alloc_data()
1024 		kfree(xd);  in xive_irq_alloc_data()
1027 	xd->target = XIVE_INVALID_TARGET;  in xive_irq_alloc_data()
1028 	irq_set_handler_data(virq, xd);  in xive_irq_alloc_data()
1037 	xive_esb_read(xd, XIVE_ESB_SET_PQ_01);  in xive_irq_alloc_data()
1039 	return 0;  in xive_irq_alloc_data()
1044 	struct xive_irq_data *xd = irq_get_handler_data(virq);  in xive_irq_free_data()  local
1046 	if (!xd)  in xive_irq_free_data()
1049 	xive_cleanup_irq_data(xd);  in xive_irq_free_data()
1050 	kfree(xd);  in xive_irq_free_data()
1059 	struct xive_irq_data *xd;  in xive_cause_ipi()  local
1063 	DBG_VERBOSE("IPI CPU %d -> %d (HW IRQ 0x%x)\n",  in xive_cause_ipi()
1066 	xd = &xc->ipi_data;  in xive_cause_ipi()
1067 	if (WARN_ON(!xd->trig_mmio))  in xive_cause_ipi()
1069 	out_be64(xd->trig_mmio, 0);  in xive_cause_ipi()
1085 	DBG_VERBOSE("IPI eoi: irq=%d [0x%lx] (HW IRQ 0x%x) pending=%02x\n",  in xive_ipi_eoi()
1121 	for (i = 0; i < nr_irqs; i++) {  in xive_ipi_irq_domain_alloc()
1126 	return 0;  in xive_ipi_irq_domain_alloc()
1163 		if (ret < 0)  in xive_init_ipis()
1188 		return 0;  in xive_request_ipi()
1194 	WARN(ret < 0, "Failed to request IPI %d: %d\n", xid->irq, ret);  in xive_request_ipi()
1210 		return 0;  in xive_setup_cpu_ipi()
1235 	pr_debug("CPU %d HW IPI 0x%x, virq %d, trig_mmio=%p\n", cpu,  in xive_setup_cpu_ipi()
1241 	return 0;  in xive_setup_cpu_ipi()
1267 				0xff, xive_ipi_irq);  in xive_cleanup_cpu_ipi()
1303 	return 0;  in xive_irq_domain_map()
1316 	*out_hwirq = intspec[0];  in xive_irq_domain_xlate()
1330 	return 0;  in xive_irq_domain_xlate()
1355 	struct xive_irq_data *xd;  in xive_irq_domain_debug_show()  local
1369 	xd = irq_data_get_irq_handler_data(irqd);  in xive_irq_domain_debug_show()
1370 	if (!xd) {  in xive_irq_domain_debug_show()
1375 	val = xive_esb_read(xd, XIVE_ESB_GET);  in xive_irq_domain_debug_show()
1376 	seq_printf(m, "%*sESB:      %s\n", ind, "", esb_names[val & 0x3]);  in xive_irq_domain_debug_show()
1377 	seq_printf(m, "%*sPstate:   %s %s\n", ind, "", xd->stale_p ? "stale" : "",  in xive_irq_domain_debug_show()
1378 		   xd->saved_p ? "saved" : "");  in xive_irq_domain_debug_show()
1379 	seq_printf(m, "%*sTarget:   %d\n", ind, "", xd->target);  in xive_irq_domain_debug_show()
1380 	seq_printf(m, "%*sChip:     %d\n", ind, "", xd->src_chip);  in xive_irq_domain_debug_show()
1381 	seq_printf(m, "%*sTrigger:  0x%016llx\n", ind, "", xd->trig_page);  in xive_irq_domain_debug_show()
1382 	seq_printf(m, "%*sEOI:      0x%016llx\n", ind, "", xd->eoi_page);  in xive_irq_domain_debug_show()
1383 	seq_printf(m, "%*sFlags:    0x%llx\n", ind, "", xd->flags);  in xive_irq_domain_debug_show()
1384 	for (i = 0; i < ARRAY_SIZE(xive_irq_flags); i++) {  in xive_irq_domain_debug_show()
1385 		if (xd->flags & xive_irq_flags[i].mask)  in xive_irq_domain_debug_show()
1414 	pr_debug("%s %d/0x%lx #%d\n", __func__, virq, hwirq, nr_irqs);  in xive_irq_domain_alloc()
1416 	for (i = 0; i < nr_irqs; i++) {  in xive_irq_domain_alloc()
1435 	return 0;  in xive_irq_domain_alloc()
1445 	for (i = 0; i < nr_irqs; i++)  in xive_irq_domain_free()
1481 	int rc = 0;  in xive_setup_cpu_queues()
1520 	/* Set CPPR to 0xff to enable flow of interrupts */  in xive_setup_cpu()
1521 	xc->cppr = 0xff;  in xive_setup_cpu()
1522 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff);  in xive_setup_cpu()
1558 	while ((irq = xive_scan_interrupts(xc, false)) != 0) {  in xive_flush_cpu_queue()
1565 		struct xive_irq_data *xd;  in xive_flush_cpu_queue()  local
1584 		xd = irq_desc_get_handler_data(desc);  in xive_flush_cpu_queue()
1589 		xd->saved_p = false;  in xive_flush_cpu_queue()
1595 		if (xd->flags & XIVE_IRQ_FLAG_LSI)  in xive_flush_cpu_queue()
1596 			xive_do_source_eoi(xd);  in xive_flush_cpu_queue()
1612 	/* Set CPPR to 0 to disable flow of interrupts */  in xive_smp_disable_cpu()
1613 	xc->cppr = 0;  in xive_smp_disable_cpu()
1614 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0);  in xive_smp_disable_cpu()
1620 	xc->cppr = 0xff;  in xive_smp_disable_cpu()
1621 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff);  in xive_smp_disable_cpu()
1642 	/* Set CPPR to 0 to disable flow of interrupts */  in xive_teardown_cpu()
1643 	xc->cppr = 0;  in xive_teardown_cpu()
1644 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0);  in xive_teardown_cpu()
1703 	memset(qpage, 0, 1 << queue_shift);  in xive_queue_page_alloc()
1720 	if (strncmp(arg, "off", 3) == 0) {  in xive_store_eoi_cmdline()
1742 			seq_printf(m, "IPI=0x%08x %s", xc->hw_ipi, buffer);  in xive_debug_show_ipi()
1760 		seq_printf(m, "IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);  in xive_debug_show_irq()
1764 	seq_printf(m, "IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",  in xive_debug_show_irq()
1783 	return 0;  in xive_irq_debug_show()
1796 	return 0;  in xive_ipi_debug_show()
1806 		for (i = 0; i < q->msk + 1; i++) {  in xive_eq_debug_show_one()
1824 	return 0;  in xive_eq_debug_show()
1863 	return 0;  in xive_core_debug_init()