42 #define DBG_VERBOSE(fmt...)	do { } while(0)
75 #define XIVE_BAD_IRQ		0x7fffffff
83  * or 0 if there is no new entry.
92 		return 0;  in xive_read_eq()
97 		return 0;  in xive_read_eq()
105 		if (q->idx == 0)  in xive_read_eq()
109 	return cur & 0x7fffffff;  in xive_read_eq()
119  * (0xff if none) and return what was found (0 if none).
137 	u32 irq = 0;  in xive_scan_interrupts()
138 	u8 prio = 0;  in xive_scan_interrupts()
141 	while (xc->pending_prio != 0) {  in xive_scan_interrupts()
175 			int p = atomic_xchg(&q->pending_count, 0);  in xive_scan_interrupts()
183 	/* If nothing was found, set CPPR to 0xff */  in xive_scan_interrupts()
184 	if (irq == 0)  in xive_scan_interrupts()
185 		prio = 0xff;  in xive_scan_interrupts()
201 static notrace u8 xive_esb_read(struct xive_irq_data *xd, u32 offset)  in xive_esb_read()  argument
206 	if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)  in xive_esb_read()
209 	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)  in xive_esb_read()
210 		val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0);  in xive_esb_read()
212 		val = in_be64(xd->eoi_mmio + offset);  in xive_esb_read()
217 static void xive_esb_write(struct xive_irq_data *xd, u32 offset, u64 data)  in xive_esb_write()  argument
220 	if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)  in xive_esb_write()
223 	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)  in xive_esb_write()
224 		xive_ops->esb_rw(xd->hw_irq, offset, data, 1);  in xive_esb_write()
226 		out_be64(xd->eoi_mmio + offset, data);  in xive_esb_write()
256 			xmon_printf("IPI=0x%08x PQ=%c%c ", xc->hw_ipi,  in xmon_xive_do_dump()
275 		xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);  in xmon_xive_get_irq_config()
279 	xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",  in xmon_xive_get_irq_config()
283 		struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xmon_xive_get_irq_config()  local
284 		u64 val = xive_esb_read(xd, XIVE_ESB_GET);  in xmon_xive_get_irq_config()
292 	return 0;  in xmon_xive_get_irq_config()
323 	DBG_VERBOSE("get_irq: got irq 0x%x, new pending=0x%02x\n",  in xive_get_irq()
328 		return 0;  in xive_get_irq()
344 	if (xive_scan_interrupts(xc, true) != 0) {  in xive_do_queue_eoi()
345 		DBG_VERBOSE("eoi: pending=0x%02x\n", xc->pending_prio);  in xive_do_queue_eoi()
354 static void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd)  in xive_do_source_eoi()  argument
356 	xd->stale_p = false;  in xive_do_source_eoi()
358 	if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)  in xive_do_source_eoi()
359 		xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0);  in xive_do_source_eoi()
360 	else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) {  in xive_do_source_eoi()
365 		 * on P9 DD1.0 needed a latch to be clared in the LPC bridge  in xive_do_source_eoi()
387 		if (xd->flags & XIVE_IRQ_FLAG_LSI)  in xive_do_source_eoi()
388 			xive_esb_read(xd, XIVE_ESB_LOAD_EOI);  in xive_do_source_eoi()
390 			eoi_val = xive_esb_read(xd, XIVE_ESB_SET_PQ_00);  in xive_do_source_eoi()
394 			if ((eoi_val & XIVE_ESB_VAL_Q) && xd->trig_mmio)  in xive_do_source_eoi()
395 				out_be64(xd->trig_mmio, 0);  in xive_do_source_eoi()
403 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_eoi()  local
406 	DBG_VERBOSE("eoi_irq: irq=%d [0x%lx] pending=%02x\n",  in xive_irq_eoi()
414 	    !(xd->flags & XIVE_IRQ_NO_EOI))  in xive_irq_eoi()
415 		xive_do_source_eoi(irqd_to_hwirq(d), xd);  in xive_irq_eoi()
417 		xd->stale_p = true;  in xive_irq_eoi()
423 	xd->saved_p = false;  in xive_irq_eoi()
434 static void xive_do_source_set_mask(struct xive_irq_data *xd,  in xive_do_source_set_mask()  argument
448 		val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01);  in xive_do_source_set_mask()
449 		if (!xd->stale_p && !!(val & XIVE_ESB_VAL_P))  in xive_do_source_set_mask()
450 			xd->saved_p = true;  in xive_do_source_set_mask()
451 		xd->stale_p = false;  in xive_do_source_set_mask()
452 	} else if (xd->saved_p) {  in xive_do_source_set_mask()
453 		xive_esb_read(xd, XIVE_ESB_SET_PQ_10);  in xive_do_source_set_mask()
454 		xd->saved_p = false;  in xive_do_source_set_mask()
456 		xive_esb_read(xd, XIVE_ESB_SET_PQ_00);  in xive_do_source_set_mask()
457 		xd->stale_p = false;  in xive_do_source_set_mask()
495 	if (WARN_ON(cpu < 0 || !xc)) {  in xive_dec_target_count()
522 	for (i = 0; i < first && cpu < nr_cpu_ids; i++)  in xive_find_target_in_mask()
561 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_pick_irq_target()  local
569 	if (xd->src_chip != XIVE_INVALID_CHIP_ID &&  in xive_pick_irq_target()
574 			if (xc->chip_id == xd->src_chip)  in xive_pick_irq_target()
583 		if (cpu >= 0)  in xive_pick_irq_target()
594 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_startup()  local
598 	xd->saved_p = false;  in xive_irq_startup()
599 	xd->stale_p = false;  in xive_irq_startup()
600 	pr_devel("xive_irq_startup: irq %d [0x%x] data @%p\n",  in xive_irq_startup()
628 	xd->target = target;  in xive_irq_startup()
641 	xive_do_source_set_mask(xd, false);  in xive_irq_startup()
643 	return 0;  in xive_irq_startup()
649 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_shutdown()  local
652 	pr_devel("xive_irq_shutdown: irq %d [0x%x] data @%p\n",  in xive_irq_shutdown()
655 	if (WARN_ON(xd->target == XIVE_INVALID_TARGET))  in xive_irq_shutdown()
659 	xive_do_source_set_mask(xd, true);  in xive_irq_shutdown()
666 				get_hard_smp_processor_id(xd->target),  in xive_irq_shutdown()
667 				0xff, XIVE_BAD_IRQ);  in xive_irq_shutdown()
669 	xive_dec_target_count(xd->target);  in xive_irq_shutdown()
670 	xd->target = XIVE_INVALID_TARGET;  in xive_irq_shutdown()
675 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_unmask()  local
677 	pr_devel("xive_irq_unmask: irq %d data @%p\n", d->irq, xd);  in xive_irq_unmask()
682 	 * be fixed by P9 DD2.0, if that is the case, firmware  in xive_irq_unmask()
685 	if (xd->flags & XIVE_IRQ_FLAG_MASK_FW) {  in xive_irq_unmask()
688 					get_hard_smp_processor_id(xd->target),  in xive_irq_unmask()
693 	xive_do_source_set_mask(xd, false);  in xive_irq_unmask()
698 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_mask()  local
700 	pr_devel("xive_irq_mask: irq %d data @%p\n", d->irq, xd);  in xive_irq_mask()
705 	 * be fixed by P9 DD2.0, if that is the case, firmware  in xive_irq_mask()
708 	if (xd->flags & XIVE_IRQ_FLAG_MASK_FW) {  in xive_irq_mask()
711 					get_hard_smp_processor_id(xd->target),  in xive_irq_mask()
712 					0xff, d->irq);  in xive_irq_mask()
716 	xive_do_source_set_mask(xd, true);  in xive_irq_mask()
723 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_affinity()  local
726 	int rc = 0;  in xive_irq_set_affinity()
742 	if (xd->target != XIVE_INVALID_TARGET &&  in xive_irq_set_affinity()
743 	    cpu_online(xd->target) &&  in xive_irq_set_affinity()
744 	    cpumask_test_cpu(xd->target, cpumask))  in xive_irq_set_affinity()
758 	old_target = xd->target;  in xive_irq_set_affinity()
768 	if (rc < 0) {  in xive_irq_set_affinity()
773 	pr_devel("  target: 0x%x\n", target);  in xive_irq_set_affinity()
774 	xd->target = target;  in xive_irq_set_affinity()
785 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_type()  local
812 	    !!(xd->flags & XIVE_IRQ_FLAG_LSI)) {  in xive_irq_set_type()
813 		pr_warn("Interrupt %d (HW 0x%x) type mismatch, Linux says %s, FW says %s\n",  in xive_irq_set_type()
816 			(xd->flags & XIVE_IRQ_FLAG_LSI) ? "Level" : "Edge");  in xive_irq_set_type()
824 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_retrigger()  local
827 	if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))  in xive_irq_retrigger()
828 		return 0;  in xive_irq_retrigger()
834 	xive_esb_read(xd, XIVE_ESB_SET_PQ_11);  in xive_irq_retrigger()
837 	 * Note: We pass "0" to the hw_irq argument in order to  in xive_irq_retrigger()
842 	xive_do_source_eoi(0, xd);  in xive_irq_retrigger()
853 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);  in xive_irq_set_vcpu_affinity()  local
862 	if (xd->flags & XIVE_IRQ_FLAG_MASK_FW)  in xive_irq_set_vcpu_affinity()
873 		pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_10);  in xive_irq_set_vcpu_affinity()
874 		if (!xd->stale_p) {  in xive_irq_set_vcpu_affinity()
875 			xd->saved_p = !!(pq & XIVE_ESB_VAL_P);  in xive_irq_set_vcpu_affinity()
876 			xd->stale_p = !xd->saved_p;  in xive_irq_set_vcpu_affinity()
880 		if (xd->target == XIVE_INVALID_TARGET) {  in xive_irq_set_vcpu_affinity()
885 			WARN_ON(xd->saved_p);  in xive_irq_set_vcpu_affinity()
887 			return 0;  in xive_irq_set_vcpu_affinity()
905 		if (xd->saved_p) {  in xive_irq_set_vcpu_affinity()
906 			xive_esb_read(xd, XIVE_ESB_SET_PQ_11);  in xive_irq_set_vcpu_affinity()
924 		if (xd->target == XIVE_INVALID_TARGET) {  in xive_irq_set_vcpu_affinity()
925 			xive_do_source_set_mask(xd, true);  in xive_irq_set_vcpu_affinity()
926 			return 0;  in xive_irq_set_vcpu_affinity()
946 					     get_hard_smp_processor_id(xd->target),  in xive_irq_set_vcpu_affinity()
963 		if (!xd->saved_p)  in xive_irq_set_vcpu_affinity()
964 			xive_do_source_eoi(hw_irq, xd);  in xive_irq_set_vcpu_affinity()
967 	return 0;  in xive_irq_set_vcpu_affinity()
974 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(data);  in xive_get_irqchip_state()  local
978 		*state = !xd->stale_p &&  in xive_get_irqchip_state()
979 			 (xd->saved_p ||  in xive_get_irqchip_state()
980 			  !!(xive_esb_read(xd, XIVE_ESB_GET) & XIVE_ESB_VAL_P));  in xive_get_irqchip_state()
981 		return 0;  in xive_get_irqchip_state()
1007 void xive_cleanup_irq_data(struct xive_irq_data *xd)  in xive_cleanup_irq_data()  argument
1009 	if (xd->eoi_mmio) {  in xive_cleanup_irq_data()
1010 		iounmap(xd->eoi_mmio);  in xive_cleanup_irq_data()
1011 		if (xd->eoi_mmio == xd->trig_mmio)  in xive_cleanup_irq_data()
1012 			xd->trig_mmio = NULL;  in xive_cleanup_irq_data()
1013 		xd->eoi_mmio = NULL;  in xive_cleanup_irq_data()
1015 	if (xd->trig_mmio) {  in xive_cleanup_irq_data()
1016 		iounmap(xd->trig_mmio);  in xive_cleanup_irq_data()
1017 		xd->trig_mmio = NULL;  in xive_cleanup_irq_data()
1024 	struct xive_irq_data *xd;  in xive_irq_alloc_data()  local
1027 	xd = kzalloc(sizeof(struct xive_irq_data), GFP_KERNEL);  in xive_irq_alloc_data()
1028 	if (!xd)  in xive_irq_alloc_data()
1030 	rc = xive_ops->populate_irq_data(hw, xd);  in xive_irq_alloc_data()
1032 		kfree(xd);  in xive_irq_alloc_data()
1035 	xd->target = XIVE_INVALID_TARGET;  in xive_irq_alloc_data()
1036 	irq_set_handler_data(virq, xd);  in xive_irq_alloc_data()
1038 	return 0;  in xive_irq_alloc_data()
1043 	struct xive_irq_data *xd = irq_get_handler_data(virq);  in xive_irq_free_data()  local
1045 	if (!xd)  in xive_irq_free_data()
1048 	xive_cleanup_irq_data(xd);  in xive_irq_free_data()
1049 	kfree(xd);  in xive_irq_free_data()
1057 	struct xive_irq_data *xd;  in xive_cause_ipi()  local
1061 	DBG_VERBOSE("IPI CPU %d -> %d (HW IRQ 0x%x)\n",  in xive_cause_ipi()
1064 	xd = &xc->ipi_data;  in xive_cause_ipi()
1065 	if (WARN_ON(!xd->trig_mmio))  in xive_cause_ipi()
1067 	out_be64(xd->trig_mmio, 0);  in xive_cause_ipi()
1083 	DBG_VERBOSE("IPI eoi: irq=%d [0x%lx] (HW IRQ 0x%x) pending=%02x\n",  in xive_ipi_eoi()
1118 	virq = irq_create_mapping(xive_irq_domain, 0);  in xive_request_ipi()
1135 	if (xc->hw_ipi != 0)  in xive_setup_cpu_ipi()
1136 		return 0;  in xive_setup_cpu_ipi()
1164 	return 0;  in xive_setup_cpu_ipi()
1172 	if (xc->hw_ipi == 0)  in xive_cleanup_cpu_ipi()
1186 				0xff, xive_ipi_irq);  in xive_cleanup_cpu_ipi()
1217 	/* IPIs are special and come up with HW number 0 */  in xive_irq_domain_map()
1218 	if (hw == 0) {  in xive_irq_domain_map()
1225 		return 0;  in xive_irq_domain_map()
1235 	return 0;  in xive_irq_domain_map()
1256 	*out_hwirq = intspec[0];  in xive_irq_domain_xlate()
1270 	return 0;  in xive_irq_domain_xlate()
1303 	int rc = 0;  in xive_setup_cpu_queues()
1344 	/* Set CPPR to 0xff to enable flow of interrupts */  in xive_setup_cpu()
1345 	xc->cppr = 0xff;  in xive_setup_cpu()
1346 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff);  in xive_setup_cpu()
1382 	while ((irq = xive_scan_interrupts(xc, false)) != 0) {  in xive_flush_cpu_queue()
1389 		struct xive_irq_data *xd;  in xive_flush_cpu_queue()  local
1396 		if (d->domain != xive_irq_domain || hw_irq == 0)  in xive_flush_cpu_queue()
1409 		xd = irq_desc_get_handler_data(desc);  in xive_flush_cpu_queue()
1414 		xd->saved_p = false;  in xive_flush_cpu_queue()
1420 		if (xd->flags & XIVE_IRQ_FLAG_LSI)  in xive_flush_cpu_queue()
1421 			xive_do_source_eoi(irqd_to_hwirq(d), xd);  in xive_flush_cpu_queue()
1437 	/* Set CPPR to 0 to disable flow of interrupts */  in xive_smp_disable_cpu()
1438 	xc->cppr = 0;  in xive_smp_disable_cpu()
1439 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0);  in xive_smp_disable_cpu()
1445 	xc->cppr = 0xff;  in xive_smp_disable_cpu()
1446 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff);  in xive_smp_disable_cpu()
1467 	/* Set CPPR to 0 to disable flow of interrupts */  in xive_teardown_cpu()
1468 	xc->cppr = 0;  in xive_teardown_cpu()
1469 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0);  in xive_teardown_cpu()
1528 	memset(qpage, 0, 1 << queue_shift);  in xive_queue_page_alloc()
1536 	return 0;  in xive_off()