40 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
43  * navi10_ih_init_register_offset - Initialize register offset for ih rings
49 static void navi10_ih_init_register_offset(struct amdgpu_device *adev)  in navi10_ih_init_register_offset()
53 	if (adev->irq.ih.ring_size) {  in navi10_ih_init_register_offset()
54 		ih_regs = &adev->irq.ih.ih_regs;  in navi10_ih_init_register_offset()
55 		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE);  in navi10_ih_init_register_offset()
56 		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI);  in navi10_ih_init_register_offset()
57 		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);  in navi10_ih_init_register_offset()
58 		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);  in navi10_ih_init_register_offset()
59 		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR);  in navi10_ih_init_register_offset()
60 		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR);  in navi10_ih_init_register_offset()
61 		ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO);  in navi10_ih_init_register_offset()
62 		ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI);  in navi10_ih_init_register_offset()
63 		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL;  in navi10_ih_init_register_offset()
66 	if (adev->irq.ih1.ring_size) {  in navi10_ih_init_register_offset()
67 		ih_regs = &adev->irq.ih1.ih_regs;  in navi10_ih_init_register_offset()
68 		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1);  in navi10_ih_init_register_offset()
69 		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1);  in navi10_ih_init_register_offset()
70 		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1);  in navi10_ih_init_register_offset()
71 		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1);  in navi10_ih_init_register_offset()
72 		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1);  in navi10_ih_init_register_offset()
73 		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1);  in navi10_ih_init_register_offset()
74 		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1;  in navi10_ih_init_register_offset()
77 	if (adev->irq.ih2.ring_size) {  in navi10_ih_init_register_offset()
78 		ih_regs = &adev->irq.ih2.ih_regs;  in navi10_ih_init_register_offset()
79 		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2);  in navi10_ih_init_register_offset()
80 		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2);  in navi10_ih_init_register_offset()
81 		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2);  in navi10_ih_init_register_offset()
82 		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2);  in navi10_ih_init_register_offset()
83 		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2);  in navi10_ih_init_register_offset()
84 		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2);  in navi10_ih_init_register_offset()
85 		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2;  in navi10_ih_init_register_offset()
90  * force_update_wptr_for_self_int - Force update the wptr for self interrupt
95  * @enabled: Enable/disable timeout flush mechanism
104 static void
110 	if (adev->ip_versions[OSSSYS_HWIP][0] < IP_VERSION(5, 0, 3))  in force_update_wptr_for_self_int()
124 		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, ih_rb_cntl))  in force_update_wptr_for_self_int()
134 		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, ih_rb_cntl))  in force_update_wptr_for_self_int()
144  * navi10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer
148  * @enable: true - enable the interrupts, false - disable the interrupts
152 static int navi10_ih_toggle_ring_interrupts(struct amdgpu_device *adev,  in navi10_ih_toggle_ring_interrupts()
154 					    bool enable)  in navi10_ih_toggle_ring_interrupts()  argument
159 	ih_regs = &ih->ih_regs;  in navi10_ih_toggle_ring_interrupts()
161 	tmp = RREG32(ih_regs->ih_rb_cntl);  in navi10_ih_toggle_ring_interrupts()
162 	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0));  in navi10_ih_toggle_ring_interrupts()
165 	if (ih == &adev->irq.ih)  in navi10_ih_toggle_ring_interrupts()
166 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));  in navi10_ih_toggle_ring_interrupts()
169 		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp))  in navi10_ih_toggle_ring_interrupts()
170 			return -ETIMEDOUT;  in navi10_ih_toggle_ring_interrupts()
172 		WREG32(ih_regs->ih_rb_cntl, tmp);  in navi10_ih_toggle_ring_interrupts()
175 	if (enable) {  in navi10_ih_toggle_ring_interrupts()
176 		ih->enabled = true;  in navi10_ih_toggle_ring_interrupts()
179 		WREG32(ih_regs->ih_rb_rptr, 0);  in navi10_ih_toggle_ring_interrupts()
180 		WREG32(ih_regs->ih_rb_wptr, 0);  in navi10_ih_toggle_ring_interrupts()
181 		ih->enabled = false;  in navi10_ih_toggle_ring_interrupts()
182 		ih->rptr = 0;  in navi10_ih_toggle_ring_interrupts()
189  * navi10_ih_toggle_interrupts - Toggle all the available interrupt ring buffers
192  * @enable: enable or disable interrupt ring buffers
196 static int navi10_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable)  in navi10_ih_toggle_interrupts()  argument
198 	struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2};  in navi10_ih_toggle_interrupts()
203 		if (ih[i]->ring_size) {  in navi10_ih_toggle_interrupts()
204 			r = navi10_ih_toggle_ring_interrupts(adev, ih[i], enable);  in navi10_ih_toggle_interrupts()
213 static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)  in navi10_ih_rb_cntl()
215 	int rb_bufsz = order_base_2(ih->ring_size / 4);  in navi10_ih_rb_cntl()
218 				   MC_SPACE, ih->use_bus_addr ? 1 : 4);  in navi10_ih_rb_cntl()
236 static uint32_t navi10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)  in navi10_ih_doorbell_rptr()
240 	if (ih->use_doorbell) {  in navi10_ih_doorbell_rptr()
243 						 ih->doorbell_index);  in navi10_ih_doorbell_rptr()
246 						 ENABLE, 1);  in navi10_ih_doorbell_rptr()
250 						 ENABLE, 0);  in navi10_ih_doorbell_rptr()
256  * navi10_ih_enable_ring - enable an ih ring buffer
261  * Enable an ih ring buffer (NAVI10)
263 static int navi10_ih_enable_ring(struct amdgpu_device *adev,  in navi10_ih_enable_ring()
269 	ih_regs = &ih->ih_regs;  in navi10_ih_enable_ring()
271 	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/  in navi10_ih_enable_ring()
272 	WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8);  in navi10_ih_enable_ring()
273 	WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff);  in navi10_ih_enable_ring()
275 	tmp = RREG32(ih_regs->ih_rb_cntl);  in navi10_ih_enable_ring()
277 	if (ih == &adev->irq.ih)  in navi10_ih_enable_ring()
278 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled);  in navi10_ih_enable_ring()
279 	if (ih == &adev->irq.ih1)  in navi10_ih_enable_ring()
283 		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) {  in navi10_ih_enable_ring()
285 			return -ETIMEDOUT;  in navi10_ih_enable_ring()
288 		WREG32(ih_regs->ih_rb_cntl, tmp);  in navi10_ih_enable_ring()
291 	if (ih == &adev->irq.ih) {  in navi10_ih_enable_ring()
293 		WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr));  in navi10_ih_enable_ring()
294 		WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF);  in navi10_ih_enable_ring()
298 	WREG32(ih_regs->ih_rb_wptr, 0);  in navi10_ih_enable_ring()
299 	WREG32(ih_regs->ih_rb_rptr, 0);  in navi10_ih_enable_ring()
301 	WREG32(ih_regs->ih_doorbell_rptr, navi10_ih_doorbell_rptr(ih));  in navi10_ih_enable_ring()
307  * navi10_ih_irq_init - init and enable the interrupt ring
312  * enable the RLC, disable interrupts, enable the IH
313  * ring buffer and enable it (NAVI).
317 static int navi10_ih_irq_init(struct amdgpu_device *adev)  in navi10_ih_irq_init()
319 	struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2};  in navi10_ih_irq_init()
329 	adev->nbio.funcs->ih_control(adev);  in navi10_ih_irq_init()
331 	if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) {  in navi10_ih_irq_init()
332 		if (ih[0]->use_bus_addr) {  in navi10_ih_irq_init()
333 			switch (adev->ip_versions[OSSSYS_HWIP][0]) {  in navi10_ih_irq_init()
353 		if (ih[i]->ring_size) {  in navi10_ih_irq_init()
361 	adev->nbio.funcs->ih_doorbell_range(adev, ih[0]->use_doorbell,  in navi10_ih_irq_init()
362 					    ih[0]->doorbell_index);  in navi10_ih_irq_init()
364 	pci_set_master(adev->pdev);  in navi10_ih_irq_init()
366 	/* enable interrupts */  in navi10_ih_irq_init()
370 	/* enable wptr force update for self int */  in navi10_ih_irq_init()
373 	if (adev->irq.ih_soft.ring_size)  in navi10_ih_irq_init()
374 		adev->irq.ih_soft.enabled = true;  in navi10_ih_irq_init()
380  * navi10_ih_irq_disable - disable interrupts
386 static void navi10_ih_irq_disable(struct amdgpu_device *adev)  in navi10_ih_irq_disable()
396  * navi10_ih_get_wptr - get the IH ring buffer wptr
406 static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,  in navi10_ih_get_wptr()
412 	if (ih == &adev->irq.ih || ih == &adev->irq.ih_soft) {  in navi10_ih_get_wptr()
414 		 * to register-based code with overflow checking below.  in navi10_ih_get_wptr()
418 		wptr = le32_to_cpu(*ih->wptr_cpu);  in navi10_ih_get_wptr()
424 	ih_regs = &ih->ih_regs;  in navi10_ih_get_wptr()
427 	wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr);  in navi10_ih_get_wptr()
436 	tmp = (wptr + 32) & ih->ptr_mask;  in navi10_ih_get_wptr()
437 	dev_warn(adev->dev, "IH ring buffer overflow "  in navi10_ih_get_wptr()
439 		 wptr, ih->rptr, tmp);  in navi10_ih_get_wptr()
440 	ih->rptr = tmp;  in navi10_ih_get_wptr()
442 	tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl);  in navi10_ih_get_wptr()
444 	WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp);  in navi10_ih_get_wptr()
446 	return (wptr & ih->ptr_mask);  in navi10_ih_get_wptr()
450  * navi10_ih_irq_rearm - rearm IRQ if lost
456 static void navi10_ih_irq_rearm(struct amdgpu_device *adev,  in navi10_ih_irq_rearm()
463 	ih_regs = &ih->ih_regs;  in navi10_ih_irq_rearm()
465 	/* Rearm IRQ / re-write doorbell if doorbell write is lost */  in navi10_ih_irq_rearm()
467 		v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr);  in navi10_ih_irq_rearm()
468 		if ((v < ih->ring_size) && (v != ih->rptr))  in navi10_ih_irq_rearm()
469 			WDOORBELL32(ih->doorbell_index, ih->rptr);  in navi10_ih_irq_rearm()
476  * navi10_ih_set_rptr - set the IH ring buffer rptr
483 static void navi10_ih_set_rptr(struct amdgpu_device *adev,  in navi10_ih_set_rptr()
488 	if (ih == &adev->irq.ih_soft)  in navi10_ih_set_rptr()
491 	if (ih->use_doorbell) {  in navi10_ih_set_rptr()
493 		*ih->rptr_cpu = ih->rptr;  in navi10_ih_set_rptr()
494 		WDOORBELL32(ih->doorbell_index, ih->rptr);  in navi10_ih_set_rptr()
499 		ih_regs = &ih->ih_regs;  in navi10_ih_set_rptr()
500 		WREG32(ih_regs->ih_rb_rptr, ih->rptr);  in navi10_ih_set_rptr()
505  * navi10_ih_self_irq - dispatch work for ring 1 and 2
513 static int navi10_ih_self_irq(struct amdgpu_device *adev,  in navi10_ih_self_irq()
517 	switch (entry->ring_id) {  in navi10_ih_self_irq()
519 		schedule_work(&adev->irq.ih1_work);  in navi10_ih_self_irq()
522 		schedule_work(&adev->irq.ih2_work);  in navi10_ih_self_irq()
529 static const struct amdgpu_irq_src_funcs navi10_ih_self_irq_funcs = {
533 static void navi10_ih_set_self_irq_funcs(struct amdgpu_device *adev)  in navi10_ih_set_self_irq_funcs()
535 	adev->irq.self_irq.num_types = 0;  in navi10_ih_set_self_irq_funcs()
536 	adev->irq.self_irq.funcs = &navi10_ih_self_irq_funcs;  in navi10_ih_set_self_irq_funcs()
539 static int navi10_ih_early_init(void *handle)  in navi10_ih_early_init()
548 static int navi10_ih_sw_init(void *handle)  in navi10_ih_sw_init()
555 				&adev->irq.self_irq);  in navi10_ih_sw_init()
563 	if ((adev->flags & AMD_IS_APU) ||  in navi10_ih_sw_init()
564 	    (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))  in navi10_ih_sw_init()
568 	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, use_bus_addr);  in navi10_ih_sw_init()
572 	adev->irq.ih.use_doorbell = true;  in navi10_ih_sw_init()
573 	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;  in navi10_ih_sw_init()
575 	adev->irq.ih1.ring_size = 0;  in navi10_ih_sw_init()
576 	adev->irq.ih2.ring_size = 0;  in navi10_ih_sw_init()
581 	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true);  in navi10_ih_sw_init()
590 static int navi10_ih_sw_fini(void *handle)  in navi10_ih_sw_fini()
599 static int navi10_ih_hw_init(void *handle)  in navi10_ih_hw_init()
606 static int navi10_ih_hw_fini(void *handle)  in navi10_ih_hw_fini()
615 static int navi10_ih_suspend(void *handle)  in navi10_ih_suspend()
622 static int navi10_ih_resume(void *handle)  in navi10_ih_resume()
629 static bool navi10_ih_is_idle(void *handle)  in navi10_ih_is_idle()
635 static int navi10_ih_wait_for_idle(void *handle)  in navi10_ih_wait_for_idle()
638 	return -ETIMEDOUT;  in navi10_ih_wait_for_idle()
641 static int navi10_ih_soft_reset(void *handle)  in navi10_ih_soft_reset()
647 static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev,  in navi10_ih_update_clockgating_state()
648 					       bool enable)  in navi10_ih_update_clockgating_state()  argument
652 	if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {  in navi10_ih_update_clockgating_state()
654 		field_val = enable ? 0 : 1;  in navi10_ih_update_clockgating_state()
672 static int navi10_ih_set_clockgating_state(void *handle,  in navi10_ih_set_clockgating_state()
682 static int navi10_ih_set_powergating_state(void *handle,  in navi10_ih_set_powergating_state()
688 static void navi10_ih_get_clockgating_state(void *handle, u64 *flags)  in navi10_ih_get_clockgating_state()
698 static const struct amd_ip_funcs navi10_ih_ip_funcs = {
716 static const struct amdgpu_ih_funcs navi10_ih_funcs = {
723 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev)  in navi10_ih_set_interrupt_funcs()
725 	if (adev->irq.ih_funcs == NULL)  in navi10_ih_set_interrupt_funcs()
726 		adev->irq.ih_funcs = &navi10_ih_funcs;  in navi10_ih_set_interrupt_funcs()