49 static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev);
58 static void cik_ih_enable_interrupts(struct amdgpu_device *adev)  in cik_ih_enable_interrupts()  argument
67 	adev->irq.ih.enabled = true;  in cik_ih_enable_interrupts()
77 static void cik_ih_disable_interrupts(struct amdgpu_device *adev)  in cik_ih_disable_interrupts()  argument
89 	adev->irq.ih.enabled = false;  in cik_ih_disable_interrupts()
90 	adev->irq.ih.rptr = 0;  in cik_ih_disable_interrupts()
104 static int cik_ih_irq_init(struct amdgpu_device *adev)  in cik_ih_irq_init()  argument
111 	cik_ih_disable_interrupts(adev);  in cik_ih_irq_init()
114 	WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);  in cik_ih_irq_init()
124 	WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);  in cik_ih_irq_init()
125 	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);  in cik_ih_irq_init()
134 	wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);  in cik_ih_irq_init()
149 	if (adev->irq.msi_enabled)  in cik_ih_irq_init()
153 	pci_set_master(adev->pdev);  in cik_ih_irq_init()
156 	cik_ih_enable_interrupts(adev);  in cik_ih_irq_init()
168 static void cik_ih_irq_disable(struct amdgpu_device *adev)  in cik_ih_irq_disable()  argument
170 	cik_ih_disable_interrupts(adev);  in cik_ih_irq_disable()
186 static u32 cik_ih_get_wptr(struct amdgpu_device *adev)  in cik_ih_get_wptr()  argument
190 	wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);  in cik_ih_get_wptr()
198 		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in cik_ih_get_wptr()
199 			wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask);  in cik_ih_get_wptr()
200 		adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask;  in cik_ih_get_wptr()
205 	return (wptr & adev->irq.ih.ptr_mask);  in cik_ih_get_wptr()
238 static bool cik_ih_prescreen_iv(struct amdgpu_device *adev)  in cik_ih_prescreen_iv()  argument
240 	u32 ring_index = adev->irq.ih.rptr >> 2;  in cik_ih_prescreen_iv()
243 	switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {  in cik_ih_prescreen_iv()
246 		pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;  in cik_ih_prescreen_iv()
247 		if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))  in cik_ih_prescreen_iv()
255 	adev->irq.ih.rptr += 16;  in cik_ih_prescreen_iv()
267 static void cik_ih_decode_iv(struct amdgpu_device *adev,  in cik_ih_decode_iv()  argument
271 	u32 ring_index = adev->irq.ih.rptr >> 2;  in cik_ih_decode_iv()
274 	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);  in cik_ih_decode_iv()
275 	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);  in cik_ih_decode_iv()
276 	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);  in cik_ih_decode_iv()
277 	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);  in cik_ih_decode_iv()
287 	adev->irq.ih.rptr += 16;  in cik_ih_decode_iv()
297 static void cik_ih_set_rptr(struct amdgpu_device *adev)  in cik_ih_set_rptr()  argument
299 	WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr);  in cik_ih_set_rptr()
304 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_early_init()  local
307 	ret = amdgpu_irq_add_domain(adev);  in cik_ih_early_init()
311 	cik_ih_set_interrupt_funcs(adev);  in cik_ih_early_init()
319 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_sw_init()  local
321 	r = amdgpu_ih_ring_init(adev, 64 * 1024, false);  in cik_ih_sw_init()
325 	r = amdgpu_irq_init(adev);  in cik_ih_sw_init()
332 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_sw_fini()  local
334 	amdgpu_irq_fini(adev);  in cik_ih_sw_fini()
335 	amdgpu_ih_ring_fini(adev);  in cik_ih_sw_fini()
336 	amdgpu_irq_remove_domain(adev);  in cik_ih_sw_fini()
344 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_hw_init()  local
346 	r = cik_ih_irq_init(adev);  in cik_ih_hw_init()
355 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_hw_fini()  local
357 	cik_ih_irq_disable(adev);  in cik_ih_hw_fini()
364 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_suspend()  local
366 	return cik_ih_hw_fini(adev);  in cik_ih_suspend()
371 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_resume()  local
373 	return cik_ih_hw_init(adev);  in cik_ih_resume()
378 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_is_idle()  local
391 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_wait_for_idle()  local
393 	for (i = 0; i < adev->usec_timeout; i++) {  in cik_ih_wait_for_idle()
405 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_ih_soft_reset()  local
416 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_ih_soft_reset()
469 static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev)  in cik_ih_set_interrupt_funcs()  argument
471 	if (adev->irq.ih_funcs == NULL)  in cik_ih_set_interrupt_funcs()
472 		adev->irq.ih_funcs = &cik_ih_funcs;  in cik_ih_set_interrupt_funcs()