36 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
45 static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)  in vega10_ih_enable_interrupts()  argument
52 	adev->irq.ih.enabled = true;  in vega10_ih_enable_interrupts()
62 static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)  in vega10_ih_disable_interrupts()  argument
72 	adev->irq.ih.enabled = false;  in vega10_ih_disable_interrupts()
73 	adev->irq.ih.rptr = 0;  in vega10_ih_disable_interrupts()
87 static int vega10_ih_irq_init(struct amdgpu_device *adev)  in vega10_ih_irq_init()  argument
96 	vega10_ih_disable_interrupts(adev);  in vega10_ih_irq_init()
98 	adev->nbio_funcs->ih_control(adev);  in vega10_ih_irq_init()
102 	if (adev->irq.ih.use_bus_addr) {  in vega10_ih_irq_init()
103 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);  in vega10_ih_irq_init()
104 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);  in vega10_ih_irq_init()
107 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);  in vega10_ih_irq_init()
108 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);  in vega10_ih_irq_init()
111 	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);  in vega10_ih_irq_init()
121 	if (adev->irq.msi_enabled)  in vega10_ih_irq_init()
127 	if (adev->irq.ih.use_bus_addr)  in vega10_ih_irq_init()
128 		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);  in vega10_ih_irq_init()
130 		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);  in vega10_ih_irq_init()
139 	if (adev->irq.ih.use_doorbell) {  in vega10_ih_irq_init()
141 						 OFFSET, adev->irq.ih.doorbell_index);  in vega10_ih_irq_init()
149 	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,  in vega10_ih_irq_init()
150 					    adev->irq.ih.doorbell_index);  in vega10_ih_irq_init()
161 	pci_set_master(adev->pdev);  in vega10_ih_irq_init()
164 	vega10_ih_enable_interrupts(adev);  in vega10_ih_irq_init()
176 static void vega10_ih_irq_disable(struct amdgpu_device *adev)  in vega10_ih_irq_disable()  argument
178 	vega10_ih_disable_interrupts(adev);  in vega10_ih_irq_disable()
194 static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)  in vega10_ih_get_wptr()  argument
198 	if (adev->irq.ih.use_bus_addr)  in vega10_ih_get_wptr()
199 		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);  in vega10_ih_get_wptr()
201 		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);  in vega10_ih_get_wptr()
210 		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;  in vega10_ih_get_wptr()
211 		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in vega10_ih_get_wptr()
212 			wptr, adev->irq.ih.rptr, tmp);  in vega10_ih_get_wptr()
213 		adev->irq.ih.rptr = tmp;  in vega10_ih_get_wptr()
219 	return (wptr & adev->irq.ih.ptr_mask);  in vega10_ih_get_wptr()
229 static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)  in vega10_ih_prescreen_iv()  argument
231 	u32 ring_index = adev->irq.ih.rptr >> 2;  in vega10_ih_prescreen_iv()
238 	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);  in vega10_ih_prescreen_iv()
239 	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);  in vega10_ih_prescreen_iv()
240 	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);  in vega10_ih_prescreen_iv()
241 	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);  in vega10_ih_prescreen_iv()
263 		if (!amdgpu_vm_pasid_fault_credit(adev, pasid))  in vega10_ih_prescreen_iv()
270 	r = amdgpu_ih_add_fault(adev, key);  in vega10_ih_prescreen_iv()
279 	spin_lock(&adev->vm_manager.pasid_lock);  in vega10_ih_prescreen_iv()
280 	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);  in vega10_ih_prescreen_iv()
283 		spin_unlock(&adev->vm_manager.pasid_lock);  in vega10_ih_prescreen_iv()
284 		amdgpu_ih_clear_fault(adev, key);  in vega10_ih_prescreen_iv()
291 		spin_unlock(&adev->vm_manager.pasid_lock);  in vega10_ih_prescreen_iv()
292 		amdgpu_ih_clear_fault(adev, key);  in vega10_ih_prescreen_iv()
295 	spin_unlock(&adev->vm_manager.pasid_lock);  in vega10_ih_prescreen_iv()
301 	adev->irq.ih.rptr += 32;  in vega10_ih_prescreen_iv()
313 static void vega10_ih_decode_iv(struct amdgpu_device *adev,  in vega10_ih_decode_iv()  argument
317 	u32 ring_index = adev->irq.ih.rptr >> 2;  in vega10_ih_decode_iv()
320 	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);  in vega10_ih_decode_iv()
321 	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);  in vega10_ih_decode_iv()
322 	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);  in vega10_ih_decode_iv()
323 	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);  in vega10_ih_decode_iv()
324 	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);  in vega10_ih_decode_iv()
325 	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);  in vega10_ih_decode_iv()
326 	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);  in vega10_ih_decode_iv()
327 	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);  in vega10_ih_decode_iv()
345 	adev->irq.ih.rptr += 32;  in vega10_ih_decode_iv()
355 static void vega10_ih_set_rptr(struct amdgpu_device *adev)  in vega10_ih_set_rptr()  argument
357 	if (adev->irq.ih.use_doorbell) {  in vega10_ih_set_rptr()
359 		if (adev->irq.ih.use_bus_addr)  in vega10_ih_set_rptr()
360 			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;  in vega10_ih_set_rptr()
362 			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;  in vega10_ih_set_rptr()
363 		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);  in vega10_ih_set_rptr()
365 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);  in vega10_ih_set_rptr()
371 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in vega10_ih_early_init()  local
373 	vega10_ih_set_interrupt_funcs(adev);  in vega10_ih_early_init()
380 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in vega10_ih_sw_init()  local
382 	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);  in vega10_ih_sw_init()
386 	adev->irq.ih.use_doorbell = true;  in vega10_ih_sw_init()
387 	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;  in vega10_ih_sw_init()
389 	adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);  in vega10_ih_sw_init()
390 	if (!adev->irq.ih.faults)  in vega10_ih_sw_init()
392 	INIT_CHASH_TABLE(adev->irq.ih.faults->hash,  in vega10_ih_sw_init()
394 	spin_lock_init(&adev->irq.ih.faults->lock);  in vega10_ih_sw_init()
395 	adev->irq.ih.faults->count = 0;  in vega10_ih_sw_init()
397 	r = amdgpu_irq_init(adev);  in vega10_ih_sw_init()
404 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in vega10_ih_sw_fini()  local
406 	amdgpu_irq_fini(adev);  in vega10_ih_sw_fini()
407 	amdgpu_ih_ring_fini(adev);  in vega10_ih_sw_fini()
409 	kfree(adev->irq.ih.faults);  in vega10_ih_sw_fini()
410 	adev->irq.ih.faults = NULL;  in vega10_ih_sw_fini()
418 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in vega10_ih_hw_init()  local
420 	r = vega10_ih_irq_init(adev);  in vega10_ih_hw_init()
429 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in vega10_ih_hw_fini()  local
431 	vega10_ih_irq_disable(adev);  in vega10_ih_hw_fini()
438 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in vega10_ih_suspend()  local
440 	return vega10_ih_hw_fini(adev);  in vega10_ih_suspend()
445 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in vega10_ih_resume()  local
447 	return vega10_ih_hw_init(adev);  in vega10_ih_resume()
505 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)  in vega10_ih_set_interrupt_funcs()  argument
507 	if (adev->irq.ih_funcs == NULL)  in vega10_ih_set_interrupt_funcs()
508 		adev->irq.ih_funcs = &vega10_ih_funcs;  in vega10_ih_set_interrupt_funcs()