41 int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,  in amdgpu_ih_ring_init()  argument
50 	ih->ring_size = ring_size;  in amdgpu_ih_ring_init()
51 	ih->ptr_mask = ih->ring_size - 1;  in amdgpu_ih_ring_init()
52 	ih->rptr = 0;  in amdgpu_ih_ring_init()
53 	ih->use_bus_addr = use_bus_addr;  in amdgpu_ih_ring_init()
58 		if (ih->ring)  in amdgpu_ih_ring_init()
64 		ih->ring = dma_alloc_coherent(adev->dev, ih->ring_size + 8,  in amdgpu_ih_ring_init()
66 		if (ih->ring == NULL)  in amdgpu_ih_ring_init()
69 		ih->gpu_addr = dma_addr;  in amdgpu_ih_ring_init()
70 		ih->wptr_addr = dma_addr + ih->ring_size;  in amdgpu_ih_ring_init()
71 		ih->wptr_cpu = &ih->ring[ih->ring_size / 4];  in amdgpu_ih_ring_init()
72 		ih->rptr_addr = dma_addr + ih->ring_size + 4;  in amdgpu_ih_ring_init()
73 		ih->rptr_cpu = &ih->ring[(ih->ring_size / 4) + 1];  in amdgpu_ih_ring_init()
87 		r = amdgpu_bo_create_kernel(adev, ih->ring_size, PAGE_SIZE,  in amdgpu_ih_ring_init()
89 					    &ih->ring_obj, &ih->gpu_addr,  in amdgpu_ih_ring_init()
90 					    (void **)&ih->ring);  in amdgpu_ih_ring_init()
97 		ih->wptr_addr = adev->wb.gpu_addr + wptr_offs * 4;  in amdgpu_ih_ring_init()
98 		ih->wptr_cpu = &adev->wb.wb[wptr_offs];  in amdgpu_ih_ring_init()
99 		ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4;  in amdgpu_ih_ring_init()
100 		ih->rptr_cpu = &adev->wb.wb[rptr_offs];  in amdgpu_ih_ring_init()
103 	init_waitqueue_head(&ih->wait_process);  in amdgpu_ih_ring_init()
116 void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)  in amdgpu_ih_ring_fini()  argument
119 	if (!ih->ring)  in amdgpu_ih_ring_fini()
122 	if (ih->use_bus_addr) {  in amdgpu_ih_ring_fini()
127 		dma_free_coherent(adev->dev, ih->ring_size + 8,  in amdgpu_ih_ring_fini()
128 				  (void *)ih->ring, ih->gpu_addr);  in amdgpu_ih_ring_fini()
129 		ih->ring = NULL;  in amdgpu_ih_ring_fini()
131 		amdgpu_bo_free_kernel(&ih->ring_obj, &ih->gpu_addr,  in amdgpu_ih_ring_fini()
132 				      (void **)&ih->ring);  in amdgpu_ih_ring_fini()
133 		amdgpu_device_wb_free(adev, (ih->wptr_addr - ih->gpu_addr) / 4);  in amdgpu_ih_ring_fini()
134 		amdgpu_device_wb_free(adev, (ih->rptr_addr - ih->gpu_addr) / 4);  in amdgpu_ih_ring_fini()
148 void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv,  in amdgpu_ih_ring_write()  argument
151 	uint32_t wptr = le32_to_cpu(*ih->wptr_cpu) >> 2;  in amdgpu_ih_ring_write()
155 	        ih->ring[wptr++] = cpu_to_le32(iv[i]);  in amdgpu_ih_ring_write()
158 	wptr &= ih->ptr_mask;  in amdgpu_ih_ring_write()
161 	if (wptr != READ_ONCE(ih->rptr)) {  in amdgpu_ih_ring_write()
163 		WRITE_ONCE(*ih->wptr_cpu, cpu_to_le32(wptr));  in amdgpu_ih_ring_write()
176 					struct amdgpu_ih_ring *ih)  in amdgpu_ih_wait_on_checkpoint_process_ts()  argument
182 	if (!ih->enabled || adev->shutdown)  in amdgpu_ih_wait_on_checkpoint_process_ts()
185 	checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);  in amdgpu_ih_wait_on_checkpoint_process_ts()
188 	checkpoint_ts = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1);  in amdgpu_ih_wait_on_checkpoint_process_ts()
190 	return wait_event_interruptible_timeout(ih->wait_process,  in amdgpu_ih_wait_on_checkpoint_process_ts()
191 		    amdgpu_ih_ts_after(checkpoint_ts, ih->processed_timestamp) ||  in amdgpu_ih_wait_on_checkpoint_process_ts()
192 		    ih->rptr == amdgpu_ih_get_wptr(adev, ih), timeout);  in amdgpu_ih_wait_on_checkpoint_process_ts()
204 int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)  in amdgpu_ih_process()  argument
209 	if (!ih->enabled || adev->shutdown)  in amdgpu_ih_process()
212 	wptr = amdgpu_ih_get_wptr(adev, ih);  in amdgpu_ih_process()
216 	DRM_DEBUG("%s: rptr %d, wptr %d\n", __func__, ih->rptr, wptr);  in amdgpu_ih_process()
221 	while (ih->rptr != wptr && --count) {  in amdgpu_ih_process()
222 		amdgpu_irq_dispatch(adev, ih);  in amdgpu_ih_process()
223 		ih->rptr &= ih->ptr_mask;  in amdgpu_ih_process()
226 	amdgpu_ih_set_rptr(adev, ih);  in amdgpu_ih_process()
227 	wake_up_all(&ih->wait_process);  in amdgpu_ih_process()
230 	wptr = amdgpu_ih_get_wptr(adev, ih);  in amdgpu_ih_process()
231 	if (wptr != ih->rptr)  in amdgpu_ih_process()
249 				struct amdgpu_ih_ring *ih,  in amdgpu_ih_decode_iv_helper()  argument
253 	u32 ring_index = ih->rptr >> 2;  in amdgpu_ih_decode_iv_helper()
256 	dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);  in amdgpu_ih_decode_iv_helper()
257 	dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);  in amdgpu_ih_decode_iv_helper()
258 	dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);  in amdgpu_ih_decode_iv_helper()
259 	dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);  in amdgpu_ih_decode_iv_helper()
260 	dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);  in amdgpu_ih_decode_iv_helper()
261 	dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);  in amdgpu_ih_decode_iv_helper()
262 	dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);  in amdgpu_ih_decode_iv_helper()
263 	dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);  in amdgpu_ih_decode_iv_helper()
280 	ih->rptr += 32;  in amdgpu_ih_decode_iv_helper()
283 uint64_t amdgpu_ih_decode_iv_ts_helper(struct amdgpu_ih_ring *ih, u32 rptr,  in amdgpu_ih_decode_iv_ts_helper()  argument
291 	ring_index = (rptr & ih->ptr_mask) >> 2;  in amdgpu_ih_decode_iv_ts_helper()
293 	dw1 = le32_to_cpu(ih->ring[ring_index + 1]);  in amdgpu_ih_decode_iv_ts_helper()
294 	dw2 = le32_to_cpu(ih->ring[ring_index + 2]);  in amdgpu_ih_decode_iv_ts_helper()