43  * and each one supports 1 ring buffer used for gfx
47  * (ring buffer, IBs, etc.), but sDMA has it's own
59  * @ring: radeon ring pointer
64 			   struct radeon_ring *ring)  in cik_sdma_get_rptr()  argument
69 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_sdma_get_rptr()
71 		if (ring->idx == R600_RING_TYPE_DMA_INDEX)  in cik_sdma_get_rptr()
86  * @ring: radeon ring pointer
91 			   struct radeon_ring *ring)  in cik_sdma_get_wptr()  argument
95 	if (ring->idx == R600_RING_TYPE_DMA_INDEX)  in cik_sdma_get_wptr()
107  * @ring: radeon ring pointer
112 		       struct radeon_ring *ring)  in cik_sdma_set_wptr()  argument
116 	if (ring->idx == R600_RING_TYPE_DMA_INDEX)  in cik_sdma_set_wptr()
121 	WREG32(reg, (ring->wptr << 2) & 0x3fffc);  in cik_sdma_set_wptr()
131  * Schedule an IB in the DMA ring (CIK).
136 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cik_sdma_ring_ib_execute()  local
137 	u32 extra_bits = (ib->vm ? ib->vm->ids[ib->ring].id : 0) & 0xf;  in cik_sdma_ring_ib_execute()
140 		u32 next_rptr = ring->wptr + 5;  in cik_sdma_ring_ib_execute()
144 		radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));  in cik_sdma_ring_ib_execute()
145 		radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);  in cik_sdma_ring_ib_execute()
146 		radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));  in cik_sdma_ring_ib_execute()
147 		radeon_ring_write(ring, 1); /* number of DWs to follow */  in cik_sdma_ring_ib_execute()
148 		radeon_ring_write(ring, next_rptr);  in cik_sdma_ring_ib_execute()
152 	while ((ring->wptr & 7) != 4)  in cik_sdma_ring_ib_execute()
153 		radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));  in cik_sdma_ring_ib_execute()
154 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));  in cik_sdma_ring_ib_execute()
155 	radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */  in cik_sdma_ring_ib_execute()
156 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr));  in cik_sdma_ring_ib_execute()
157 	radeon_ring_write(ring, ib->length_dw);  in cik_sdma_ring_ib_execute()
162  * cik_sdma_hdp_flush_ring_emit - emit an hdp flush on the DMA ring
165  * @ridx: radeon ring index
167  * Emit an hdp flush packet on the requested DMA ring.
172 	struct radeon_ring *ring = &rdev->ring[ridx];  in cik_sdma_hdp_flush_ring_emit()  local
182 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));  in cik_sdma_hdp_flush_ring_emit()
183 	radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);  in cik_sdma_hdp_flush_ring_emit()
184 	radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);  in cik_sdma_hdp_flush_ring_emit()
185 	radeon_ring_write(ring, ref_and_mask); /* reference */  in cik_sdma_hdp_flush_ring_emit()
186 	radeon_ring_write(ring, ref_and_mask); /* mask */  in cik_sdma_hdp_flush_ring_emit()
187 	radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */  in cik_sdma_hdp_flush_ring_emit()
191  * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
196  * Add a DMA fence packet to the ring to write
203 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_sdma_fence_ring_emit()  local
204 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_sdma_fence_ring_emit()
207 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));  in cik_sdma_fence_ring_emit()
208 	radeon_ring_write(ring, lower_32_bits(addr));  in cik_sdma_fence_ring_emit()
209 	radeon_ring_write(ring, upper_32_bits(addr));  in cik_sdma_fence_ring_emit()
210 	radeon_ring_write(ring, fence->seq);  in cik_sdma_fence_ring_emit()
212 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));  in cik_sdma_fence_ring_emit()
214 	cik_sdma_hdp_flush_ring_emit(rdev, fence->ring);  in cik_sdma_fence_ring_emit()
218  * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
221  * @ring: radeon_ring structure holding ring information
225  * Add a DMA semaphore packet to the ring wait on or signal
229 				  struct radeon_ring *ring,  in cik_sdma_semaphore_ring_emit()  argument
236 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));  in cik_sdma_semaphore_ring_emit()
237 	radeon_ring_write(ring, addr & 0xfffffff8);  in cik_sdma_semaphore_ring_emit()
238 	radeon_ring_write(ring, upper_32_bits(addr));  in cik_sdma_semaphore_ring_emit()
248  * Stop the gfx async dma ring buffers (CIK).
269 	rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;  in cik_sdma_gfx_stop()
270 	rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;  in cik_sdma_gfx_stop()
362  * Set up the gfx DMA ring buffers and enable them (CIK).
367 	struct radeon_ring *ring;  in cik_sdma_gfx_resume()  local
375 			ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_sdma_gfx_resume()
379 			ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_sdma_gfx_resume()
387 		/* Set ring buffer size in dwords */  in cik_sdma_gfx_resume()
388 		rb_bufsz = order_base_2(ring->ring_size / 4);  in cik_sdma_gfx_resume()
395 		/* Initialize the ring buffer's read and write pointers */  in cik_sdma_gfx_resume()
408 		WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8);  in cik_sdma_gfx_resume()
409 		WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40);  in cik_sdma_gfx_resume()
411 		ring->wptr = 0;  in cik_sdma_gfx_resume()
412 		WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);  in cik_sdma_gfx_resume()
424 		ring->ready = true;  in cik_sdma_gfx_resume()
426 		r = radeon_ring_test(rdev, ring->idx, ring);  in cik_sdma_gfx_resume()
428 			ring->ready = false;  in cik_sdma_gfx_resume()
561 	radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);  in cik_sdma_fini()
562 	radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);  in cik_sdma_fini()
587 	struct radeon_ring *ring = &rdev->ring[ring_index];  in cik_copy_dma()  local
596 	r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);  in cik_copy_dma()
604 	radeon_sync_rings(rdev, &sync, ring->idx);  in cik_copy_dma()
611 		radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));  in cik_copy_dma()
612 		radeon_ring_write(ring, cur_size_in_bytes);  in cik_copy_dma()
613 		radeon_ring_write(ring, 0); /* src/dst endian swap */  in cik_copy_dma()
614 		radeon_ring_write(ring, lower_32_bits(src_offset));  in cik_copy_dma()
615 		radeon_ring_write(ring, upper_32_bits(src_offset));  in cik_copy_dma()
616 		radeon_ring_write(ring, lower_32_bits(dst_offset));  in cik_copy_dma()
617 		radeon_ring_write(ring, upper_32_bits(dst_offset));  in cik_copy_dma()
622 	r = radeon_fence_emit(rdev, &fence, ring->idx);  in cik_copy_dma()
624 		radeon_ring_unlock_undo(rdev, ring);  in cik_copy_dma()
629 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_copy_dma()
639  * @ring: radeon_ring structure holding ring information
646 		       struct radeon_ring *ring)  in cik_sdma_ring_test()  argument
654 	if (ring->idx == R600_RING_TYPE_DMA_INDEX)  in cik_sdma_ring_test()
664 	r = radeon_ring_lock(rdev, ring, 5);  in cik_sdma_ring_test()
666 		DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);  in cik_sdma_ring_test()
669 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));  in cik_sdma_ring_test()
670 	radeon_ring_write(ring, lower_32_bits(gpu_addr));  in cik_sdma_ring_test()
671 	radeon_ring_write(ring, upper_32_bits(gpu_addr));  in cik_sdma_ring_test()
672 	radeon_ring_write(ring, 1); /* number of DWs to follow */  in cik_sdma_ring_test()
673 	radeon_ring_write(ring, 0xDEADBEEF);  in cik_sdma_ring_test()
674 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_sdma_ring_test()
684 		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);  in cik_sdma_ring_test()
686 		DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",  in cik_sdma_ring_test()
687 			  ring->idx, tmp);  in cik_sdma_ring_test()
697  * @ring: radeon_ring structure holding ring information
699  * Test a simple IB in the DMA ring (CIK).
702 int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_sdma_ib_test()  argument
711 	if (ring->idx == R600_RING_TYPE_DMA_INDEX)  in cik_sdma_ib_test()
721 	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);  in cik_sdma_ib_test()
757 		DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);  in cik_sdma_ib_test()
770  * @ring: radeon_ring structure holding ring information
775 bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_sdma_is_lockup()  argument
780 	if (ring->idx == R600_RING_TYPE_DMA_INDEX)  in cik_sdma_is_lockup()
786 		radeon_ring_lockup_update(rdev, ring);  in cik_sdma_is_lockup()
789 	return radeon_ring_test_lockup(rdev, ring);  in cik_sdma_is_lockup()
947 void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,  in cik_dma_vm_flush()  argument
953 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
955 		radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);  in cik_dma_vm_flush()
957 		radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);  in cik_dma_vm_flush()
959 	radeon_ring_write(ring, pd_addr >> 12);  in cik_dma_vm_flush()
962 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
963 	radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);  in cik_dma_vm_flush()
964 	radeon_ring_write(ring, VMID(vm_id));  in cik_dma_vm_flush()
966 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
967 	radeon_ring_write(ring, SH_MEM_BASES >> 2);  in cik_dma_vm_flush()
968 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
970 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
971 	radeon_ring_write(ring, SH_MEM_CONFIG >> 2);  in cik_dma_vm_flush()
972 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
974 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
975 	radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2);  in cik_dma_vm_flush()
976 	radeon_ring_write(ring, 1);  in cik_dma_vm_flush()
978 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
979 	radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2);  in cik_dma_vm_flush()
980 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
982 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
983 	radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);  in cik_dma_vm_flush()
984 	radeon_ring_write(ring, VMID(0));  in cik_dma_vm_flush()
987 	cik_sdma_hdp_flush_ring_emit(rdev, ring->idx);  in cik_dma_vm_flush()
990 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
991 	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);  in cik_dma_vm_flush()
992 	radeon_ring_write(ring, 1 << vm_id);  in cik_dma_vm_flush()
994 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));  in cik_dma_vm_flush()
995 	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);  in cik_dma_vm_flush()
996 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
997 	radeon_ring_write(ring, 0); /* reference */  in cik_dma_vm_flush()
998 	radeon_ring_write(ring, 0); /* mask */  in cik_dma_vm_flush()
999 	radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */  in cik_dma_vm_flush()