38  * Most engines on the GPU are fed via ring buffers.  Ring
44  * pointers are equal, the ring is idle.  When the host
45  * writes commands to the ring buffer, it increments the
49 static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
52  * radeon_ring_supports_scratch_reg - check if the ring supports
56  * @ring: radeon_ring structure holding ring information
58  * Check if a specific ring supports writing to scratch registers (all asics).
59  * Returns true if the ring supports writing to scratch regs, false if not.
62 				      struct radeon_ring *ring)  in radeon_ring_supports_scratch_reg()  argument
64 	switch (ring->idx) {  in radeon_ring_supports_scratch_reg()
78  * @ring: radeon_ring structure holding ring information
80  * Update the free dw slots in the ring buffer (all asics).
82 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_free_size()  argument
84 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_ring_free_size()
87 	ring->ring_free_dw = rptr + (ring->ring_size / 4);  in radeon_ring_free_size()
88 	ring->ring_free_dw -= ring->wptr;  in radeon_ring_free_size()
89 	ring->ring_free_dw &= ring->ptr_mask;  in radeon_ring_free_size()
90 	if (!ring->ring_free_dw) {  in radeon_ring_free_size()
91 		/* this is an empty ring */  in radeon_ring_free_size()
92 		ring->ring_free_dw = ring->ring_size / 4;  in radeon_ring_free_size()
94 		radeon_ring_lockup_update(rdev, ring);  in radeon_ring_free_size()
99  * radeon_ring_alloc - allocate space on the ring buffer
102  * @ring: radeon_ring structure holding ring information
103  * @ndw: number of dwords to allocate in the ring buffer
105  * Allocate @ndw dwords in the ring buffer (all asics).
108 int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)  in radeon_ring_alloc()  argument
112 	/* make sure we aren't trying to allocate more space than there is on the ring */  in radeon_ring_alloc()
113 	if (ndw > (ring->ring_size / 4))  in radeon_ring_alloc()
117 	radeon_ring_free_size(rdev, ring);  in radeon_ring_alloc()
118 	ndw = (ndw + ring->align_mask) & ~ring->align_mask;  in radeon_ring_alloc()
119 	while (ndw > (ring->ring_free_dw - 1)) {  in radeon_ring_alloc()
120 		radeon_ring_free_size(rdev, ring);  in radeon_ring_alloc()
121 		if (ndw < ring->ring_free_dw) {  in radeon_ring_alloc()
124 		r = radeon_fence_wait_next(rdev, ring->idx);  in radeon_ring_alloc()
128 	ring->count_dw = ndw;  in radeon_ring_alloc()
129 	ring->wptr_old = ring->wptr;  in radeon_ring_alloc()
134  * radeon_ring_lock - lock the ring and allocate space on it
137  * @ring: radeon_ring structure holding ring information
138  * @ndw: number of dwords to allocate in the ring buffer
140  * Lock the ring and allocate @ndw dwords in the ring buffer
144 int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)  in radeon_ring_lock()  argument
149 	r = radeon_ring_alloc(rdev, ring, ndw);  in radeon_ring_lock()
159  * commands on the ring buffer
162  * @ring: radeon_ring structure holding ring information
166  * execute new commands on the ring buffer (all asics).
168 void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_commit()  argument
171 	/* If we are emitting the HDP flush via the ring buffer, we need to  in radeon_ring_commit()
174 	if (hdp_flush && rdev->asic->ring[ring->idx]->hdp_flush)  in radeon_ring_commit()
175 		rdev->asic->ring[ring->idx]->hdp_flush(rdev, ring);  in radeon_ring_commit()
177 	while (ring->wptr & ring->align_mask) {  in radeon_ring_commit()
178 		radeon_ring_write(ring, ring->nop);  in radeon_ring_commit()
186 	radeon_ring_set_wptr(rdev, ring);  in radeon_ring_commit()
191  * commands on the ring buffer and unlock it
194  * @ring: radeon_ring structure holding ring information
197  * Call radeon_ring_commit() then unlock the ring (all asics).
199 void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_unlock_commit()  argument
202 	radeon_ring_commit(rdev, ring, hdp_flush);  in radeon_ring_unlock_commit()
209  * @ring: radeon_ring structure holding ring information
213 void radeon_ring_undo(struct radeon_ring *ring)  in radeon_ring_undo()  argument
215 	ring->wptr = ring->wptr_old;  in radeon_ring_undo()
219  * radeon_ring_unlock_undo - reset the wptr and unlock the ring
221  * @ring: radeon_ring structure holding ring information
223  * Call radeon_ring_undo() then unlock the ring (all asics).
225 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_unlock_undo()  argument
227 	radeon_ring_undo(ring);  in radeon_ring_unlock_undo()
234  * @ring: radeon_ring structure holding ring information
239 			       struct radeon_ring *ring)  in radeon_ring_lockup_update()  argument
241 	atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring));  in radeon_ring_lockup_update()
242 	atomic64_set(&ring->last_activity, jiffies_64);  in radeon_ring_lockup_update()
246  * radeon_ring_test_lockup() - check if ring is lockedup by recording information
248  * @ring:       radeon_ring structure holding ring information
251 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_test_lockup()  argument
253 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_ring_test_lockup()
254 	uint64_t last = atomic64_read(&ring->last_activity);  in radeon_ring_test_lockup()
257 	if (rptr != atomic_read(&ring->last_rptr)) {  in radeon_ring_test_lockup()
258 		/* ring is still working, no lockup */  in radeon_ring_test_lockup()
259 		radeon_ring_lockup_update(rdev, ring);  in radeon_ring_test_lockup()
265 		dev_err(rdev->dev, "ring %d stalled for more than %llumsec\n",  in radeon_ring_test_lockup()
266 			ring->idx, elapsed);  in radeon_ring_test_lockup()
274  * radeon_ring_backup - Back up the content of a ring
277  * @ring: the ring we want to back up
279  * Saves all unprocessed commits from a ring, returns the number of dwords saved.
281 unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_backup()  argument
286 	/* just in case lock the ring */  in radeon_ring_backup()
290 	if (ring->ring_obj == NULL) {  in radeon_ring_backup()
296 	if (!radeon_fence_count_emitted(rdev, ring->idx)) {  in radeon_ring_backup()
301 	/* calculate the number of dw on the ring */  in radeon_ring_backup()
302 	if (ring->rptr_save_reg)  in radeon_ring_backup()
303 		ptr = RREG32(ring->rptr_save_reg);  in radeon_ring_backup()
305 		ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);  in radeon_ring_backup()
312 	size = ring->wptr + (ring->ring_size / 4);  in radeon_ring_backup()
314 	size &= ring->ptr_mask;  in radeon_ring_backup()
320 	/* and then save the content of the ring */  in radeon_ring_backup()
327 		(*data)[i] = ring->ring[ptr++];  in radeon_ring_backup()
328 		ptr &= ring->ptr_mask;  in radeon_ring_backup()
336  * radeon_ring_restore - append saved commands to the ring again
339  * @ring: ring to append commands to
343  * Allocates space on the ring and restore the previously saved commands.
345 int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_restore()  argument
353 	/* restore the saved ring content */  in radeon_ring_restore()
354 	r = radeon_ring_lock(rdev, ring, size);  in radeon_ring_restore()
359 		radeon_ring_write(ring, data[i]);  in radeon_ring_restore()
362 	radeon_ring_unlock_commit(rdev, ring, false);  in radeon_ring_restore()
368  * radeon_ring_init - init driver ring struct.
371  * @ring: radeon_ring structure holding ring information
372  * @ring_size: size of the ring
374  * @nop: nop packet for this ring
376  * Initialize the driver information for the selected ring (all asics).
379 int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,  in radeon_ring_init()  argument
384 	ring->ring_size = ring_size;  in radeon_ring_init()
385 	ring->rptr_offs = rptr_offs;  in radeon_ring_init()
386 	ring->nop = nop;  in radeon_ring_init()
387 	/* Allocate ring buffer */  in radeon_ring_init()
388 	if (ring->ring_obj == NULL) {  in radeon_ring_init()
389 		r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,  in radeon_ring_init()
391 				     NULL, &ring->ring_obj);  in radeon_ring_init()
393 			dev_err(rdev->dev, "(%d) ring create failed\n", r);  in radeon_ring_init()
396 		r = radeon_bo_reserve(ring->ring_obj, false);  in radeon_ring_init()
399 		r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,  in radeon_ring_init()
400 					&ring->gpu_addr);  in radeon_ring_init()
402 			radeon_bo_unreserve(ring->ring_obj);  in radeon_ring_init()
403 			dev_err(rdev->dev, "(%d) ring pin failed\n", r);  in radeon_ring_init()
406 		r = radeon_bo_kmap(ring->ring_obj,  in radeon_ring_init()
407 				       (void **)&ring->ring);  in radeon_ring_init()
408 		radeon_bo_unreserve(ring->ring_obj);  in radeon_ring_init()
410 			dev_err(rdev->dev, "(%d) ring map failed\n", r);  in radeon_ring_init()
414 	ring->ptr_mask = (ring->ring_size / 4) - 1;  in radeon_ring_init()
415 	ring->ring_free_dw = ring->ring_size / 4;  in radeon_ring_init()
417 		u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);  in radeon_ring_init()
418 		ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;  in radeon_ring_init()
419 		ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];  in radeon_ring_init()
421 	if (radeon_debugfs_ring_init(rdev, ring)) {  in radeon_ring_init()
424 	radeon_ring_lockup_update(rdev, ring);  in radeon_ring_init()
429  * radeon_ring_fini - tear down the driver ring struct.
432  * @ring: radeon_ring structure holding ring information
434  * Tear down the driver information for the selected ring (all asics).
436 void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_fini()  argument
442 	ring_obj = ring->ring_obj;  in radeon_ring_fini()
443 	ring->ready = false;  in radeon_ring_fini()
444 	ring->ring = NULL;  in radeon_ring_fini()
445 	ring->ring_obj = NULL;  in radeon_ring_fini()
470 	struct radeon_ring *ring = &rdev->ring[ridx];  in radeon_debugfs_ring_info()  local
475 	radeon_ring_free_size(rdev, ring);  in radeon_debugfs_ring_info()
476 	count = (ring->ring_size / 4) - ring->ring_free_dw;  in radeon_debugfs_ring_info()
478 	wptr = radeon_ring_get_wptr(rdev, ring);  in radeon_debugfs_ring_info()
482 	rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_debugfs_ring_info()
486 	if (ring->rptr_save_reg) {  in radeon_debugfs_ring_info()
487 		rptr_next = RREG32(ring->rptr_save_reg);  in radeon_debugfs_ring_info()
489 			   ring->rptr_save_reg, rptr_next, rptr_next);  in radeon_debugfs_ring_info()
494 		   ring->wptr, ring->wptr);  in radeon_debugfs_ring_info()
496 		   ring->last_semaphore_signal_addr);  in radeon_debugfs_ring_info()
498 		   ring->last_semaphore_wait_addr);  in radeon_debugfs_ring_info()
499 	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);  in radeon_debugfs_ring_info()
500 	seq_printf(m, "%u dwords in ring\n", count);  in radeon_debugfs_ring_info()
502 	if (!ring->ring)  in radeon_debugfs_ring_info()
508 	i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;  in radeon_debugfs_ring_info()
510 		seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);  in radeon_debugfs_ring_info()
516 		i = (i + 1) & ring->ptr_mask;  in radeon_debugfs_ring_info()
543 static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_debugfs_ring_init()  argument
552 		if (&rdev->ring[ridx] != ring)  in radeon_debugfs_ring_init()