54 	struct amdgpu_ring		*ring;  member
96 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)  in amdgpu_fence_write()  argument
98 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
112 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)  in amdgpu_fence_read()  argument
114 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
134 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,  in amdgpu_fence_emit()  argument
137 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_emit()
146 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
147 	fence->ring = ring;  in amdgpu_fence_emit()
149 		       &ring->fence_drv.lock,  in amdgpu_fence_emit()
150 		       adev->fence_context + ring->idx,  in amdgpu_fence_emit()
152 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
155 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
182 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s)  in amdgpu_fence_emit_polling()  argument
189 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit_polling()
190 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit_polling()
205 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)  in amdgpu_fence_schedule_fallback()  argument
207 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
220 void amdgpu_fence_process(struct amdgpu_ring *ring)  in amdgpu_fence_process()  argument
222 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
227 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
228 		seq = amdgpu_fence_read(ring);  in amdgpu_fence_process()
232 	if (seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
233 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_process()
274 	struct amdgpu_ring *ring = from_timer(ring, t,  in amdgpu_fence_fallback()  local
277 	amdgpu_fence_process(ring);  in amdgpu_fence_fallback()
289 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)  in amdgpu_fence_wait_empty()  argument
291 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
298 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
322 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,  in amdgpu_fence_wait_polling()  argument
329 		seq = amdgpu_fence_read(ring);  in amdgpu_fence_wait_polling()
345 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)  in amdgpu_fence_count_emitted()  argument
352 	amdgpu_fence_process(ring);  in amdgpu_fence_count_emitted()
354 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
355 	emitted += READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
372 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_start_ring()  argument
376 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_start_ring()
379 	if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {  in amdgpu_fence_driver_start_ring()
380 		ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];  in amdgpu_fence_driver_start_ring()
381 		ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);  in amdgpu_fence_driver_start_ring()
385 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;  in amdgpu_fence_driver_start_ring()
386 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;  in amdgpu_fence_driver_start_ring()
388 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
391 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
392 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
393 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
396 		"cpu addr 0x%p\n", ring->idx,  in amdgpu_fence_driver_start_ring()
397 		ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);  in amdgpu_fence_driver_start_ring()
411 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_init_ring()  argument
421 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
422 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
423 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
424 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
425 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
427 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);  in amdgpu_fence_driver_init_ring()
429 	ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
430 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
431 	ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
433 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
437 	if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ) {  in amdgpu_fence_driver_init_ring()
439 		if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)  in amdgpu_fence_driver_init_ring()
440 				&& !amdgpu_sriov_vf(ring->adev))  in amdgpu_fence_driver_init_ring()
445 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,  in amdgpu_fence_driver_init_ring()
447 				   timeout, ring->name);  in amdgpu_fence_driver_init_ring()
450 				  ring->name);  in amdgpu_fence_driver_init_ring()
492 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_fini()  local
494 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_fini()
496 		r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_fini()
499 			amdgpu_fence_driver_force_completion(ring);  in amdgpu_fence_driver_fini()
501 		amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_fini()
502 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_fini()
503 		drm_sched_fini(&ring->sched);  in amdgpu_fence_driver_fini()
504 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_fini()
505 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_fini()
506 			dma_fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_fini()
507 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_fini()
508 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_fini()
509 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_fini()
526 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_suspend()  local
527 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_suspend()
531 		r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_suspend()
534 			amdgpu_fence_driver_force_completion(ring);  in amdgpu_fence_driver_suspend()
538 		amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_suspend()
539 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_suspend()
560 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_resume()  local
561 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_resume()
565 		amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_resume()
566 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_resume()
576 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)  in amdgpu_fence_driver_force_completion()  argument
578 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
579 	amdgpu_fence_process(ring);  in amdgpu_fence_driver_force_completion()
594 	return (const char *)fence->ring->name;  in amdgpu_fence_get_timeline_name()
608 	struct amdgpu_ring *ring = fence->ring;  in amdgpu_fence_enable_signaling()  local
610 	if (!timer_pending(&ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
611 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_enable_signaling()
613 	DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);  in amdgpu_fence_enable_signaling()
664 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_debugfs_fence_info()  local
665 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info()
668 		amdgpu_fence_process(ring);  in amdgpu_debugfs_fence_info()
670 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);  in amdgpu_debugfs_fence_info()
672 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info()
674 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info()
676 		if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)  in amdgpu_debugfs_fence_info()
681 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));  in amdgpu_debugfs_fence_info()
684 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));  in amdgpu_debugfs_fence_info()
687 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));  in amdgpu_debugfs_fence_info()