49  * are no longer in use by the associated ring on the GPU and
57 	struct amdgpu_ring		*ring;  member
94  * @ring: ring the fence is associated with
99 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)  in amdgpu_fence_write()  argument
101 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
110  * @ring: ring the fence is associated with
115 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)  in amdgpu_fence_read()  argument
117 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
129  * amdgpu_fence_emit - emit a fence on the requested ring
131  * @ring: ring the fence is associated with
134  * Emits a fence command on the requested ring (all asics).
137 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,  in amdgpu_fence_emit()  argument
140 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_emit()
150 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
151 	fence->ring = ring;  in amdgpu_fence_emit()
153 		       &ring->fence_drv.lock,  in amdgpu_fence_emit()
154 		       adev->fence_context + ring->idx,  in amdgpu_fence_emit()
156 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
159 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
176 	 * emitting the fence would mess up the hardware ring buffer.  in amdgpu_fence_emit()
186  * amdgpu_fence_emit_polling - emit a fence on the requeste ring
188  * @ring: ring the fence is associated with
191  * Emits a fence command on the requested ring (all asics).
195 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,  in amdgpu_fence_emit_polling()  argument
204 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit_polling()
205 	r = amdgpu_fence_wait_polling(ring,  in amdgpu_fence_emit_polling()
206 				      seq - ring->fence_drv.num_fences_mask,  in amdgpu_fence_emit_polling()
211 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit_polling()
222  * @ring: pointer to struct amdgpu_ring
226 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)  in amdgpu_fence_schedule_fallback()  argument
228 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
235  * @ring: pointer to struct amdgpu_ring
243 bool amdgpu_fence_process(struct amdgpu_ring *ring)  in amdgpu_fence_process()  argument
245 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
246 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_process()
251 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
252 		seq = amdgpu_fence_read(ring);  in amdgpu_fence_process()
256 	if (del_timer(&ring->fence_drv.fallback_timer) &&  in amdgpu_fence_process()
257 	    seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
258 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_process()
303 	struct amdgpu_ring *ring = from_timer(ring, t,  in amdgpu_fence_fallback()  local
306 	if (amdgpu_fence_process(ring))  in amdgpu_fence_fallback()
307 		DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name);  in amdgpu_fence_fallback()
314  * @ring: ring index the fence is associated with
316  * Wait for all fences on the requested ring to signal (all asics).
319 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)  in amdgpu_fence_wait_empty()  argument
321 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
328 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
345  * @ring: ring index the fence is associated with
349  * Wait for all fences on the requested ring to signal (all asics).
352 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,  in amdgpu_fence_wait_polling()  argument
359 		seq = amdgpu_fence_read(ring);  in amdgpu_fence_wait_polling()
369  * @ring: ring the fence is associated with
371  * Get the number of fences emitted on the requested ring (all asics).
372  * Returns the number of emitted fences on the ring.  Used by the
373  * dynpm code to ring track activity.
375 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)  in amdgpu_fence_count_emitted()  argument
379 	/* We are not protected by ring lock when reading the last sequence  in amdgpu_fence_count_emitted()
382 	amdgpu_fence_process(ring);  in amdgpu_fence_count_emitted()
384 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
385 	emitted += READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
391  * ready for use on the requested ring.
393  * @ring: ring to start the fence driver on
394  * @irq_src: interrupt source to use for this ring
395  * @irq_type: interrupt type to use for this ring
402 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_start_ring()  argument
406 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_start_ring()
409 	if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {  in amdgpu_fence_driver_start_ring()
410 		ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];  in amdgpu_fence_driver_start_ring()
411 		ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);  in amdgpu_fence_driver_start_ring()
415 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;  in amdgpu_fence_driver_start_ring()
416 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;  in amdgpu_fence_driver_start_ring()
418 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
423 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
424 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
425 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
427 	DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",  in amdgpu_fence_driver_start_ring()
428 		      ring->name, ring->fence_drv.gpu_addr);  in amdgpu_fence_driver_start_ring()
434  * for the requested ring.
436  * @ring: ring to init the fence driver on
439  * Init the fence driver for the requested ring (all asics).
442 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_init_ring()  argument
445 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_init_ring()
455 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
456 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
457 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
458 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
459 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
461 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);  in amdgpu_fence_driver_init_ring()
463 	ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
464 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
465 	ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
467 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
471 	if (!ring->no_scheduler) {  in amdgpu_fence_driver_init_ring()
472 		switch (ring->funcs->type) {  in amdgpu_fence_driver_init_ring()
487 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,  in amdgpu_fence_driver_init_ring()
489 				   timeout, ring->name);  in amdgpu_fence_driver_init_ring()
491 			DRM_ERROR("Failed to create scheduler on ring %s.\n",  in amdgpu_fence_driver_init_ring()
492 				  ring->name);  in amdgpu_fence_driver_init_ring()
531 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_fini()  local
533 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_fini()
535 		r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_fini()
538 			amdgpu_fence_driver_force_completion(ring);  in amdgpu_fence_driver_fini()
540 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_fini()
541 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_fini()
542 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_fini()
543 		if (!ring->no_scheduler)  in amdgpu_fence_driver_fini()
544 			drm_sched_fini(&ring->sched);  in amdgpu_fence_driver_fini()
545 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_fini()
546 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_fini()
547 			dma_fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_fini()
548 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_fini()
549 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_fini()
550 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_fini()
567 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_suspend()  local
568 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_suspend()
572 		r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_suspend()
575 			amdgpu_fence_driver_force_completion(ring);  in amdgpu_fence_driver_suspend()
579 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_suspend()
580 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_suspend()
581 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_suspend()
602 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_resume()  local
603 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_resume()
607 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_resume()
608 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_resume()
609 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_resume()
614  * amdgpu_fence_driver_force_completion - force signal latest fence of ring
616  * @ring: fence of the ring to signal
619 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)  in amdgpu_fence_driver_force_completion()  argument
621 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
622 	amdgpu_fence_process(ring);  in amdgpu_fence_driver_force_completion()
637 	return (const char *)fence->ring->name;  in amdgpu_fence_get_timeline_name()
651 	struct amdgpu_ring *ring = fence->ring;  in amdgpu_fence_enable_signaling()  local
653 	if (!timer_pending(&ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
654 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_enable_signaling()
656 	DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);  in amdgpu_fence_enable_signaling()
707 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_debugfs_fence_info()  local
708 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info()
711 		amdgpu_fence_process(ring);  in amdgpu_debugfs_fence_info()
713 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);  in amdgpu_debugfs_fence_info()
715 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info()
717 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info()
719 		if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||  in amdgpu_debugfs_fence_info()
720 		    ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {  in amdgpu_debugfs_fence_info()
722 				   le32_to_cpu(*ring->trail_fence_cpu_addr));  in amdgpu_debugfs_fence_info()
724 				   ring->trail_seq);  in amdgpu_debugfs_fence_info()
727 		if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)  in amdgpu_debugfs_fence_info()
732 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));  in amdgpu_debugfs_fence_info()
735 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));  in amdgpu_debugfs_fence_info()
738 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));  in amdgpu_debugfs_fence_info()