101 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
117 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
150 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
153 		       &ring->fence_drv.lock,  in amdgpu_fence_emit()
156 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
159 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
204 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit_polling()
206 				      seq - ring->fence_drv.num_fences_mask,  in amdgpu_fence_emit_polling()
211 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit_polling()
228 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
245 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
251 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
256 	if (del_timer(&ring->fence_drv.fallback_timer) &&  in amdgpu_fence_process()
257 	    seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
304 					      fence_drv.fallback_timer);  in amdgpu_fence_fallback()
321 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
328 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
384 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
385 	emitted += READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
410 		ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];  in amdgpu_fence_driver_start_ring()
411 		ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);  in amdgpu_fence_driver_start_ring()
415 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;  in amdgpu_fence_driver_start_ring()
416 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;  in amdgpu_fence_driver_start_ring()
418 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
423 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
424 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
425 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
428 		      ring->name, ring->fence_drv.gpu_addr);  in amdgpu_fence_driver_start_ring()
455 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
456 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
457 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
458 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
459 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
461 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);  in amdgpu_fence_driver_init_ring()
463 	ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
464 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
465 	ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
467 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
533 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_fini()
540 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_fini()
541 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_fini()
542 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_fini()
545 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_fini()
546 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_fini()
547 			dma_fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_fini()
548 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_fini()
549 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_fini()
550 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_fini()
568 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_suspend()
579 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_suspend()
580 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_suspend()
581 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_suspend()
603 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_resume()
607 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_resume()
608 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_resume()
609 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_resume()
621 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
653 	if (!timer_pending(&ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
708 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info()
715 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info()
717 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info()
732 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));  in amdgpu_debugfs_fence_info()
735 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));  in amdgpu_debugfs_fence_info()
738 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));  in amdgpu_debugfs_fence_info()