100 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
116 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
160 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
166 				&ring->fence_drv.lock,  in amdgpu_fence_emit()
176 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
179 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
225 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit_polling()
227 				      seq - ring->fence_drv.num_fences_mask,  in amdgpu_fence_emit_polling()
232 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit_polling()
249 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
266 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
272 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
277 	if (del_timer(&ring->fence_drv.fallback_timer) &&  in amdgpu_fence_process()
278 	    seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
325 					      fence_drv.fallback_timer);  in amdgpu_fence_fallback()
341 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
348 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
404 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
405 	emitted += READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
430 		ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];  in amdgpu_fence_driver_start_ring()
431 		ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);  in amdgpu_fence_driver_start_ring()
435 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;  in amdgpu_fence_driver_start_ring()
436 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;  in amdgpu_fence_driver_start_ring()
438 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
440 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
441 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
442 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
445 		      ring->name, ring->fence_drv.gpu_addr);  in amdgpu_fence_driver_start_ring()
474 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
475 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
476 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
477 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
478 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
480 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);  in amdgpu_fence_driver_init_ring()
482 	ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
483 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
484 	ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
486 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
552 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_fini()
567 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_hw_fini()
568 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_fini()
569 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_fini()
571 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_hw_fini()
582 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_sw_fini()
588 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_sw_fini()
589 			dma_fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_sw_fini()
590 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_sw_fini()
591 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_sw_fini()
592 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_sw_fini()
614 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_init()
623 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_hw_init()
624 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_init()
625 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_init()
637 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
684 	if (!timer_pending(&ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
750 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info_show()
757 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info_show()
759 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info_show()
774 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));  in amdgpu_debugfs_fence_info_show()
777 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));  in amdgpu_debugfs_fence_info_show()
780 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));  in amdgpu_debugfs_fence_info_show()