103 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
119 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
163 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
172 				       &ring->fence_drv.lock,  in amdgpu_fence_emit()
179 				       &ring->fence_drv.lock,  in amdgpu_fence_emit()
183 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
186 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
232 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit_polling()
234 				      seq - ring->fence_drv.num_fences_mask,  in amdgpu_fence_emit_polling()
239 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit_polling()
256 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
273 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
278 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
283 	if (del_timer(&ring->fence_drv.fallback_timer) &&  in amdgpu_fence_process()
284 	    seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
326 					      fence_drv.fallback_timer);  in amdgpu_fence_fallback()
342 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
349 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
404 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
405 	emitted += READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
430 		ring->fence_drv.cpu_addr = ring->fence_cpu_addr;  in amdgpu_fence_driver_start_ring()
431 		ring->fence_drv.gpu_addr = ring->fence_gpu_addr;  in amdgpu_fence_driver_start_ring()
435 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;  in amdgpu_fence_driver_start_ring()
436 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;  in amdgpu_fence_driver_start_ring()
438 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
440 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
441 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
442 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
445 		      ring->name, ring->fence_drv.gpu_addr);  in amdgpu_fence_driver_start_ring()
468 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
469 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
470 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
471 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
472 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
474 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);  in amdgpu_fence_driver_init_ring()
476 	ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
477 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
478 	ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
481 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
519 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_fini()
531 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_hw_fini()
532 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_fini()
533 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_fini()
535 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_hw_fini()
547 		if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)  in amdgpu_fence_driver_isr_toggle()
564 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_sw_fini()
570 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_sw_fini()
571 			dma_fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_sw_fini()
572 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_sw_fini()
573 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_sw_fini()
574 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_sw_fini()
596 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_init()
600 		if (ring->fence_drv.irq_src)  in amdgpu_fence_driver_hw_init()
601 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_init()
602 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_init()
617 	for (i = 0; i <= ring->fence_drv.num_fences_mask; i++) {  in amdgpu_fence_driver_clear_job_fences()
618 		ptr = &ring->fence_drv.fences[i];  in amdgpu_fence_driver_clear_job_fences()
635 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
670 	if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
687 	if (!timer_pending(&to_amdgpu_ring(job->base.sched)->fence_drv.fallback_timer))  in amdgpu_job_fence_enable_signaling()
774 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info_show()
781 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info_show()
783 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info_show()
798 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));  in amdgpu_debugfs_fence_info_show()
801 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));  in amdgpu_debugfs_fence_info_show()
804 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));  in amdgpu_debugfs_fence_info_show()