81 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned int ndw)  in amdgpu_ring_alloc()  argument
85 	ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask;  in amdgpu_ring_alloc()
90 	if (WARN_ON_ONCE(ndw > ring->max_dw))  in amdgpu_ring_alloc()
93 	ring->count_dw = ndw;  in amdgpu_ring_alloc()
94 	ring->wptr_old = ring->wptr;  in amdgpu_ring_alloc()
96 	if (ring->funcs->begin_use)  in amdgpu_ring_alloc()
97 		ring->funcs->begin_use(ring);  in amdgpu_ring_alloc()
109 void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)  in amdgpu_ring_insert_nop()  argument
114 		amdgpu_ring_write(ring, ring->funcs->nop);  in amdgpu_ring_insert_nop()
125 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)  in amdgpu_ring_generic_pad_ib()  argument
127 	while (ib->length_dw & ring->funcs->align_mask)  in amdgpu_ring_generic_pad_ib()
128 		ib->ptr[ib->length_dw++] = ring->funcs->nop;  in amdgpu_ring_generic_pad_ib()
140 void amdgpu_ring_commit(struct amdgpu_ring *ring)  in amdgpu_ring_commit()  argument
145 	count = ring->funcs->align_mask + 1 -  in amdgpu_ring_commit()
146 		(ring->wptr & ring->funcs->align_mask);  in amdgpu_ring_commit()
147 	count %= ring->funcs->align_mask + 1;  in amdgpu_ring_commit()
148 	ring->funcs->insert_nop(ring, count);  in amdgpu_ring_commit()
151 	amdgpu_ring_set_wptr(ring);  in amdgpu_ring_commit()
153 	if (ring->funcs->end_use)  in amdgpu_ring_commit()
154 		ring->funcs->end_use(ring);  in amdgpu_ring_commit()
164 void amdgpu_ring_undo(struct amdgpu_ring *ring)  in amdgpu_ring_undo()  argument
166 	ring->wptr = ring->wptr_old;  in amdgpu_ring_undo()
168 	if (ring->funcs->end_use)  in amdgpu_ring_undo()
169 		ring->funcs->end_use(ring);  in amdgpu_ring_undo()
172 #define amdgpu_ring_get_gpu_addr(ring, offset)				\  argument
173 	(ring->is_mes_queue ?						\
174 	 (ring->mes_ctx->meta_data_gpu_addr + offset) :			\
175 	 (ring->adev->wb.gpu_addr + offset * 4))
177 #define amdgpu_ring_get_cpu_addr(ring, offset)				\  argument
178 	(ring->is_mes_queue ?						\
179 	 (void *)((uint8_t *)(ring->mes_ctx->meta_data_ptr) + offset) : \
180 	 (&ring->adev->wb.wb[offset]))
196 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,  in amdgpu_ring_init()  argument
213 	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)  in amdgpu_ring_init()
215 	else if (ring == &adev->sdma.instance[0].page)  in amdgpu_ring_init()
218 	if (ring->adev == NULL) {  in amdgpu_ring_init()
222 		ring->adev = adev;  in amdgpu_ring_init()
223 		ring->num_hw_submission = sched_hw_submission;  in amdgpu_ring_init()
224 		ring->sched_score = sched_score;  in amdgpu_ring_init()
225 		ring->vmid_wait = dma_fence_get_stub();  in amdgpu_ring_init()
227 		if (!ring->is_mes_queue) {  in amdgpu_ring_init()
228 			ring->idx = adev->num_rings++;  in amdgpu_ring_init()
229 			adev->rings[ring->idx] = ring;  in amdgpu_ring_init()
232 		r = amdgpu_fence_driver_init_ring(ring);  in amdgpu_ring_init()
237 	if (ring->is_mes_queue) {  in amdgpu_ring_init()
238 		ring->rptr_offs = amdgpu_mes_ctx_get_offs(ring,  in amdgpu_ring_init()
240 		ring->wptr_offs = amdgpu_mes_ctx_get_offs(ring,  in amdgpu_ring_init()
242 		ring->fence_offs = amdgpu_mes_ctx_get_offs(ring,  in amdgpu_ring_init()
244 		ring->trail_fence_offs = amdgpu_mes_ctx_get_offs(ring,  in amdgpu_ring_init()
246 		ring->cond_exe_offs = amdgpu_mes_ctx_get_offs(ring,  in amdgpu_ring_init()
249 		r = amdgpu_device_wb_get(adev, &ring->rptr_offs);  in amdgpu_ring_init()
255 		r = amdgpu_device_wb_get(adev, &ring->wptr_offs);  in amdgpu_ring_init()
261 		r = amdgpu_device_wb_get(adev, &ring->fence_offs);  in amdgpu_ring_init()
267 		r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs);  in amdgpu_ring_init()
273 		r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs);  in amdgpu_ring_init()
280 	ring->fence_gpu_addr =  in amdgpu_ring_init()
281 		amdgpu_ring_get_gpu_addr(ring, ring->fence_offs);  in amdgpu_ring_init()
282 	ring->fence_cpu_addr =  in amdgpu_ring_init()
283 		amdgpu_ring_get_cpu_addr(ring, ring->fence_offs);  in amdgpu_ring_init()
285 	ring->rptr_gpu_addr =  in amdgpu_ring_init()
286 		amdgpu_ring_get_gpu_addr(ring, ring->rptr_offs);  in amdgpu_ring_init()
287 	ring->rptr_cpu_addr =  in amdgpu_ring_init()
288 		amdgpu_ring_get_cpu_addr(ring, ring->rptr_offs);  in amdgpu_ring_init()
290 	ring->wptr_gpu_addr =  in amdgpu_ring_init()
291 		amdgpu_ring_get_gpu_addr(ring, ring->wptr_offs);  in amdgpu_ring_init()
292 	ring->wptr_cpu_addr =  in amdgpu_ring_init()
293 		amdgpu_ring_get_cpu_addr(ring, ring->wptr_offs);  in amdgpu_ring_init()
295 	ring->trail_fence_gpu_addr =  in amdgpu_ring_init()
296 		amdgpu_ring_get_gpu_addr(ring, ring->trail_fence_offs);  in amdgpu_ring_init()
297 	ring->trail_fence_cpu_addr =  in amdgpu_ring_init()
298 		amdgpu_ring_get_cpu_addr(ring, ring->trail_fence_offs);  in amdgpu_ring_init()
300 	ring->cond_exe_gpu_addr =  in amdgpu_ring_init()
301 		amdgpu_ring_get_gpu_addr(ring, ring->cond_exe_offs);  in amdgpu_ring_init()
302 	ring->cond_exe_cpu_addr =  in amdgpu_ring_init()
303 		amdgpu_ring_get_cpu_addr(ring, ring->cond_exe_offs);  in amdgpu_ring_init()
306 	*ring->cond_exe_cpu_addr = 1;  in amdgpu_ring_init()
308 	r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type);  in amdgpu_ring_init()
314 	max_ibs_dw = ring->funcs->emit_frame_size +  in amdgpu_ring_init()
315 		     amdgpu_ring_max_ibs(ring->funcs->type) * ring->funcs->emit_ib_size;  in amdgpu_ring_init()
316 	max_ibs_dw = (max_ibs_dw + ring->funcs->align_mask) & ~ring->funcs->align_mask;  in amdgpu_ring_init()
321 	ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission);  in amdgpu_ring_init()
323 	ring->buf_mask = (ring->ring_size / 4) - 1;  in amdgpu_ring_init()
324 	ring->ptr_mask = ring->funcs->support_64bit_ptrs ?  in amdgpu_ring_init()
325 		0xffffffffffffffff : ring->buf_mask;  in amdgpu_ring_init()
328 	if (ring->is_mes_queue) {  in amdgpu_ring_init()
331 		BUG_ON(ring->ring_size > PAGE_SIZE*4);  in amdgpu_ring_init()
333 		offset = amdgpu_mes_ctx_get_offs(ring,  in amdgpu_ring_init()
335 		ring->gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);  in amdgpu_ring_init()
336 		ring->ring = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);  in amdgpu_ring_init()
337 		amdgpu_ring_clear_ring(ring);  in amdgpu_ring_init()
339 	} else if (ring->ring_obj == NULL) {  in amdgpu_ring_init()
340 		r = amdgpu_bo_create_kernel(adev, ring->ring_size + ring->funcs->extra_dw, PAGE_SIZE,  in amdgpu_ring_init()
342 					    &ring->ring_obj,  in amdgpu_ring_init()
343 					    &ring->gpu_addr,  in amdgpu_ring_init()
344 					    (void **)&ring->ring);  in amdgpu_ring_init()
349 		amdgpu_ring_clear_ring(ring);  in amdgpu_ring_init()
352 	ring->max_dw = max_dw;  in amdgpu_ring_init()
353 	ring->hw_prio = hw_prio;  in amdgpu_ring_init()
355 	if (!ring->no_scheduler) {  in amdgpu_ring_init()
356 		hw_ip = ring->funcs->type;  in amdgpu_ring_init()
359 			&ring->sched;  in amdgpu_ring_init()
372 void amdgpu_ring_fini(struct amdgpu_ring *ring)  in amdgpu_ring_fini()  argument
376 	if (!(ring->adev) ||  in amdgpu_ring_fini()
377 	    (!ring->is_mes_queue && !(ring->adev->rings[ring->idx])))  in amdgpu_ring_fini()
380 	ring->sched.ready = false;  in amdgpu_ring_fini()
382 	if (!ring->is_mes_queue) {  in amdgpu_ring_fini()
383 		amdgpu_device_wb_free(ring->adev, ring->rptr_offs);  in amdgpu_ring_fini()
384 		amdgpu_device_wb_free(ring->adev, ring->wptr_offs);  in amdgpu_ring_fini()
386 		amdgpu_device_wb_free(ring->adev, ring->cond_exe_offs);  in amdgpu_ring_fini()
387 		amdgpu_device_wb_free(ring->adev, ring->fence_offs);  in amdgpu_ring_fini()
389 		amdgpu_bo_free_kernel(&ring->ring_obj,  in amdgpu_ring_fini()
390 				      &ring->gpu_addr,  in amdgpu_ring_fini()
391 				      (void **)&ring->ring);  in amdgpu_ring_fini()
393 		kfree(ring->fence_drv.fences);  in amdgpu_ring_fini()
396 	dma_fence_put(ring->vmid_wait);  in amdgpu_ring_fini()
397 	ring->vmid_wait = NULL;  in amdgpu_ring_fini()
398 	ring->me = 0;  in amdgpu_ring_fini()
400 	if (!ring->is_mes_queue)  in amdgpu_ring_fini()
401 		ring->adev->rings[ring->idx] = NULL;  in amdgpu_ring_fini()
416 void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,  in amdgpu_ring_emit_reg_write_reg_wait_helper()  argument
420 	amdgpu_ring_emit_wreg(ring, reg0, ref);  in amdgpu_ring_emit_reg_write_reg_wait_helper()
421 	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);  in amdgpu_ring_emit_reg_write_reg_wait_helper()
433 bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,  in amdgpu_ring_soft_recovery()  argument
440 	if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence)  in amdgpu_ring_soft_recovery()
448 	atomic_inc(&ring->adev->gpu_reset_counter);  in amdgpu_ring_soft_recovery()
451 		ring->funcs->soft_recovery(ring, vmid);  in amdgpu_ring_soft_recovery()
471 	struct amdgpu_ring *ring = file_inode(f)->i_private;  in amdgpu_debugfs_ring_read()  local
481 		early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask;  in amdgpu_debugfs_ring_read()
482 		early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask;  in amdgpu_debugfs_ring_read()
483 		early[2] = ring->wptr & ring->buf_mask;  in amdgpu_debugfs_ring_read()
496 		if (*pos >= (ring->ring_size + 12))  in amdgpu_debugfs_ring_read()
499 		value = ring->ring[(*pos - 12)/4];  in amdgpu_debugfs_ring_read()
521 	struct amdgpu_ring *ring = file_inode(f)->i_private;  in amdgpu_debugfs_mqd_read()  local
531 	r = amdgpu_bo_reserve(ring->mqd_obj, false);  in amdgpu_debugfs_mqd_read()
535 	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&mqd);  in amdgpu_debugfs_mqd_read()
537 		amdgpu_bo_unreserve(ring->mqd_obj);  in amdgpu_debugfs_mqd_read()
542 		if (*pos >= ring->mqd_size)  in amdgpu_debugfs_mqd_read()
556 	amdgpu_bo_kunmap(ring->mqd_obj);  in amdgpu_debugfs_mqd_read()
558 	amdgpu_bo_unreserve(ring->mqd_obj);  in amdgpu_debugfs_mqd_read()
573 	struct amdgpu_ring *ring = data;  in amdgpu_debugfs_ring_error()  local
575 	amdgpu_fence_driver_set_error(ring, val);  in amdgpu_debugfs_ring_error()
585 			      struct amdgpu_ring *ring)  in amdgpu_debugfs_ring_init()  argument
592 	sprintf(name, "amdgpu_ring_%s", ring->name);  in amdgpu_debugfs_ring_init()
593 	debugfs_create_file_size(name, S_IFREG | 0444, root, ring,  in amdgpu_debugfs_ring_init()
595 				 ring->ring_size + 12);  in amdgpu_debugfs_ring_init()
597 	if (ring->mqd_obj) {  in amdgpu_debugfs_ring_init()
598 		sprintf(name, "amdgpu_mqd_%s", ring->name);  in amdgpu_debugfs_ring_init()
599 		debugfs_create_file_size(name, S_IFREG | 0444, root, ring,  in amdgpu_debugfs_ring_init()
601 					 ring->mqd_size);  in amdgpu_debugfs_ring_init()
604 	sprintf(name, "amdgpu_error_%s", ring->name);  in amdgpu_debugfs_ring_init()
605 	debugfs_create_file(name, 0200, root, ring,  in amdgpu_debugfs_ring_init()
620 int amdgpu_ring_test_helper(struct amdgpu_ring *ring)  in amdgpu_ring_test_helper()  argument
622 	struct amdgpu_device *adev = ring->adev;  in amdgpu_ring_test_helper()
625 	r = amdgpu_ring_test_ring(ring);  in amdgpu_ring_test_helper()
628 			      ring->name, r);  in amdgpu_ring_test_helper()
631 			      ring->name);  in amdgpu_ring_test_helper()
633 	ring->sched.ready = !r;  in amdgpu_ring_test_helper()
637 static void amdgpu_ring_to_mqd_prop(struct amdgpu_ring *ring,  in amdgpu_ring_to_mqd_prop()  argument
640 	struct amdgpu_device *adev = ring->adev;  in amdgpu_ring_to_mqd_prop()
644 	prop->mqd_gpu_addr = ring->mqd_gpu_addr;  in amdgpu_ring_to_mqd_prop()
645 	prop->hqd_base_gpu_addr = ring->gpu_addr;  in amdgpu_ring_to_mqd_prop()
646 	prop->rptr_gpu_addr = ring->rptr_gpu_addr;  in amdgpu_ring_to_mqd_prop()
647 	prop->wptr_gpu_addr = ring->wptr_gpu_addr;  in amdgpu_ring_to_mqd_prop()
648 	prop->queue_size = ring->ring_size;  in amdgpu_ring_to_mqd_prop()
649 	prop->eop_gpu_addr = ring->eop_gpu_addr;  in amdgpu_ring_to_mqd_prop()
650 	prop->use_doorbell = ring->use_doorbell;  in amdgpu_ring_to_mqd_prop()
651 	prop->doorbell_index = ring->doorbell_index;  in amdgpu_ring_to_mqd_prop()
656 	prop->hqd_active = ring->funcs->type == AMDGPU_RING_TYPE_KIQ;  in amdgpu_ring_to_mqd_prop()
658 	if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE &&  in amdgpu_ring_to_mqd_prop()
659 	     amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) ||  in amdgpu_ring_to_mqd_prop()
660 	    (ring->funcs->type == AMDGPU_RING_TYPE_GFX &&  in amdgpu_ring_to_mqd_prop()
661 	     amdgpu_gfx_is_high_priority_graphics_queue(adev, ring))) {  in amdgpu_ring_to_mqd_prop()
667 int amdgpu_ring_init_mqd(struct amdgpu_ring *ring)  in amdgpu_ring_init_mqd()  argument
669 	struct amdgpu_device *adev = ring->adev;  in amdgpu_ring_init_mqd()
673 	amdgpu_ring_to_mqd_prop(ring, &prop);  in amdgpu_ring_init_mqd()
675 	ring->wptr = 0;  in amdgpu_ring_init_mqd()
677 	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)  in amdgpu_ring_init_mqd()
680 		mqd_mgr = &adev->mqds[ring->funcs->type];  in amdgpu_ring_init_mqd()
682 	return mqd_mgr->init_mqd(adev, ring->mqd_ptr, &prop);  in amdgpu_ring_init_mqd()
685 void amdgpu_ring_ib_begin(struct amdgpu_ring *ring)  in amdgpu_ring_ib_begin()  argument
687 	if (ring->is_sw_ring)  in amdgpu_ring_ib_begin()
688 		amdgpu_sw_ring_ib_begin(ring);  in amdgpu_ring_ib_begin()
691 void amdgpu_ring_ib_end(struct amdgpu_ring *ring)  in amdgpu_ring_ib_end()  argument
693 	if (ring->is_sw_ring)  in amdgpu_ring_ib_end()
694 		amdgpu_sw_ring_ib_end(ring);  in amdgpu_ring_ib_end()
697 void amdgpu_ring_ib_on_emit_cntl(struct amdgpu_ring *ring)  in amdgpu_ring_ib_on_emit_cntl()  argument
699 	if (ring->is_sw_ring)  in amdgpu_ring_ib_on_emit_cntl()
700 		amdgpu_sw_ring_ib_mark_offset(ring, AMDGPU_MUX_OFFSET_TYPE_CONTROL);  in amdgpu_ring_ib_on_emit_cntl()
703 void amdgpu_ring_ib_on_emit_ce(struct amdgpu_ring *ring)  in amdgpu_ring_ib_on_emit_ce()  argument
705 	if (ring->is_sw_ring)  in amdgpu_ring_ib_on_emit_ce()
706 		amdgpu_sw_ring_ib_mark_offset(ring, AMDGPU_MUX_OFFSET_TYPE_CE);  in amdgpu_ring_ib_on_emit_ce()
709 void amdgpu_ring_ib_on_emit_de(struct amdgpu_ring *ring)  in amdgpu_ring_ib_on_emit_de()  argument
711 	if (ring->is_sw_ring)  in amdgpu_ring_ib_on_emit_de()
712 		amdgpu_sw_ring_ib_mark_offset(ring, AMDGPU_MUX_OFFSET_TYPE_DE);  in amdgpu_ring_ib_on_emit_de()