1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
4 #include <linux/dma-buf-map.h>
34 			return -ENOMEM;  in lima_sched_slab_init()
43 	if (!--lima_fence_slab_refcnt) {  in lima_sched_slab_fini()
63 	return f->pipe->base.name;  in lima_fence_get_timeline_name()
78 	call_rcu(&f->base.rcu, lima_fence_release_rcu);  in lima_fence_release()
95 	fence->pipe = pipe;  in lima_fence_create()
96 	dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,  in lima_fence_create()
97 		       pipe->fence_context, ++pipe->fence_seqno);  in lima_fence_create()
115 			 struct lima_vm *vm)  in lima_sched_task_init()  argument
119 	task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);  in lima_sched_task_init()
120 	if (!task->bos)  in lima_sched_task_init()
121 		return -ENOMEM;  in lima_sched_task_init()
124 		drm_gem_object_get(&bos[i]->base.base);  in lima_sched_task_init()
126 	err = drm_sched_job_init(&task->base, &context->base, vm);  in lima_sched_task_init()
128 		kfree(task->bos);  in lima_sched_task_init()
132 	task->num_bos = num_bos;  in lima_sched_task_init()
133 	task->vm = lima_vm_get(vm);  in lima_sched_task_init()
135 	xa_init_flags(&task->deps, XA_FLAGS_ALLOC);  in lima_sched_task_init()
146 	drm_sched_job_cleanup(&task->base);  in lima_sched_task_fini()
148 	xa_for_each(&task->deps, index, fence) {  in lima_sched_task_fini()
151 	xa_destroy(&task->deps);  in lima_sched_task_fini()
153 	if (task->bos) {  in lima_sched_task_fini()
154 		for (i = 0; i < task->num_bos; i++)  in lima_sched_task_fini()
155 			drm_gem_object_put(&task->bos[i]->base.base);  in lima_sched_task_fini()
156 		kfree(task->bos);  in lima_sched_task_fini()
159 	lima_vm_put(task->vm);  in lima_sched_task_fini()
166 	struct drm_gpu_scheduler *sched = &pipe->base;  in lima_sched_context_init()
168 	return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL,  in lima_sched_context_init()
175 	drm_sched_entity_fini(&context->base);  in lima_sched_context_fini()
181 	struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);  in lima_sched_context_queue_task()
184 	drm_sched_entity_push_job(&task->base, &context->base);  in lima_sched_context_queue_task()
193 	if (!xa_empty(&task->deps))  in lima_sched_dependency()
194 		return xa_erase(&task->deps, task->last_dep++);  in lima_sched_dependency()
204 	ret = pm_runtime_resume_and_get(ldev->dev);  in lima_pm_busy()
208 	lima_devfreq_record_busy(&ldev->devfreq);  in lima_pm_busy()
214 	lima_devfreq_record_idle(&ldev->devfreq);  in lima_pm_idle()
217 	pm_runtime_mark_last_busy(ldev->dev);  in lima_pm_idle()
218 	pm_runtime_put_autosuspend(ldev->dev);  in lima_pm_idle()
224 	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);  in lima_sched_run_job()
225 	struct lima_device *ldev = pipe->ldev;  in lima_sched_run_job()
230 	if (job->s_fence->finished.error < 0)  in lima_sched_run_job()
239 		dma_fence_put(&fence->base);  in lima_sched_run_job()
243 	task->fence = &fence->base;  in lima_sched_run_job()
248 	dma_fence_get(task->fence);  in lima_sched_run_job()
250 	pipe->current_task = task;  in lima_sched_run_job()
267 	for (i = 0; i < pipe->num_l2_cache; i++)  in lima_sched_run_job()
268 		lima_l2_cache_flush(pipe->l2_cache[i]);  in lima_sched_run_job()
270 	lima_vm_put(pipe->current_vm);  in lima_sched_run_job()
271 	pipe->current_vm = lima_vm_get(task->vm);  in lima_sched_run_job()
273 	if (pipe->bcast_mmu)  in lima_sched_run_job()
274 		lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm);  in lima_sched_run_job()
276 		for (i = 0; i < pipe->num_mmu; i++)  in lima_sched_run_job()
277 			lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm);  in lima_sched_run_job()
282 	pipe->error = false;  in lima_sched_run_job()
283 	pipe->task_run(pipe, task);  in lima_sched_run_job()
285 	return task->fence;  in lima_sched_run_job()
291 	struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched);  in lima_sched_build_error_task_list()
292 	struct lima_ip *ip = pipe->processor[0];  in lima_sched_build_error_task_list()
293 	int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp;  in lima_sched_build_error_task_list()
294 	struct lima_device *dev = ip->dev;  in lima_sched_build_error_task_list()
296 		container_of(task->base.entity,  in lima_sched_build_error_task_list()
306 	struct dma_buf_map map;  in lima_sched_build_error_task_list()  local
309 	mutex_lock(&dev->error_task_list_lock);  in lima_sched_build_error_task_list()
311 	if (dev->dump.num_tasks >= lima_max_error_tasks) {  in lima_sched_build_error_task_list()
312 		dev_info(dev->dev, "fail to save task state from %s pid %d: "  in lima_sched_build_error_task_list()
313 			 "error task list is full\n", ctx->pname, ctx->pid);  in lima_sched_build_error_task_list()
318 	size = sizeof(struct lima_dump_chunk) + pipe->frame_size;  in lima_sched_build_error_task_list()
320 	size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname);  in lima_sched_build_error_task_list()
324 	for (i = 0; i < task->num_bos; i++) {  in lima_sched_build_error_task_list()
325 		struct lima_bo *bo = task->bos[i];  in lima_sched_build_error_task_list()
328 		size += bo->heap_size ? bo->heap_size : lima_bo_size(bo);  in lima_sched_build_error_task_list()
335 		dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n",  in lima_sched_build_error_task_list()
340 	et->data = et + 1;  in lima_sched_build_error_task_list()
341 	et->size = task_size;  in lima_sched_build_error_task_list()
343 	dt = et->data;  in lima_sched_build_error_task_list()
345 	dt->id = pipe_id;  in lima_sched_build_error_task_list()
346 	dt->size = size;  in lima_sched_build_error_task_list()
350 	chunk->id = LIMA_DUMP_CHUNK_FRAME;  in lima_sched_build_error_task_list()
351 	chunk->size = pipe->frame_size;  in lima_sched_build_error_task_list()
352 	memcpy(chunk + 1, task->frame, pipe->frame_size);  in lima_sched_build_error_task_list()
353 	dt->num_chunks++;  in lima_sched_build_error_task_list()
355 	chunk = (void *)(chunk + 1) + chunk->size;  in lima_sched_build_error_task_list()
357 	chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME;  in lima_sched_build_error_task_list()
358 	chunk->size = sizeof(ctx->pname);  in lima_sched_build_error_task_list()
359 	memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname));  in lima_sched_build_error_task_list()
360 	dt->num_chunks++;  in lima_sched_build_error_task_list()
362 	pid_chunk = (void *)(chunk + 1) + chunk->size;  in lima_sched_build_error_task_list()
364 	pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID;  in lima_sched_build_error_task_list()
365 	pid_chunk->pid = ctx->pid;  in lima_sched_build_error_task_list()
366 	dt->num_chunks++;  in lima_sched_build_error_task_list()
368 	buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size;  in lima_sched_build_error_task_list()
369 	for (i = 0; i < task->num_bos; i++) {  in lima_sched_build_error_task_list()
370 		struct lima_bo *bo = task->bos[i];  in lima_sched_build_error_task_list()
374 		buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER;  in lima_sched_build_error_task_list()
375 		buffer_chunk->va = lima_vm_get_va(task->vm, bo);  in lima_sched_build_error_task_list()
377 		if (bo->heap_size) {  in lima_sched_build_error_task_list()
378 			buffer_chunk->size = bo->heap_size;  in lima_sched_build_error_task_list()
380 			data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT,  in lima_sched_build_error_task_list()
387 			memcpy(buffer_chunk + 1, data, buffer_chunk->size);  in lima_sched_build_error_task_list()
391 			buffer_chunk->size = lima_bo_size(bo);  in lima_sched_build_error_task_list()
393 			ret = drm_gem_shmem_vmap(&bo->base.base, &map);  in lima_sched_build_error_task_list()
399 			memcpy(buffer_chunk + 1, map.vaddr, buffer_chunk->size);  in lima_sched_build_error_task_list()
401 			drm_gem_shmem_vunmap(&bo->base.base, &map);  in lima_sched_build_error_task_list()
404 		buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size;  in lima_sched_build_error_task_list()
405 		dt->num_chunks++;  in lima_sched_build_error_task_list()
408 	list_add(&et->list, &dev->error_task_list);  in lima_sched_build_error_task_list()
409 	dev->dump.size += et->size;  in lima_sched_build_error_task_list()
410 	dev->dump.num_tasks++;  in lima_sched_build_error_task_list()
412 	dev_info(dev->dev, "save error task state success\n");  in lima_sched_build_error_task_list()
415 	mutex_unlock(&dev->error_task_list_lock);  in lima_sched_build_error_task_list()
420 	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);  in lima_sched_timedout_job()
422 	struct lima_device *ldev = pipe->ldev;  in lima_sched_timedout_job()
424 	if (!pipe->error)  in lima_sched_timedout_job()
427 	drm_sched_stop(&pipe->base, &task->base);  in lima_sched_timedout_job()
429 	drm_sched_increase_karma(&task->base);  in lima_sched_timedout_job()
433 	pipe->task_error(pipe);  in lima_sched_timedout_job()
435 	if (pipe->bcast_mmu)  in lima_sched_timedout_job()
436 		lima_mmu_page_fault_resume(pipe->bcast_mmu);  in lima_sched_timedout_job()
440 		for (i = 0; i < pipe->num_mmu; i++)  in lima_sched_timedout_job()
441 			lima_mmu_page_fault_resume(pipe->mmu[i]);  in lima_sched_timedout_job()
444 	lima_vm_put(pipe->current_vm);  in lima_sched_timedout_job()
445 	pipe->current_vm = NULL;  in lima_sched_timedout_job()
446 	pipe->current_task = NULL;  in lima_sched_timedout_job()
450 	drm_sched_resubmit_jobs(&pipe->base);  in lima_sched_timedout_job()
451 	drm_sched_start(&pipe->base, true);  in lima_sched_timedout_job()
459 	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);  in lima_sched_free_job()
460 	struct lima_vm *vm = task->vm;  in lima_sched_free_job()  local
461 	struct lima_bo **bos = task->bos;  in lima_sched_free_job()
464 	dma_fence_put(task->fence);  in lima_sched_free_job()
466 	for (i = 0; i < task->num_bos; i++)  in lima_sched_free_job()
467 		lima_vm_bo_del(vm, bos[i]);  in lima_sched_free_job()
470 	kmem_cache_free(pipe->task_slab, task);  in lima_sched_free_job()
486 	for (i = 0; i < pipe->num_l2_cache; i++)  in lima_sched_recover_work()
487 		lima_l2_cache_flush(pipe->l2_cache[i]);  in lima_sched_recover_work()
489 	if (pipe->bcast_mmu) {  in lima_sched_recover_work()
490 		lima_mmu_flush_tlb(pipe->bcast_mmu);  in lima_sched_recover_work()
492 		for (i = 0; i < pipe->num_mmu; i++)  in lima_sched_recover_work()
493 			lima_mmu_flush_tlb(pipe->mmu[i]);  in lima_sched_recover_work()
496 	if (pipe->task_recover(pipe))  in lima_sched_recover_work()
497 		drm_sched_fault(&pipe->base);  in lima_sched_recover_work()
505 	pipe->fence_context = dma_fence_context_alloc(1);  in lima_sched_pipe_init()
506 	spin_lock_init(&pipe->fence_lock);  in lima_sched_pipe_init()
508 	INIT_WORK(&pipe->recover_work, lima_sched_recover_work);  in lima_sched_pipe_init()
510 	return drm_sched_init(&pipe->base, &lima_sched_ops, 1,  in lima_sched_pipe_init()
518 	drm_sched_fini(&pipe->base);  in lima_sched_pipe_fini()
523 	struct lima_sched_task *task = pipe->current_task;  in lima_sched_pipe_task_done()
524 	struct lima_device *ldev = pipe->ldev;  in lima_sched_pipe_task_done()
526 	if (pipe->error) {  in lima_sched_pipe_task_done()
527 		if (task && task->recoverable)  in lima_sched_pipe_task_done()
528 			schedule_work(&pipe->recover_work);  in lima_sched_pipe_task_done()
530 			drm_sched_fault(&pipe->base);  in lima_sched_pipe_task_done()
532 		pipe->task_fini(pipe);  in lima_sched_pipe_task_done()
533 		dma_fence_signal(task->fence);  in lima_sched_pipe_task_done()