1 // SPDX-License-Identifier: GPL-2.0
9 #include <linux/dma-resv.h>
23 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
24 #define job_read(dev, reg) readl(dev->iomem + (reg))
67 	switch (f->queue) {  in panfrost_fence_get_timeline_name()
69 		return "panfrost-js-0";  in panfrost_fence_get_timeline_name()
71 		return "panfrost-js-1";  in panfrost_fence_get_timeline_name()
73 		return "panfrost-js-2";  in panfrost_fence_get_timeline_name()
87 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_fence_create()
91 		return ERR_PTR(-ENOMEM);  in panfrost_fence_create()
93 	fence->dev = pfdev->ddev;  in panfrost_fence_create()
94 	fence->queue = js_num;  in panfrost_fence_create()
95 	fence->seqno = ++js->queue[js_num].emit_seqno;  in panfrost_fence_create()
96 	dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,  in panfrost_fence_create()
97 		       js->queue[js_num].fence_context, fence->seqno);  in panfrost_fence_create()
99 	return &fence->base;  in panfrost_fence_create()
108 	if (job->requirements & PANFROST_JD_REQ_FS)  in panfrost_job_get_slot()
113 	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {  in panfrost_job_get_slot()
114 		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&  in panfrost_job_get_slot()
115 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
117 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
135 	affinity = pfdev->features.shader_present;  in panfrost_job_write_affinity()
143 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()
145 	u64 jc_head = job->jc;  in panfrost_job_hw_submit()
148 	panfrost_devfreq_record_busy(&pfdev->pfdevfreq);  in panfrost_job_hw_submit()
150 	ret = pm_runtime_get_sync(pfdev->dev);  in panfrost_job_hw_submit()
158 	cfg = panfrost_mmu_as_get(pfdev, &job->file_priv->mmu);  in panfrost_job_hw_submit()
163 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
165 	/* start MMU, medium priority, cache clean/flush on end, clean/flush on  in panfrost_job_hw_submit()
180 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
183 	dev_dbg(pfdev->dev, "JS: Submitting atom %p to js[%d] with head=0x%llx",  in panfrost_job_hw_submit()
196 		implicit_fences[i] = dma_resv_get_excl_rcu(bos[i]->resv);  in panfrost_acquire_object_fences()
206 		dma_resv_add_excl_fence(bos[i]->resv, fence);  in panfrost_attach_object_fences()
211 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()
213 	struct drm_sched_entity *entity = &job->file_priv->sched_entity[slot];  in panfrost_job_push()
217 	mutex_lock(&pfdev->sched_lock);  in panfrost_job_push()
219 	ret = drm_gem_lock_reservations(job->bos, job->bo_count,  in panfrost_job_push()
222 		mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
226 	ret = drm_sched_job_init(&job->base, entity, NULL);  in panfrost_job_push()
228 		mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
232 	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);  in panfrost_job_push()
234 	kref_get(&job->refcount); /* put by scheduler job completion */  in panfrost_job_push()
236 	panfrost_acquire_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
237 				       job->implicit_fences);  in panfrost_job_push()
239 	drm_sched_entity_push_job(&job->base, entity);  in panfrost_job_push()
241 	mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
243 	panfrost_attach_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
244 				      job->render_done_fence);  in panfrost_job_push()
247 	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);  in panfrost_job_push()
258 	if (job->in_fences) {  in panfrost_job_cleanup()
259 		for (i = 0; i < job->in_fence_count; i++)  in panfrost_job_cleanup()
260 			dma_fence_put(job->in_fences[i]);  in panfrost_job_cleanup()
261 		kvfree(job->in_fences);  in panfrost_job_cleanup()
263 	if (job->implicit_fences) {  in panfrost_job_cleanup()
264 		for (i = 0; i < job->bo_count; i++)  in panfrost_job_cleanup()
265 			dma_fence_put(job->implicit_fences[i]);  in panfrost_job_cleanup()
266 		kvfree(job->implicit_fences);  in panfrost_job_cleanup()
268 	dma_fence_put(job->done_fence);  in panfrost_job_cleanup()
269 	dma_fence_put(job->render_done_fence);  in panfrost_job_cleanup()
271 	if (job->mappings) {  in panfrost_job_cleanup()
272 		for (i = 0; i < job->bo_count; i++) {  in panfrost_job_cleanup()
273 			if (!job->mappings[i])  in panfrost_job_cleanup()
276 			atomic_dec(&job->mappings[i]->obj->gpu_usecount);  in panfrost_job_cleanup()
277 			panfrost_gem_mapping_put(job->mappings[i]);  in panfrost_job_cleanup()
279 		kvfree(job->mappings);  in panfrost_job_cleanup()
282 	if (job->bos) {  in panfrost_job_cleanup()
283 		for (i = 0; i < job->bo_count; i++)  in panfrost_job_cleanup()
284 			drm_gem_object_put(job->bos[i]);  in panfrost_job_cleanup()
286 		kvfree(job->bos);  in panfrost_job_cleanup()
294 	kref_put(&job->refcount, panfrost_job_cleanup);  in panfrost_job_put()
314 	for (i = 0; i < job->in_fence_count; i++) {  in panfrost_job_dependency()
315 		if (job->in_fences[i]) {  in panfrost_job_dependency()
316 			fence = job->in_fences[i];  in panfrost_job_dependency()
317 			job->in_fences[i] = NULL;  in panfrost_job_dependency()
323 	for (i = 0; i < job->bo_count; i++) {  in panfrost_job_dependency()
324 		if (job->implicit_fences[i]) {  in panfrost_job_dependency()
325 			fence = job->implicit_fences[i];  in panfrost_job_dependency()
326 			job->implicit_fences[i] = NULL;  in panfrost_job_dependency()
337 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()
341 	if (unlikely(job->base.s_fence->finished.error))  in panfrost_job_run()
344 	pfdev->jobs[slot] = job;  in panfrost_job_run()
350 	if (job->done_fence)  in panfrost_job_run()
351 		dma_fence_put(job->done_fence);  in panfrost_job_run()
352 	job->done_fence = dma_fence_get(fence);  in panfrost_job_run()
375 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()
384 	if (dma_fence_is_signaled(job->done_fence))  in panfrost_job_timedout()
387 …dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sch…  in panfrost_job_timedout()
395 	if (!mutex_trylock(&pfdev->reset_lock))  in panfrost_job_timedout()
399 		struct drm_gpu_scheduler *sched = &pfdev->js->queue[i].sched;  in panfrost_job_timedout()
404 			cancel_delayed_work_sync(&sched->work_tdr);  in panfrost_job_timedout()
409 	spin_lock_irqsave(&pfdev->js->job_lock, flags);  in panfrost_job_timedout()
411 		if (pfdev->jobs[i]) {  in panfrost_job_timedout()
412 			pm_runtime_put_noidle(pfdev->dev);  in panfrost_job_timedout()
413 			panfrost_devfreq_record_idle(&pfdev->pfdevfreq);  in panfrost_job_timedout()
414 			pfdev->jobs[i] = NULL;  in panfrost_job_timedout()
417 	spin_unlock_irqrestore(&pfdev->js->job_lock, flags);  in panfrost_job_timedout()
422 		drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);  in panfrost_job_timedout()
426 		drm_sched_start(&pfdev->js->queue[i].sched, true);  in panfrost_job_timedout()
428 	mutex_unlock(&pfdev->reset_lock);  in panfrost_job_timedout()
444 	dev_dbg(pfdev->dev, "jobslot irq status=%x\n", status);  in panfrost_job_irq_handler()
449 	pm_runtime_mark_last_busy(pfdev->dev);  in panfrost_job_irq_handler()
462 			dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",  in panfrost_job_irq_handler()
468 			drm_sched_fault(&pfdev->js->queue[j].sched);  in panfrost_job_irq_handler()
474 			spin_lock(&pfdev->js->job_lock);  in panfrost_job_irq_handler()
475 			job = pfdev->jobs[j];  in panfrost_job_irq_handler()
478 				pfdev->jobs[j] = NULL;  in panfrost_job_irq_handler()
480 				panfrost_mmu_as_put(pfdev, &job->file_priv->mmu);  in panfrost_job_irq_handler()
481 				panfrost_devfreq_record_idle(&pfdev->pfdevfreq);  in panfrost_job_irq_handler()
483 				dma_fence_signal_locked(job->done_fence);  in panfrost_job_irq_handler()
484 				pm_runtime_put_autosuspend(pfdev->dev);  in panfrost_job_irq_handler()
486 			spin_unlock(&pfdev->js->job_lock);  in panfrost_job_irq_handler()
500 	pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);  in panfrost_job_init()
502 		return -ENOMEM;  in panfrost_job_init()
504 	spin_lock_init(&js->job_lock);  in panfrost_job_init()
506 	irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");  in panfrost_job_init()
508 		return -ENODEV;  in panfrost_job_init()
510 	ret = devm_request_irq(pfdev->dev, irq, panfrost_job_irq_handler,  in panfrost_job_init()
511 			       IRQF_SHARED, KBUILD_MODNAME "-job", pfdev);  in panfrost_job_init()
513 		dev_err(pfdev->dev, "failed to request job irq");  in panfrost_job_init()
518 		js->queue[j].fence_context = dma_fence_context_alloc(1);  in panfrost_job_init()
520 		ret = drm_sched_init(&js->queue[j].sched,  in panfrost_job_init()
522 				     1, 0, msecs_to_jiffies(500),  in panfrost_job_init()
525 			dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);  in panfrost_job_init()
535 	for (j--; j >= 0; j--)  in panfrost_job_init()
536 		drm_sched_fini(&js->queue[j].sched);  in panfrost_job_init()
543 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_fini()
549 		drm_sched_fini(&js->queue[j].sched);  in panfrost_job_fini()
555 	struct panfrost_device *pfdev = panfrost_priv->pfdev;  in panfrost_job_open()
556 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_open()
561 		sched = &js->queue[i].sched;  in panfrost_job_open()
562 		ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i],  in panfrost_job_open()
576 		drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);  in panfrost_job_close()
581 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_is_idle()
586 		if (atomic_read(&js->queue[i].sched.hw_rq_count))  in panfrost_job_is_idle()