Lines Matching +full:mmu +full:- +full:500 +full:s
1 // SPDX-License-Identifier: GPL-2.0
10 #include <linux/dma-resv.h>
25 #define JOB_TIMEOUT_MS 500
27 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
28 #define job_read(dev, reg) readl(dev->iomem + (reg))
71 switch (f->queue) { in panfrost_fence_get_timeline_name()
73 return "panfrost-js-0"; in panfrost_fence_get_timeline_name()
75 return "panfrost-js-1"; in panfrost_fence_get_timeline_name()
77 return "panfrost-js-2"; in panfrost_fence_get_timeline_name()
91 struct panfrost_job_slot *js = pfdev->js; in panfrost_fence_create()
95 return ERR_PTR(-ENOMEM); in panfrost_fence_create()
97 fence->dev = pfdev->ddev; in panfrost_fence_create()
98 fence->queue = js_num; in panfrost_fence_create()
99 fence->seqno = ++js->queue[js_num].emit_seqno; in panfrost_fence_create()
100 dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock, in panfrost_fence_create()
101 js->queue[js_num].fence_context, fence->seqno); in panfrost_fence_create()
103 return &fence->base; in panfrost_fence_create()
112 if (job->requirements & PANFROST_JD_REQ_FS) in panfrost_job_get_slot()
117 if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) { in panfrost_job_get_slot()
118 if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) && in panfrost_job_get_slot()
119 (job->pfdev->features.nr_core_groups == 2)) in panfrost_job_get_slot()
121 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987)) in panfrost_job_get_slot()
139 affinity = pfdev->features.shader_present; in panfrost_job_write_affinity()
148 struct panfrost_fence *f = to_panfrost_fence(job->done_fence); in panfrost_get_job_chain_flag()
150 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) in panfrost_get_job_chain_flag()
153 return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0; in panfrost_get_job_chain_flag()
159 struct panfrost_job *job = pfdev->jobs[slot][0]; in panfrost_dequeue_job()
162 pfdev->jobs[slot][0] = pfdev->jobs[slot][1]; in panfrost_dequeue_job()
163 pfdev->jobs[slot][1] = NULL; in panfrost_dequeue_job()
175 if (!pfdev->jobs[slot][0]) { in panfrost_enqueue_job()
176 pfdev->jobs[slot][0] = job; in panfrost_enqueue_job()
180 WARN_ON(pfdev->jobs[slot][1]); in panfrost_enqueue_job()
181 pfdev->jobs[slot][1] = job; in panfrost_enqueue_job()
183 panfrost_get_job_chain_flag(pfdev->jobs[slot][0])); in panfrost_enqueue_job()
189 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_hw_submit()
192 u64 jc_head = job->jc; in panfrost_job_hw_submit()
195 panfrost_devfreq_record_busy(&pfdev->pfdevfreq); in panfrost_job_hw_submit()
197 ret = pm_runtime_get_sync(pfdev->dev); in panfrost_job_hw_submit()
205 cfg = panfrost_mmu_as_get(pfdev, job->mmu); in panfrost_job_hw_submit()
210 panfrost_job_write_affinity(pfdev, job->requirements, js); in panfrost_job_hw_submit()
212 /* start MMU, medium priority, cache clean/flush on end, clean/flush on in panfrost_job_hw_submit()
228 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id); in panfrost_job_hw_submit()
232 spin_lock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
235 if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_hw_submit()
237 dev_dbg(pfdev->dev, in panfrost_job_hw_submit()
241 spin_unlock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
251 ret = dma_resv_reserve_fences(bos[i]->resv, 1); in panfrost_acquire_object_fences()
272 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE); in panfrost_attach_object_fences()
277 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_push()
281 ret = drm_gem_lock_reservations(job->bos, job->bo_count, in panfrost_job_push()
286 mutex_lock(&pfdev->sched_lock); in panfrost_job_push()
287 drm_sched_job_arm(&job->base); in panfrost_job_push()
289 job->render_done_fence = dma_fence_get(&job->base.s_fence->finished); in panfrost_job_push()
291 ret = panfrost_acquire_object_fences(job->bos, job->bo_count, in panfrost_job_push()
292 &job->base); in panfrost_job_push()
294 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
298 kref_get(&job->refcount); /* put by scheduler job completion */ in panfrost_job_push()
300 drm_sched_entity_push_job(&job->base); in panfrost_job_push()
302 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
304 panfrost_attach_object_fences(job->bos, job->bo_count, in panfrost_job_push()
305 job->render_done_fence); in panfrost_job_push()
308 drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx); in panfrost_job_push()
319 dma_fence_put(job->done_fence); in panfrost_job_cleanup()
320 dma_fence_put(job->render_done_fence); in panfrost_job_cleanup()
322 if (job->mappings) { in panfrost_job_cleanup()
323 for (i = 0; i < job->bo_count; i++) { in panfrost_job_cleanup()
324 if (!job->mappings[i]) in panfrost_job_cleanup()
327 atomic_dec(&job->mappings[i]->obj->gpu_usecount); in panfrost_job_cleanup()
328 panfrost_gem_mapping_put(job->mappings[i]); in panfrost_job_cleanup()
330 kvfree(job->mappings); in panfrost_job_cleanup()
333 if (job->bos) { in panfrost_job_cleanup()
334 for (i = 0; i < job->bo_count; i++) in panfrost_job_cleanup()
335 drm_gem_object_put(job->bos[i]); in panfrost_job_cleanup()
337 kvfree(job->bos); in panfrost_job_cleanup()
345 kref_put(&job->refcount, panfrost_job_cleanup); in panfrost_job_put()
360 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_run()
364 if (unlikely(job->base.s_fence->finished.error)) in panfrost_job_run()
370 if (!job->jc) in panfrost_job_run()
377 if (job->done_fence) in panfrost_job_run()
378 dma_fence_put(job->done_fence); in panfrost_job_run()
379 job->done_fence = dma_fence_get(fence); in panfrost_job_run()
408 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
413 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
421 job->jc = job_read(pfdev, JS_TAIL_LO(js)) | in panfrost_job_handle_err()
427 /* Job has been hard-stopped, flag it as canceled */ in panfrost_job_handle_err()
428 dma_fence_set_error(job->done_fence, -ECANCELED); in panfrost_job_handle_err()
429 job->jc = 0; in panfrost_job_handle_err()
431 /* We might want to provide finer-grained error code based on in panfrost_job_handle_err()
435 dma_fence_set_error(job->done_fence, -EINVAL); in panfrost_job_handle_err()
436 job->jc = 0; in panfrost_job_handle_err()
439 panfrost_mmu_as_put(pfdev, job->mmu); in panfrost_job_handle_err()
440 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_err()
443 dma_fence_signal_locked(job->done_fence); in panfrost_job_handle_err()
445 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_err()
448 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_handle_err()
449 drm_sched_fault(&pfdev->js->queue[js].sched); in panfrost_job_handle_err()
456 /* Set ->jc to 0 to avoid re-submitting an already finished job (can in panfrost_job_handle_done()
459 job->jc = 0; in panfrost_job_handle_done()
460 panfrost_mmu_as_put(pfdev, job->mmu); in panfrost_job_handle_done()
461 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_done()
463 dma_fence_signal_locked(job->done_fence); in panfrost_job_handle_done()
464 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_done()
492 * there's no reset pending. in panfrost_job_handle_irq()
508 * To prevent that, let's repeat this clear+read steps in panfrost_job_handle_irq()
525 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) { in panfrost_job_handle_irq()
530 * are inactive, but one job remains in pfdev->jobs[j], in panfrost_job_handle_irq()
552 if (!failed[j] || !pfdev->jobs[j][0]) in panfrost_job_handle_irq()
555 if (pfdev->jobs[j][0]->jc == 0) { in panfrost_job_handle_irq()
559 dma_fence_set_error(canceled->done_fence, -ECANCELED); in panfrost_job_handle_irq()
561 } else if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_handle_irq()
573 pm_runtime_mark_last_busy(pfdev->dev); in panfrost_job_handle_irqs()
575 spin_lock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
577 spin_unlock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
612 if (!atomic_read(&pfdev->reset.pending)) in panfrost_reset()
629 drm_sched_stop(&pfdev->js->queue[i].sched, bad); in panfrost_reset()
640 synchronize_irq(pfdev->js->irq); in panfrost_reset()
643 /* Cancel the next job and soft-stop the running job. */ in panfrost_reset()
648 /* Wait at most 10ms for soft-stops to complete */ in panfrost_reset()
649 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state, in panfrost_reset()
654 dev_err(pfdev->dev, "Soft-stop failed\n"); in panfrost_reset()
660 * stuck jobs. Let's make sure the PM counters stay balanced by in panfrost_reset()
664 spin_lock(&pfdev->js->job_lock); in panfrost_reset()
666 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) { in panfrost_reset()
667 pm_runtime_put_noidle(pfdev->dev); in panfrost_reset()
668 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_reset()
671 memset(pfdev->jobs, 0, sizeof(pfdev->jobs)); in panfrost_reset()
672 spin_unlock(&pfdev->js->job_lock); in panfrost_reset()
683 atomic_set(&pfdev->reset.pending, 0); in panfrost_reset()
694 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched); in panfrost_reset()
699 drm_sched_start(&pfdev->js->queue[i].sched, true); in panfrost_reset()
701 /* Re-enable job interrupts now that everything has been restarted. */ in panfrost_reset()
703 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | in panfrost_reset()
704 GENMASK(NUM_JOB_SLOTS - 1, 0)); in panfrost_reset()
713 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_timedout()
720 if (dma_fence_is_signaled(job->done_fence)) in panfrost_job_timedout()
723 …dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sch… in panfrost_job_timedout()
733 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_timedout()
759 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | in panfrost_job_irq_handler_thread()
760 GENMASK(NUM_JOB_SLOTS - 1, 0)); in panfrost_job_irq_handler_thread()
784 * so let's just advertise one entry in that case. in panfrost_job_init()
789 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL); in panfrost_job_init()
791 return -ENOMEM; in panfrost_job_init()
793 INIT_WORK(&pfdev->reset.work, panfrost_reset_work); in panfrost_job_init()
794 spin_lock_init(&js->job_lock); in panfrost_job_init()
796 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job"); in panfrost_job_init()
797 if (js->irq <= 0) in panfrost_job_init()
798 return -ENODEV; in panfrost_job_init()
800 ret = devm_request_threaded_irq(pfdev->dev, js->irq, in panfrost_job_init()
803 IRQF_SHARED, KBUILD_MODNAME "-job", in panfrost_job_init()
806 dev_err(pfdev->dev, "failed to request job irq"); in panfrost_job_init()
810 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0); in panfrost_job_init()
811 if (!pfdev->reset.wq) in panfrost_job_init()
812 return -ENOMEM; in panfrost_job_init()
815 js->queue[j].fence_context = dma_fence_context_alloc(1); in panfrost_job_init()
817 ret = drm_sched_init(&js->queue[j].sched, in panfrost_job_init()
821 pfdev->reset.wq, in panfrost_job_init()
822 NULL, "pan_js", pfdev->dev); in panfrost_job_init()
824 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret); in panfrost_job_init()
834 for (j--; j >= 0; j--) in panfrost_job_init()
835 drm_sched_fini(&js->queue[j].sched); in panfrost_job_init()
837 destroy_workqueue(pfdev->reset.wq); in panfrost_job_init()
843 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_fini()
849 drm_sched_fini(&js->queue[j].sched); in panfrost_job_fini()
852 cancel_work_sync(&pfdev->reset.work); in panfrost_job_fini()
853 destroy_workqueue(pfdev->reset.wq); in panfrost_job_fini()
858 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_open()
859 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_open()
864 sched = &js->queue[i].sched; in panfrost_job_open()
865 ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i], in panfrost_job_open()
876 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_close()
880 drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]); in panfrost_job_close()
882 /* Kill in-flight jobs */ in panfrost_job_close()
883 spin_lock(&pfdev->js->job_lock); in panfrost_job_close()
885 struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i]; in panfrost_job_close()
888 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) { in panfrost_job_close()
889 struct panfrost_job *job = pfdev->jobs[i][j]; in panfrost_job_close()
892 if (!job || job->base.entity != entity) in panfrost_job_close()
901 job->jc = 0; in panfrost_job_close()
915 spin_unlock(&pfdev->js->job_lock); in panfrost_job_close()
920 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_is_idle()
925 if (atomic_read(&js->queue[i].sched.hw_rq_count)) in panfrost_job_is_idle()