87 static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)  in panfrost_fence_create()  argument
90 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_fence_create()
96 	fence->dev = pfdev->ddev;  in panfrost_fence_create()
118 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
120 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
127 static void panfrost_job_write_affinity(struct panfrost_device *pfdev,  in panfrost_job_write_affinity()  argument
138 	affinity = pfdev->features.shader_present;  in panfrost_job_write_affinity()
140 	job_write(pfdev, JS_AFFINITY_NEXT_LO(js), affinity & 0xFFFFFFFF);  in panfrost_job_write_affinity()
141 	job_write(pfdev, JS_AFFINITY_NEXT_HI(js), affinity >> 32);  in panfrost_job_write_affinity()
149 	if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))  in panfrost_get_job_chain_flag()
156 panfrost_dequeue_job(struct panfrost_device *pfdev, int slot)  in panfrost_dequeue_job()  argument
158 	struct panfrost_job *job = pfdev->jobs[slot][0];  in panfrost_dequeue_job()
161 	pfdev->jobs[slot][0] = pfdev->jobs[slot][1];  in panfrost_dequeue_job()
162 	pfdev->jobs[slot][1] = NULL;  in panfrost_dequeue_job()
168 panfrost_enqueue_job(struct panfrost_device *pfdev, int slot,  in panfrost_enqueue_job()  argument
174 	if (!pfdev->jobs[slot][0]) {  in panfrost_enqueue_job()
175 		pfdev->jobs[slot][0] = job;  in panfrost_enqueue_job()
179 	WARN_ON(pfdev->jobs[slot][1]);  in panfrost_enqueue_job()
180 	pfdev->jobs[slot][1] = job;  in panfrost_enqueue_job()
182 		panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));  in panfrost_enqueue_job()
188 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()  local
194 	panfrost_devfreq_record_busy(&pfdev->pfdevfreq);  in panfrost_job_hw_submit()
196 	ret = pm_runtime_get_sync(pfdev->dev);  in panfrost_job_hw_submit()
200 	if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) {  in panfrost_job_hw_submit()
204 	cfg = panfrost_mmu_as_get(pfdev, job->file_priv->mmu);  in panfrost_job_hw_submit()
206 	job_write(pfdev, JS_HEAD_NEXT_LO(js), jc_head & 0xFFFFFFFF);  in panfrost_job_hw_submit()
207 	job_write(pfdev, JS_HEAD_NEXT_HI(js), jc_head >> 32);  in panfrost_job_hw_submit()
209 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
218 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))  in panfrost_job_hw_submit()
221 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))  in panfrost_job_hw_submit()
224 	job_write(pfdev, JS_CONFIG_NEXT(js), cfg);  in panfrost_job_hw_submit()
226 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))  in panfrost_job_hw_submit()
227 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
231 	spin_lock(&pfdev->js->job_lock);  in panfrost_job_hw_submit()
232 	subslot = panfrost_enqueue_job(pfdev, js, job);  in panfrost_job_hw_submit()
234 	if (!atomic_read(&pfdev->reset.pending)) {  in panfrost_job_hw_submit()
235 		job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);  in panfrost_job_hw_submit()
236 		dev_dbg(pfdev->dev,  in panfrost_job_hw_submit()
240 	spin_unlock(&pfdev->js->job_lock);  in panfrost_job_hw_submit()
271 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()  local
283 	mutex_lock(&pfdev->sched_lock);  in panfrost_job_push()
287 		mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
296 		mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
304 	mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
380 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()  local
393 	fence = panfrost_fence_create(pfdev, slot);  in panfrost_job_run()
406 void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)  in panfrost_job_enable_interrupts()  argument
415 	job_write(pfdev, JOB_INT_CLEAR, irq_mask);  in panfrost_job_enable_interrupts()
416 	job_write(pfdev, JOB_INT_MASK, irq_mask);  in panfrost_job_enable_interrupts()
419 static void panfrost_job_handle_err(struct panfrost_device *pfdev,  in panfrost_job_handle_err()  argument
423 	u32 js_status = job_read(pfdev, JS_STATUS(js));  in panfrost_job_handle_err()
428 		dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",  in panfrost_job_handle_err()
430 			job_read(pfdev, JS_HEAD_LO(js)),  in panfrost_job_handle_err()
431 			job_read(pfdev, JS_TAIL_LO(js)));  in panfrost_job_handle_err()
433 		dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",  in panfrost_job_handle_err()
435 			job_read(pfdev, JS_HEAD_LO(js)),  in panfrost_job_handle_err()
436 			job_read(pfdev, JS_TAIL_LO(js)));  in panfrost_job_handle_err()
441 		job->jc = job_read(pfdev, JS_TAIL_LO(js)) |  in panfrost_job_handle_err()
442 			  ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32);  in panfrost_job_handle_err()
459 	panfrost_mmu_as_put(pfdev, job->file_priv->mmu);  in panfrost_job_handle_err()
460 	panfrost_devfreq_record_idle(&pfdev->pfdevfreq);  in panfrost_job_handle_err()
465 	pm_runtime_put_autosuspend(pfdev->dev);  in panfrost_job_handle_err()
467 	if (panfrost_exception_needs_reset(pfdev, js_status)) {  in panfrost_job_handle_err()
468 		atomic_set(&pfdev->reset.pending, 1);  in panfrost_job_handle_err()
469 		drm_sched_fault(&pfdev->js->queue[js].sched);  in panfrost_job_handle_err()
473 static void panfrost_job_handle_done(struct panfrost_device *pfdev,  in panfrost_job_handle_done()  argument
480 	panfrost_mmu_as_put(pfdev, job->file_priv->mmu);  in panfrost_job_handle_done()
481 	panfrost_devfreq_record_idle(&pfdev->pfdevfreq);  in panfrost_job_handle_done()
484 	pm_runtime_put_autosuspend(pfdev->dev);  in panfrost_job_handle_done()
487 static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status)  in panfrost_job_handle_irq()  argument
504 					done[j][1] = panfrost_dequeue_job(pfdev, j);  in panfrost_job_handle_irq()
506 					done[j][0] = panfrost_dequeue_job(pfdev, j);  in panfrost_job_handle_irq()
514 				job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);  in panfrost_job_handle_irq()
515 				failed[j] = panfrost_dequeue_job(pfdev, j);  in panfrost_job_handle_irq()
531 		job_write(pfdev, JOB_INT_CLEAR, status);  in panfrost_job_handle_irq()
533 		js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask;  in panfrost_job_handle_irq()
535 		status = job_read(pfdev, JOB_INT_RAWSTAT);  in panfrost_job_handle_irq()
544 			panfrost_job_handle_err(pfdev, failed[j], j);  in panfrost_job_handle_irq()
545 		} else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {  in panfrost_job_handle_irq()
556 				done[j][0] = panfrost_dequeue_job(pfdev, j);  in panfrost_job_handle_irq()
558 				done[j][1] = panfrost_dequeue_job(pfdev, j);  in panfrost_job_handle_irq()
562 			panfrost_job_handle_done(pfdev, done[j][i]);  in panfrost_job_handle_irq()
572 		if (!failed[j] || !pfdev->jobs[j][0])  in panfrost_job_handle_irq()
575 		if (pfdev->jobs[j][0]->jc == 0) {  in panfrost_job_handle_irq()
577 			struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j);  in panfrost_job_handle_irq()
580 			panfrost_job_handle_done(pfdev, canceled);  in panfrost_job_handle_irq()
581 		} else if (!atomic_read(&pfdev->reset.pending)) {  in panfrost_job_handle_irq()
583 			job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START);  in panfrost_job_handle_irq()
588 static void panfrost_job_handle_irqs(struct panfrost_device *pfdev)  in panfrost_job_handle_irqs()  argument
590 	u32 status = job_read(pfdev, JOB_INT_RAWSTAT);  in panfrost_job_handle_irqs()
593 		pm_runtime_mark_last_busy(pfdev->dev);  in panfrost_job_handle_irqs()
595 		spin_lock(&pfdev->js->job_lock);  in panfrost_job_handle_irqs()
596 		panfrost_job_handle_irq(pfdev, status);  in panfrost_job_handle_irqs()
597 		spin_unlock(&pfdev->js->job_lock);  in panfrost_job_handle_irqs()
598 		status = job_read(pfdev, JOB_INT_RAWSTAT);  in panfrost_job_handle_irqs()
602 static u32 panfrost_active_slots(struct panfrost_device *pfdev,  in panfrost_active_slots()  argument
610 	rawstat = job_read(pfdev, JOB_INT_RAWSTAT);  in panfrost_active_slots()
624 panfrost_reset(struct panfrost_device *pfdev,  in panfrost_reset()  argument
632 	if (!atomic_read(&pfdev->reset.pending))  in panfrost_reset()
649 		drm_sched_stop(&pfdev->js->queue[i].sched, bad);  in panfrost_reset()
659 	job_write(pfdev, JOB_INT_MASK, 0);  in panfrost_reset()
660 	synchronize_irq(pfdev->js->irq);  in panfrost_reset()
664 		job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);  in panfrost_reset()
665 		job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP);  in panfrost_reset()
669 	ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,  in panfrost_reset()
670 				 !panfrost_active_slots(pfdev, &js_state_mask, js_state),  in panfrost_reset()
674 		dev_err(pfdev->dev, "Soft-stop failed\n");  in panfrost_reset()
677 	panfrost_job_handle_irqs(pfdev);  in panfrost_reset()
684 	spin_lock(&pfdev->js->job_lock);  in panfrost_reset()
686 		for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) {  in panfrost_reset()
687 			pm_runtime_put_noidle(pfdev->dev);  in panfrost_reset()
688 			panfrost_devfreq_record_idle(&pfdev->pfdevfreq);  in panfrost_reset()
691 	memset(pfdev->jobs, 0, sizeof(pfdev->jobs));  in panfrost_reset()
692 	spin_unlock(&pfdev->js->job_lock);  in panfrost_reset()
695 	panfrost_device_reset(pfdev);  in panfrost_reset()
700 	job_write(pfdev, JOB_INT_MASK, 0);  in panfrost_reset()
703 	atomic_set(&pfdev->reset.pending, 0);  in panfrost_reset()
714 		drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);  in panfrost_reset()
719 		drm_sched_start(&pfdev->js->queue[i].sched, true);  in panfrost_reset()
722 	job_write(pfdev, JOB_INT_MASK,  in panfrost_reset()
733 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()  local
743 …dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sch…  in panfrost_job_timedout()
745 		job_read(pfdev, JS_CONFIG(js)),  in panfrost_job_timedout()
746 		job_read(pfdev, JS_STATUS(js)),  in panfrost_job_timedout()
747 		job_read(pfdev, JS_HEAD_LO(js)),  in panfrost_job_timedout()
748 		job_read(pfdev, JS_TAIL_LO(js)),  in panfrost_job_timedout()
751 	atomic_set(&pfdev->reset.pending, 1);  in panfrost_job_timedout()
752 	panfrost_reset(pfdev, sched_job);  in panfrost_job_timedout()
759 	struct panfrost_device *pfdev;  in panfrost_reset_work()  local
761 	pfdev = container_of(work, struct panfrost_device, reset.work);  in panfrost_reset_work()
762 	panfrost_reset(pfdev, NULL);  in panfrost_reset_work()
774 	struct panfrost_device *pfdev = data;  in panfrost_job_irq_handler_thread()  local
776 	panfrost_job_handle_irqs(pfdev);  in panfrost_job_irq_handler_thread()
777 	job_write(pfdev, JOB_INT_MASK,  in panfrost_job_irq_handler_thread()
785 	struct panfrost_device *pfdev = data;  in panfrost_job_irq_handler()  local
786 	u32 status = job_read(pfdev, JOB_INT_STAT);  in panfrost_job_irq_handler()
791 	job_write(pfdev, JOB_INT_MASK, 0);  in panfrost_job_irq_handler()
795 int panfrost_job_init(struct panfrost_device *pfdev)  in panfrost_job_init()  argument
805 	if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))  in panfrost_job_init()
808 	pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);  in panfrost_job_init()
812 	INIT_WORK(&pfdev->reset.work, panfrost_reset_work);  in panfrost_job_init()
815 	js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");  in panfrost_job_init()
819 	ret = devm_request_threaded_irq(pfdev->dev, js->irq,  in panfrost_job_init()
823 					pfdev);  in panfrost_job_init()
825 		dev_err(pfdev->dev, "failed to request job irq");  in panfrost_job_init()
829 	pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0);  in panfrost_job_init()
830 	if (!pfdev->reset.wq)  in panfrost_job_init()
840 				     pfdev->reset.wq,  in panfrost_job_init()
843 			dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);  in panfrost_job_init()
848 	panfrost_job_enable_interrupts(pfdev);  in panfrost_job_init()
856 	destroy_workqueue(pfdev->reset.wq);  in panfrost_job_init()
860 void panfrost_job_fini(struct panfrost_device *pfdev)  in panfrost_job_fini()  argument
862 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_fini()
865 	job_write(pfdev, JOB_INT_MASK, 0);  in panfrost_job_fini()
871 	cancel_work_sync(&pfdev->reset.work);  in panfrost_job_fini()
872 	destroy_workqueue(pfdev->reset.wq);  in panfrost_job_fini()
877 	struct panfrost_device *pfdev = panfrost_priv->pfdev;  in panfrost_job_open()  local
878 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_open()
895 	struct panfrost_device *pfdev = panfrost_priv->pfdev;  in panfrost_job_close()  local
902 	spin_lock(&pfdev->js->job_lock);  in panfrost_job_close()
907 		for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {  in panfrost_job_close()
908 			struct panfrost_job *job = pfdev->jobs[i][j];  in panfrost_job_close()
916 				job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);  in panfrost_job_close()
923 			if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {  in panfrost_job_close()
931 			job_write(pfdev, JS_COMMAND(i), cmd);  in panfrost_job_close()
934 	spin_unlock(&pfdev->js->job_lock);  in panfrost_job_close()
937 int panfrost_job_is_idle(struct panfrost_device *pfdev)  in panfrost_job_is_idle()  argument
939 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_is_idle()