105 static int panfrost_job_get_slot(struct panfrost_job *job)  in panfrost_job_get_slot()  argument
111 	if (job->requirements & PANFROST_JD_REQ_FS)  in panfrost_job_get_slot()
116 	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {  in panfrost_job_get_slot()
117 		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&  in panfrost_job_get_slot()
118 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
120 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
145 panfrost_get_job_chain_flag(const struct panfrost_job *job)  in panfrost_get_job_chain_flag()  argument
147 	struct panfrost_fence *f = to_panfrost_fence(job->done_fence);  in panfrost_get_job_chain_flag()
149 	if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))  in panfrost_get_job_chain_flag()
158 	struct panfrost_job *job = pfdev->jobs[slot][0];  in panfrost_dequeue_job()  local
160 	WARN_ON(!job);  in panfrost_dequeue_job()
164 	return job;  in panfrost_dequeue_job()
169 		     struct panfrost_job *job)  in panfrost_enqueue_job()  argument
171 	if (WARN_ON(!job))  in panfrost_enqueue_job()
175 		pfdev->jobs[slot][0] = job;  in panfrost_enqueue_job()
180 	pfdev->jobs[slot][1] = job;  in panfrost_enqueue_job()
181 	WARN_ON(panfrost_get_job_chain_flag(job) ==  in panfrost_enqueue_job()
186 static void panfrost_job_hw_submit(struct panfrost_job *job, int js)  in panfrost_job_hw_submit()  argument
188 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()
191 	u64 jc_head = job->jc;  in panfrost_job_hw_submit()
204 	cfg = panfrost_mmu_as_get(pfdev, job->file_priv->mmu);  in panfrost_job_hw_submit()
209 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
216 		panfrost_get_job_chain_flag(job);  in panfrost_job_hw_submit()
227 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
232 	subslot = panfrost_enqueue_job(pfdev, js, job);  in panfrost_job_hw_submit()
233 	/* Don't queue the job if a reset is in progress */  in panfrost_job_hw_submit()
238 			job, js, subslot, jc_head, cfg & 0xf);  in panfrost_job_hw_submit()
269 int panfrost_job_push(struct panfrost_job *job)  in panfrost_job_push()  argument
271 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()
272 	int slot = panfrost_job_get_slot(job);  in panfrost_job_push()
273 	struct drm_sched_entity *entity = &job->file_priv->sched_entity[slot];  in panfrost_job_push()
278 	ret = drm_gem_lock_reservations(job->bos, job->bo_count,  in panfrost_job_push()
285 	ret = drm_sched_job_init(&job->base, entity, NULL);  in panfrost_job_push()
291 	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);  in panfrost_job_push()
293 	ret = panfrost_acquire_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
294 					     &job->deps);  in panfrost_job_push()
300 	kref_get(&job->refcount); /* put by scheduler job completion */  in panfrost_job_push()
302 	drm_sched_entity_push_job(&job->base, entity);  in panfrost_job_push()
306 	panfrost_attach_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
307 				      job->render_done_fence);  in panfrost_job_push()
310 	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);  in panfrost_job_push()
317 	struct panfrost_job *job = container_of(ref, struct panfrost_job,  in panfrost_job_cleanup()  local
323 	xa_for_each(&job->deps, index, fence) {  in panfrost_job_cleanup()
326 	xa_destroy(&job->deps);  in panfrost_job_cleanup()
328 	dma_fence_put(job->done_fence);  in panfrost_job_cleanup()
329 	dma_fence_put(job->render_done_fence);  in panfrost_job_cleanup()
331 	if (job->mappings) {  in panfrost_job_cleanup()
332 		for (i = 0; i < job->bo_count; i++) {  in panfrost_job_cleanup()
333 			if (!job->mappings[i])  in panfrost_job_cleanup()
336 			atomic_dec(&job->mappings[i]->obj->gpu_usecount);  in panfrost_job_cleanup()
337 			panfrost_gem_mapping_put(job->mappings[i]);  in panfrost_job_cleanup()
339 		kvfree(job->mappings);  in panfrost_job_cleanup()
342 	if (job->bos) {  in panfrost_job_cleanup()
343 		for (i = 0; i < job->bo_count; i++)  in panfrost_job_cleanup()
344 			drm_gem_object_put(job->bos[i]);  in panfrost_job_cleanup()
346 		kvfree(job->bos);  in panfrost_job_cleanup()
349 	kfree(job);  in panfrost_job_cleanup()
352 void panfrost_job_put(struct panfrost_job *job)  in panfrost_job_put()  argument
354 	kref_put(&job->refcount, panfrost_job_cleanup);  in panfrost_job_put()
359 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_free()  local
363 	panfrost_job_put(job);  in panfrost_job_free()
369 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_dependency()  local
371 	if (!xa_empty(&job->deps))  in panfrost_job_dependency()
372 		return xa_erase(&job->deps, job->last_dep++);  in panfrost_job_dependency()
379 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_run()  local
380 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()
381 	int slot = panfrost_job_get_slot(job);  in panfrost_job_run()
384 	if (unlikely(job->base.s_fence->finished.error))  in panfrost_job_run()
387 	/* Nothing to execute: can happen if the job has finished while  in panfrost_job_run()
390 	if (!job->jc)  in panfrost_job_run()
397 	if (job->done_fence)  in panfrost_job_run()
398 		dma_fence_put(job->done_fence);  in panfrost_job_run()
399 	job->done_fence = dma_fence_get(fence);  in panfrost_job_run()
401 	panfrost_job_hw_submit(job, slot);  in panfrost_job_run()
420 				    struct panfrost_job *job,  in panfrost_job_handle_err()  argument
440 		/* Update the job head so we can resume */  in panfrost_job_handle_err()
441 		job->jc = job_read(pfdev, JS_TAIL_LO(js)) |  in panfrost_job_handle_err()
444 		/* The job will be resumed, don't signal the fence */  in panfrost_job_handle_err()
447 		/* Job has been hard-stopped, flag it as canceled */  in panfrost_job_handle_err()
448 		dma_fence_set_error(job->done_fence, -ECANCELED);  in panfrost_job_handle_err()
449 		job->jc = 0;  in panfrost_job_handle_err()
455 		dma_fence_set_error(job->done_fence, -EINVAL);  in panfrost_job_handle_err()
456 		job->jc = 0;  in panfrost_job_handle_err()
459 	panfrost_mmu_as_put(pfdev, job->file_priv->mmu);  in panfrost_job_handle_err()
463 		dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_err()
474 				     struct panfrost_job *job)  in panfrost_job_handle_done()  argument
476 	/* Set ->jc to 0 to avoid re-submitting an already finished job (can  in panfrost_job_handle_done()
479 	job->jc = 0;  in panfrost_job_handle_done()
480 	panfrost_mmu_as_put(pfdev, job->file_priv->mmu);  in panfrost_job_handle_done()
483 	dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_done()
523 		 * is racy. If we only have one job done at the time we  in panfrost_job_handle_irq()
524 		 * read JOB_INT_RAWSTAT but the second job fails before we  in panfrost_job_handle_irq()
546 			/* When the current job doesn't fail, the JM dequeues  in panfrost_job_handle_irq()
547 			 * the next job without waiting for an ACK, this means  in panfrost_job_handle_irq()
550 			 * are inactive, but one job remains in pfdev->jobs[j],  in panfrost_job_handle_irq()
553 			 * job in _NEXT (see above).  in panfrost_job_handle_irq()
576 			/* The job was cancelled, signal the fence now */  in panfrost_job_handle_irq()
582 			/* Requeue the job we removed if no reset is pending */  in panfrost_job_handle_irq()
639 	 * to release job resources. We should rework the code to follow this  in panfrost_reset()
656 	/* Mask job interrupts and synchronize to make sure we won't be  in panfrost_reset()
663 		/* Cancel the next job and soft-stop the running job. */  in panfrost_reset()
682 	 * panfrost_devfreq_record_idle() for each stuck job.  in panfrost_reset()
697 	/* panfrost_device_reset() unmasks job interrupts, but we want to  in panfrost_reset()
708 	 * while resubmitting jobs because the job submission logic will  in panfrost_reset()
721 	/* Re-enable job interrupts now that everything has been restarted. */  in panfrost_reset()
732 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_timedout()  local
733 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()
734 	int js = panfrost_job_get_slot(job);  in panfrost_job_timedout()
740 	if (dma_fence_is_signaled(job->done_fence))  in panfrost_job_timedout()
802 	 * disambiguation stopping the right job in the close path is tricky,  in panfrost_job_init()
815 	js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");  in panfrost_job_init()
822 					IRQF_SHARED, KBUILD_MODNAME "-job",  in panfrost_job_init()
825 		dev_err(pfdev->dev, "failed to request job irq");  in panfrost_job_init()
908 			struct panfrost_job *job = pfdev->jobs[i][j];  in panfrost_job_close()  local
911 			if (!job || job->base.entity != entity)  in panfrost_job_close()
915 				/* Try to cancel the job before it starts */  in panfrost_job_close()
917 				/* Reset the job head so it doesn't get restarted if  in panfrost_job_close()
918 				 * the job in the first slot failed.  in panfrost_job_close()
920 				job->jc = 0;  in panfrost_job_close()
924 				cmd = panfrost_get_job_chain_flag(job) ?  in panfrost_job_close()