106 int panfrost_job_get_slot(struct panfrost_job *job)  in panfrost_job_get_slot()  argument
112 	if (job->requirements & PANFROST_JD_REQ_FS)  in panfrost_job_get_slot()
117 	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {  in panfrost_job_get_slot()
118 		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&  in panfrost_job_get_slot()
119 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
121 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
146 panfrost_get_job_chain_flag(const struct panfrost_job *job)  in panfrost_get_job_chain_flag()  argument
148 	struct panfrost_fence *f = to_panfrost_fence(job->done_fence);  in panfrost_get_job_chain_flag()
150 	if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))  in panfrost_get_job_chain_flag()
159 	struct panfrost_job *job = pfdev->jobs[slot][0];  in panfrost_dequeue_job()  local
161 	WARN_ON(!job);  in panfrost_dequeue_job()
165 	return job;  in panfrost_dequeue_job()
170 		     struct panfrost_job *job)  in panfrost_enqueue_job()  argument
172 	if (WARN_ON(!job))  in panfrost_enqueue_job()
176 		pfdev->jobs[slot][0] = job;  in panfrost_enqueue_job()
181 	pfdev->jobs[slot][1] = job;  in panfrost_enqueue_job()
182 	WARN_ON(panfrost_get_job_chain_flag(job) ==  in panfrost_enqueue_job()
187 static void panfrost_job_hw_submit(struct panfrost_job *job, int js)  in panfrost_job_hw_submit()  argument
189 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()
192 	u64 jc_head = job->jc;  in panfrost_job_hw_submit()
205 	cfg = panfrost_mmu_as_get(pfdev, job->mmu);  in panfrost_job_hw_submit()
210 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
217 		panfrost_get_job_chain_flag(job);  in panfrost_job_hw_submit()
228 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
233 	subslot = panfrost_enqueue_job(pfdev, js, job);  in panfrost_job_hw_submit()
234 	/* Don't queue the job if a reset is in progress */  in panfrost_job_hw_submit()
239 			job, js, subslot, jc_head, cfg & 0xf);  in panfrost_job_hw_submit()
246 					  struct drm_sched_job *job)  in panfrost_acquire_object_fences()  argument
256 		ret = drm_sched_job_add_implicit_dependencies(job, bos[i],  in panfrost_acquire_object_fences()
275 int panfrost_job_push(struct panfrost_job *job)  in panfrost_job_push()  argument
277 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()
281 	ret = drm_gem_lock_reservations(job->bos, job->bo_count,  in panfrost_job_push()
287 	drm_sched_job_arm(&job->base);  in panfrost_job_push()
289 	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);  in panfrost_job_push()
291 	ret = panfrost_acquire_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
292 					     &job->base);  in panfrost_job_push()
298 	kref_get(&job->refcount); /* put by scheduler job completion */  in panfrost_job_push()
300 	drm_sched_entity_push_job(&job->base);  in panfrost_job_push()
304 	panfrost_attach_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
305 				      job->render_done_fence);  in panfrost_job_push()
308 	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);  in panfrost_job_push()
315 	struct panfrost_job *job = container_of(ref, struct panfrost_job,  in panfrost_job_cleanup()  local
319 	dma_fence_put(job->done_fence);  in panfrost_job_cleanup()
320 	dma_fence_put(job->render_done_fence);  in panfrost_job_cleanup()
322 	if (job->mappings) {  in panfrost_job_cleanup()
323 		for (i = 0; i < job->bo_count; i++) {  in panfrost_job_cleanup()
324 			if (!job->mappings[i])  in panfrost_job_cleanup()
327 			atomic_dec(&job->mappings[i]->obj->gpu_usecount);  in panfrost_job_cleanup()
328 			panfrost_gem_mapping_put(job->mappings[i]);  in panfrost_job_cleanup()
330 		kvfree(job->mappings);  in panfrost_job_cleanup()
333 	if (job->bos) {  in panfrost_job_cleanup()
334 		for (i = 0; i < job->bo_count; i++)  in panfrost_job_cleanup()
335 			drm_gem_object_put(job->bos[i]);  in panfrost_job_cleanup()
337 		kvfree(job->bos);  in panfrost_job_cleanup()
340 	kfree(job);  in panfrost_job_cleanup()
343 void panfrost_job_put(struct panfrost_job *job)  in panfrost_job_put()  argument
345 	kref_put(&job->refcount, panfrost_job_cleanup);  in panfrost_job_put()
350 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_free()  local
354 	panfrost_job_put(job);  in panfrost_job_free()
359 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_run()  local
360 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()
361 	int slot = panfrost_job_get_slot(job);  in panfrost_job_run()
364 	if (unlikely(job->base.s_fence->finished.error))  in panfrost_job_run()
367 	/* Nothing to execute: can happen if the job has finished while  in panfrost_job_run()
370 	if (!job->jc)  in panfrost_job_run()
377 	if (job->done_fence)  in panfrost_job_run()
378 		dma_fence_put(job->done_fence);  in panfrost_job_run()
379 	job->done_fence = dma_fence_get(fence);  in panfrost_job_run()
381 	panfrost_job_hw_submit(job, slot);  in panfrost_job_run()
400 				    struct panfrost_job *job,  in panfrost_job_handle_err()  argument
420 		/* Update the job head so we can resume */  in panfrost_job_handle_err()
421 		job->jc = job_read(pfdev, JS_TAIL_LO(js)) |  in panfrost_job_handle_err()
424 		/* The job will be resumed, don't signal the fence */  in panfrost_job_handle_err()
427 		/* Job has been hard-stopped, flag it as canceled */  in panfrost_job_handle_err()
428 		dma_fence_set_error(job->done_fence, -ECANCELED);  in panfrost_job_handle_err()
429 		job->jc = 0;  in panfrost_job_handle_err()
435 		dma_fence_set_error(job->done_fence, -EINVAL);  in panfrost_job_handle_err()
436 		job->jc = 0;  in panfrost_job_handle_err()
439 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_err()
443 		dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_err()
454 				     struct panfrost_job *job)  in panfrost_job_handle_done()  argument
456 	/* Set ->jc to 0 to avoid re-submitting an already finished job (can  in panfrost_job_handle_done()
459 	job->jc = 0;  in panfrost_job_handle_done()
460 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_done()
463 	dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_done()
503 		 * is racy. If we only have one job done at the time we  in panfrost_job_handle_irq()
504 		 * read JOB_INT_RAWSTAT but the second job fails before we  in panfrost_job_handle_irq()
526 			/* When the current job doesn't fail, the JM dequeues  in panfrost_job_handle_irq()
527 			 * the next job without waiting for an ACK, this means  in panfrost_job_handle_irq()
530 			 * are inactive, but one job remains in pfdev->jobs[j],  in panfrost_job_handle_irq()
533 			 * job in _NEXT (see above).  in panfrost_job_handle_irq()
556 			/* The job was cancelled, signal the fence now */  in panfrost_job_handle_irq()
562 			/* Requeue the job we removed if no reset is pending */  in panfrost_job_handle_irq()
619 	 * to release job resources. We should rework the code to follow this  in panfrost_reset()
636 	/* Mask job interrupts and synchronize to make sure we won't be  in panfrost_reset()
643 		/* Cancel the next job and soft-stop the running job. */  in panfrost_reset()
662 	 * panfrost_devfreq_record_idle() for each stuck job.  in panfrost_reset()
677 	/* panfrost_device_reset() unmasks job interrupts, but we want to  in panfrost_reset()
688 	 * while resubmitting jobs because the job submission logic will  in panfrost_reset()
701 	/* Re-enable job interrupts now that everything has been restarted. */  in panfrost_reset()
712 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_timedout()  local
713 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()
714 	int js = panfrost_job_get_slot(job);  in panfrost_job_timedout()
720 	if (dma_fence_is_signaled(job->done_fence))  in panfrost_job_timedout()
731 	panfrost_core_dump(job);  in panfrost_job_timedout()
783 	 * disambiguation stopping the right job in the close path is tricky,  in panfrost_job_init()
796 	js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");  in panfrost_job_init()
803 					IRQF_SHARED, KBUILD_MODNAME "-job",  in panfrost_job_init()
806 		dev_err(pfdev->dev, "failed to request job irq");  in panfrost_job_init()
889 			struct panfrost_job *job = pfdev->jobs[i][j];  in panfrost_job_close()  local
892 			if (!job || job->base.entity != entity)  in panfrost_job_close()
896 				/* Try to cancel the job before it starts */  in panfrost_job_close()
898 				/* Reset the job head so it doesn't get restarted if  in panfrost_job_close()
899 				 * the job in the first slot failed.  in panfrost_job_close()
901 				job->jc = 0;  in panfrost_job_close()
905 				cmd = panfrost_get_job_chain_flag(job) ?  in panfrost_job_close()