39 	struct etnaviv_gem_submit *submit;  in submit_create()  local
40 	size_t sz = size_vstruct(nr_bos, sizeof(submit->bos[0]), sizeof(*submit));  in submit_create()
42 	submit = kzalloc(sz, GFP_KERNEL);  in submit_create()
43 	if (!submit)  in submit_create()
46 	submit->pmrs = kcalloc(nr_pmrs, sizeof(struct etnaviv_perfmon_request),  in submit_create()
48 	if (!submit->pmrs) {  in submit_create()
49 		kfree(submit);  in submit_create()
52 	submit->nr_pmrs = nr_pmrs;  in submit_create()
54 	submit->gpu = gpu;  in submit_create()
55 	kref_init(&submit->refcount);  in submit_create()
57 	return submit;  in submit_create()
60 static int submit_lookup_objects(struct etnaviv_gem_submit *submit,  in submit_lookup_objects()  argument
79 		submit->bos[i].flags = bo->flags;  in submit_lookup_objects()
98 		submit->bos[i].obj = to_etnaviv_bo(obj);  in submit_lookup_objects()
102 	submit->nr_bos = i;  in submit_lookup_objects()
108 static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i)  in submit_unlock_object()  argument
110 	if (submit->bos[i].flags & BO_LOCKED) {  in submit_unlock_object()
111 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;  in submit_unlock_object()
114 		submit->bos[i].flags &= ~BO_LOCKED;  in submit_unlock_object()
118 static int submit_lock_objects(struct etnaviv_gem_submit *submit,  in submit_lock_objects()  argument
124 	for (i = 0; i < submit->nr_bos; i++) {  in submit_lock_objects()
125 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;  in submit_lock_objects()
132 		if (!(submit->bos[i].flags & BO_LOCKED)) {  in submit_lock_objects()
140 			submit->bos[i].flags |= BO_LOCKED;  in submit_lock_objects()
150 		submit_unlock_object(submit, i);  in submit_lock_objects()
153 		submit_unlock_object(submit, slow_locked);  in submit_lock_objects()
158 		etnaviv_obj = submit->bos[contended].obj;  in submit_lock_objects()
164 			submit->bos[contended].flags |= BO_LOCKED;  in submit_lock_objects()
173 static int submit_fence_sync(struct etnaviv_gem_submit *submit)  in submit_fence_sync()  argument
177 	for (i = 0; i < submit->nr_bos; i++) {  in submit_fence_sync()
178 		struct etnaviv_gem_submit_bo *bo = &submit->bos[i];  in submit_fence_sync()
187 		if (submit->flags & ETNA_SUBMIT_NO_IMPLICIT)  in submit_fence_sync()
205 static void submit_attach_object_fences(struct etnaviv_gem_submit *submit)  in submit_attach_object_fences()  argument
209 	for (i = 0; i < submit->nr_bos; i++) {  in submit_attach_object_fences()
210 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;  in submit_attach_object_fences()
212 		if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE)  in submit_attach_object_fences()
214 							  submit->out_fence);  in submit_attach_object_fences()
217 							    submit->out_fence);  in submit_attach_object_fences()
219 		submit_unlock_object(submit, i);  in submit_attach_object_fences()
223 static int submit_pin_objects(struct etnaviv_gem_submit *submit)  in submit_pin_objects()  argument
227 	for (i = 0; i < submit->nr_bos; i++) {  in submit_pin_objects()
228 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;  in submit_pin_objects()
232 						  submit->gpu);  in submit_pin_objects()
239 		submit->bos[i].flags |= BO_PINNED;  in submit_pin_objects()
240 		submit->bos[i].mapping = mapping;  in submit_pin_objects()
246 static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx,  in submit_bo()  argument
249 	if (idx >= submit->nr_bos) {  in submit_bo()
251 				idx, submit->nr_bos);  in submit_bo()
255 	*bo = &submit->bos[idx];  in submit_bo()
261 static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream,  in submit_reloc()  argument
294 		ret = submit_bo(submit, r->reloc_idx, &bo);  in submit_reloc()
311 static int submit_perfmon_validate(struct etnaviv_gem_submit *submit,  in submit_perfmon_validate()  argument
316 	for (i = 0; i < submit->nr_pmrs; i++) {  in submit_perfmon_validate()
321 		ret = submit_bo(submit, r->read_idx, &bo);  in submit_perfmon_validate()
346 		submit->pmrs[i].flags = r->flags;  in submit_perfmon_validate()
347 		submit->pmrs[i].domain = r->domain;  in submit_perfmon_validate()
348 		submit->pmrs[i].signal = r->signal;  in submit_perfmon_validate()
349 		submit->pmrs[i].sequence = r->sequence;  in submit_perfmon_validate()
350 		submit->pmrs[i].offset = r->read_offset;  in submit_perfmon_validate()
351 		submit->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base);  in submit_perfmon_validate()
359 	struct etnaviv_gem_submit *submit =  in submit_cleanup()  local
363 	if (submit->runtime_resumed)  in submit_cleanup()
364 		pm_runtime_put_autosuspend(submit->gpu->dev);  in submit_cleanup()
366 	if (submit->cmdbuf.suballoc)  in submit_cleanup()
367 		etnaviv_cmdbuf_free(&submit->cmdbuf);  in submit_cleanup()
369 	for (i = 0; i < submit->nr_bos; i++) {  in submit_cleanup()
370 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;  in submit_cleanup()
373 		if (submit->bos[i].flags & BO_PINNED) {  in submit_cleanup()
374 			etnaviv_gem_mapping_unreference(submit->bos[i].mapping);  in submit_cleanup()
376 			submit->bos[i].mapping = NULL;  in submit_cleanup()
377 			submit->bos[i].flags &= ~BO_PINNED;  in submit_cleanup()
381 		submit_unlock_object(submit, i);  in submit_cleanup()
385 	wake_up_all(&submit->gpu->fence_event);  in submit_cleanup()
387 	if (submit->in_fence)  in submit_cleanup()
388 		dma_fence_put(submit->in_fence);  in submit_cleanup()
389 	if (submit->out_fence) {  in submit_cleanup()
391 		mutex_lock(&submit->gpu->fence_lock);  in submit_cleanup()
392 		idr_remove(&submit->gpu->fence_idr, submit->out_fence_id);  in submit_cleanup()
393 		mutex_unlock(&submit->gpu->fence_lock);  in submit_cleanup()
394 		dma_fence_put(submit->out_fence);  in submit_cleanup()
396 	kfree(submit->pmrs);  in submit_cleanup()
397 	kfree(submit);  in submit_cleanup()
400 void etnaviv_submit_put(struct etnaviv_gem_submit *submit)  in etnaviv_submit_put()  argument
402 	kref_put(&submit->refcount, submit_cleanup);  in etnaviv_submit_put()
414 	struct etnaviv_gem_submit *submit;  in etnaviv_ioctl_gem_submit()  local
498 	submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs);  in etnaviv_ioctl_gem_submit()
499 	if (!submit) {  in etnaviv_ioctl_gem_submit()
504 	ret = etnaviv_cmdbuf_init(gpu->cmdbuf_suballoc, &submit->cmdbuf,  in etnaviv_ioctl_gem_submit()
509 	submit->cmdbuf.ctx = file->driver_priv;  in etnaviv_ioctl_gem_submit()
510 	submit->exec_state = args->exec_state;  in etnaviv_ioctl_gem_submit()
511 	submit->flags = args->flags;  in etnaviv_ioctl_gem_submit()
513 	ret = submit_lookup_objects(submit, file, bos, args->nr_bos);  in etnaviv_ioctl_gem_submit()
524 		submit->in_fence = sync_file_get_fence(args->fence_fd);  in etnaviv_ioctl_gem_submit()
525 		if (!submit->in_fence) {  in etnaviv_ioctl_gem_submit()
531 	ret = submit_pin_objects(submit);  in etnaviv_ioctl_gem_submit()
535 	ret = submit_reloc(submit, stream, args->stream_size / 4,  in etnaviv_ioctl_gem_submit()
540 	ret = submit_perfmon_validate(submit, args->exec_state, pmrs);  in etnaviv_ioctl_gem_submit()
544 	memcpy(submit->cmdbuf.vaddr, stream, args->stream_size);  in etnaviv_ioctl_gem_submit()
546 	ret = submit_lock_objects(submit, &ticket);  in etnaviv_ioctl_gem_submit()
550 	ret = submit_fence_sync(submit);  in etnaviv_ioctl_gem_submit()
554 	ret = etnaviv_sched_push_job(&ctx->sched_entity[args->pipe], submit);  in etnaviv_ioctl_gem_submit()
558 	submit_attach_object_fences(submit);  in etnaviv_ioctl_gem_submit()
567 		sync_file = sync_file_create(submit->out_fence);  in etnaviv_ioctl_gem_submit()
576 	args->fence = submit->out_fence_id;  in etnaviv_ioctl_gem_submit()
579 	etnaviv_submit_put(submit);  in etnaviv_ioctl_gem_submit()