34 	struct msm_gem_submit *submit;  in submit_create()  local
35 	uint64_t sz = struct_size(submit, bos, nr_bos) +  in submit_create()
36 				  ((u64)nr_cmds * sizeof(submit->cmd[0]));  in submit_create()
41 	submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);  in submit_create()
42 	if (!submit)  in submit_create()
45 	submit->dev = dev;  in submit_create()
46 	submit->aspace = queue->ctx->aspace;  in submit_create()
47 	submit->gpu = gpu;  in submit_create()
48 	submit->fence = NULL;  in submit_create()
49 	submit->cmd = (void *)&submit->bos[nr_bos];  in submit_create()
50 	submit->queue = queue;  in submit_create()
51 	submit->ring = gpu->rb[queue->prio];  in submit_create()
54 	submit->nr_bos = 0;  in submit_create()
55 	submit->nr_cmds = 0;  in submit_create()
57 	INIT_LIST_HEAD(&submit->node);  in submit_create()
58 	INIT_LIST_HEAD(&submit->bo_list);  in submit_create()
60 	return submit;  in submit_create()
63 void msm_gem_submit_free(struct msm_gem_submit *submit)  in msm_gem_submit_free()  argument
65 	dma_fence_put(submit->fence);  in msm_gem_submit_free()
66 	list_del(&submit->node);  in msm_gem_submit_free()
67 	put_pid(submit->pid);  in msm_gem_submit_free()
68 	msm_submitqueue_put(submit->queue);  in msm_gem_submit_free()
70 	kfree(submit);  in msm_gem_submit_free()
73 static int submit_lookup_objects(struct msm_gem_submit *submit,  in submit_lookup_objects()  argument
87 		submit->bos[i].flags = 0;  in submit_lookup_objects()
106 		submit->bos[i].handle = submit_bo.handle;  in submit_lookup_objects()
107 		submit->bos[i].flags = submit_bo.flags;  in submit_lookup_objects()
109 		submit->bos[i].iova  = submit_bo.presumed;  in submit_lookup_objects()
121 		obj = idr_find(&file->object_idr, submit->bos[i].handle);  in submit_lookup_objects()
123 			DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i);  in submit_lookup_objects()
132 					submit->bos[i].handle, i);  in submit_lookup_objects()
139 		submit->bos[i].obj = msm_obj;  in submit_lookup_objects()
141 		list_add_tail(&msm_obj->submit_entry, &submit->bo_list);  in submit_lookup_objects()
148 	submit->nr_bos = i;  in submit_lookup_objects()
153 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit,  in submit_unlock_unpin_bo()  argument
156 	struct msm_gem_object *msm_obj = submit->bos[i].obj;  in submit_unlock_unpin_bo()
158 	if (submit->bos[i].flags & BO_PINNED)  in submit_unlock_unpin_bo()
159 		msm_gem_unpin_iova(&msm_obj->base, submit->aspace);  in submit_unlock_unpin_bo()
161 	if (submit->bos[i].flags & BO_LOCKED)  in submit_unlock_unpin_bo()
164 	if (backoff && !(submit->bos[i].flags & BO_VALID))  in submit_unlock_unpin_bo()
165 		submit->bos[i].iova = 0;  in submit_unlock_unpin_bo()
167 	submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);  in submit_unlock_unpin_bo()
171 static int submit_lock_objects(struct msm_gem_submit *submit)  in submit_lock_objects()  argument
176 	for (i = 0; i < submit->nr_bos; i++) {  in submit_lock_objects()
177 		struct msm_gem_object *msm_obj = submit->bos[i].obj;  in submit_lock_objects()
184 		if (!(submit->bos[i].flags & BO_LOCKED)) {  in submit_lock_objects()
186 							  &submit->ticket);  in submit_lock_objects()
189 			submit->bos[i].flags |= BO_LOCKED;  in submit_lock_objects()
193 	ww_acquire_done(&submit->ticket);  in submit_lock_objects()
199 		submit_unlock_unpin_bo(submit, i, true);  in submit_lock_objects()
202 		submit_unlock_unpin_bo(submit, slow_locked, true);  in submit_lock_objects()
205 		struct msm_gem_object *msm_obj = submit->bos[contended].obj;  in submit_lock_objects()
208 						       &submit->ticket);  in submit_lock_objects()
210 			submit->bos[contended].flags |= BO_LOCKED;  in submit_lock_objects()
219 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)  in submit_fence_sync()  argument
223 	for (i = 0; i < submit->nr_bos; i++) {  in submit_fence_sync()
224 		struct msm_gem_object *msm_obj = submit->bos[i].obj;  in submit_fence_sync()
225 		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;  in submit_fence_sync()
242 		ret = msm_gem_sync_object(&msm_obj->base, submit->ring->fctx,  in submit_fence_sync()
251 static int submit_pin_objects(struct msm_gem_submit *submit)  in submit_pin_objects()  argument
255 	submit->valid = true;  in submit_pin_objects()
257 	for (i = 0; i < submit->nr_bos; i++) {  in submit_pin_objects()
258 		struct msm_gem_object *msm_obj = submit->bos[i].obj;  in submit_pin_objects()
263 				submit->aspace, &iova);  in submit_pin_objects()
268 		submit->bos[i].flags |= BO_PINNED;  in submit_pin_objects()
270 		if (iova == submit->bos[i].iova) {  in submit_pin_objects()
271 			submit->bos[i].flags |= BO_VALID;  in submit_pin_objects()
273 			submit->bos[i].iova = iova;  in submit_pin_objects()
275 			submit->bos[i].flags &= ~BO_VALID;  in submit_pin_objects()
276 			submit->valid = false;  in submit_pin_objects()
283 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,  in submit_bo()  argument
286 	if (idx >= submit->nr_bos) {  in submit_bo()
288 				idx, submit->nr_bos);  in submit_bo()
293 		*obj = submit->bos[idx].obj;  in submit_bo()
295 		*iova = submit->bos[idx].iova;  in submit_bo()
297 		*valid = !!(submit->bos[idx].flags & BO_VALID);  in submit_bo()
303 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,  in submit_reloc()  argument
359 		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);  in submit_reloc()
384 static void submit_cleanup(struct msm_gem_submit *submit)  in submit_cleanup()  argument
388 	for (i = 0; i < submit->nr_bos; i++) {  in submit_cleanup()
389 		struct msm_gem_object *msm_obj = submit->bos[i].obj;  in submit_cleanup()
390 		submit_unlock_unpin_bo(submit, i, false);  in submit_cleanup()
583 	struct msm_gem_submit *submit;  in msm_ioctl_gem_submit()  local
680 	submit = submit_create(dev, gpu, queue, args->nr_bos,  in msm_ioctl_gem_submit()
682 	if (!submit) {  in msm_ioctl_gem_submit()
687 	submit->pid = pid;  in msm_ioctl_gem_submit()
688 	submit->ident = submitid;  in msm_ioctl_gem_submit()
691 		submit->in_rb = true;  in msm_ioctl_gem_submit()
693 	ret = submit_lookup_objects(submit, args, file);  in msm_ioctl_gem_submit()
698 	ww_acquire_init(&submit->ticket, &reservation_ww_class);  in msm_ioctl_gem_submit()
700 	ret = submit_lock_objects(submit);  in msm_ioctl_gem_submit()
704 	ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));  in msm_ioctl_gem_submit()
708 	ret = submit_pin_objects(submit);  in msm_ioctl_gem_submit()
737 		ret = submit_bo(submit, submit_cmd.submit_idx,  in msm_ioctl_gem_submit()
757 		submit->cmd[i].type = submit_cmd.type;  in msm_ioctl_gem_submit()
758 		submit->cmd[i].size = submit_cmd.size / 4;  in msm_ioctl_gem_submit()
759 		submit->cmd[i].iova = iova + submit_cmd.submit_offset;  in msm_ioctl_gem_submit()
760 		submit->cmd[i].idx  = submit_cmd.submit_idx;  in msm_ioctl_gem_submit()
762 		if (submit->valid)  in msm_ioctl_gem_submit()
765 		ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,  in msm_ioctl_gem_submit()
771 	submit->nr_cmds = i;  in msm_ioctl_gem_submit()
773 	submit->fence = msm_fence_alloc(ring->fctx);  in msm_ioctl_gem_submit()
774 	if (IS_ERR(submit->fence)) {  in msm_ioctl_gem_submit()
775 		ret = PTR_ERR(submit->fence);  in msm_ioctl_gem_submit()
776 		submit->fence = NULL;  in msm_ioctl_gem_submit()
781 		sync_file = sync_file_create(submit->fence);  in msm_ioctl_gem_submit()
788 	msm_gpu_submit(gpu, submit);  in msm_ioctl_gem_submit()
790 	args->fence = submit->fence->seqno;  in msm_ioctl_gem_submit()
799 	                      submit->fence);  in msm_ioctl_gem_submit()
803 	submit_cleanup(submit);  in msm_ioctl_gem_submit()
805 		ww_acquire_fini(&submit->ticket);  in msm_ioctl_gem_submit()
807 		msm_gem_submit_free(submit);  in msm_ioctl_gem_submit()