61 static void release_registered_pages(struct tee_shm *shm)  in release_registered_pages()  argument
63 	if (shm->pages) {  in release_registered_pages()
64 		if (shm->flags & TEE_SHM_USER_MAPPED)  in release_registered_pages()
65 			unpin_user_pages(shm->pages, shm->num_pages);  in release_registered_pages()
67 			shm_put_kernel_pages(shm->pages, shm->num_pages);  in release_registered_pages()
69 		kfree(shm->pages);  in release_registered_pages()
73 static void tee_shm_release(struct tee_device *teedev, struct tee_shm *shm)  in tee_shm_release()  argument
75 	if (shm->flags & TEE_SHM_POOL) {  in tee_shm_release()
76 		teedev->pool->ops->free(teedev->pool, shm);  in tee_shm_release()
77 	} else if (shm->flags & TEE_SHM_DYNAMIC) {  in tee_shm_release()
78 		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);  in tee_shm_release()
82 				"unregister shm %p failed: %d", shm, rc);  in tee_shm_release()
84 		release_registered_pages(shm);  in tee_shm_release()
87 	teedev_ctx_put(shm->ctx);  in tee_shm_release()
89 	kfree(shm);  in tee_shm_release()
98 	struct tee_shm *shm;  in shm_alloc_helper()  local
111 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in shm_alloc_helper()
112 	if (!shm) {  in shm_alloc_helper()
117 	refcount_set(&shm->refcount, 1);  in shm_alloc_helper()
118 	shm->flags = flags;  in shm_alloc_helper()
119 	shm->id = id;  in shm_alloc_helper()
127 	shm->ctx = ctx;  in shm_alloc_helper()
129 	rc = teedev->pool->ops->alloc(teedev->pool, shm, size, align);  in shm_alloc_helper()
136 	return shm;  in shm_alloc_helper()
138 	kfree(shm);  in shm_alloc_helper()
160 	struct tee_shm *shm;  in tee_shm_alloc_user_buf()  local
170 	shm = shm_alloc_helper(ctx, size, PAGE_SIZE, flags, id);  in tee_shm_alloc_user_buf()
171 	if (IS_ERR(shm)) {  in tee_shm_alloc_user_buf()
175 		return shm;  in tee_shm_alloc_user_buf()
179 	ret = idr_replace(&teedev->idr, shm, id);  in tee_shm_alloc_user_buf()
182 		tee_shm_free(shm);  in tee_shm_alloc_user_buf()
186 	return shm;  in tee_shm_alloc_user_buf()
238 	struct tee_shm *shm;  in register_shm_helper()  local
255 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in register_shm_helper()
256 	if (!shm) {  in register_shm_helper()
261 	refcount_set(&shm->refcount, 1);  in register_shm_helper()
262 	shm->flags = flags;  in register_shm_helper()
263 	shm->ctx = ctx;  in register_shm_helper()
264 	shm->id = id;  in register_shm_helper()
267 	shm->offset = addr - start;  in register_shm_helper()
268 	shm->size = length;  in register_shm_helper()
270 	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);  in register_shm_helper()
271 	if (!shm->pages) {  in register_shm_helper()
278 					 shm->pages);  in register_shm_helper()
280 		rc = shm_get_kernel_pages(start, num_pages, shm->pages);  in register_shm_helper()
282 		shm->num_pages = rc;  in register_shm_helper()
290 	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,  in register_shm_helper()
291 					     shm->num_pages, start);  in register_shm_helper()
297 	return shm;  in register_shm_helper()
300 		unpin_user_pages(shm->pages, shm->num_pages);  in register_shm_helper()
302 		shm_put_kernel_pages(shm->pages, shm->num_pages);  in register_shm_helper()
303 	kfree(shm->pages);  in register_shm_helper()
305 	kfree(shm);  in register_shm_helper()
326 	struct tee_shm *shm;  in tee_shm_register_user_buf()  local
339 	shm = register_shm_helper(ctx, addr, length, flags, id);  in tee_shm_register_user_buf()
340 	if (IS_ERR(shm)) {  in tee_shm_register_user_buf()
344 		return shm;  in tee_shm_register_user_buf()
348 	ret = idr_replace(&teedev->idr, shm, id);  in tee_shm_register_user_buf()
351 		tee_shm_free(shm);  in tee_shm_register_user_buf()
355 	return shm;  in tee_shm_register_user_buf()
385 	struct tee_shm *shm = filp->private_data;  in tee_shm_fop_mmap()  local
389 	if (shm->flags & TEE_SHM_USER_MAPPED)  in tee_shm_fop_mmap()
393 	if (vma->vm_pgoff + vma_pages(vma) > shm->size >> PAGE_SHIFT)  in tee_shm_fop_mmap()
396 	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,  in tee_shm_fop_mmap()
411 int tee_shm_get_fd(struct tee_shm *shm)  in tee_shm_get_fd()  argument
415 	if (shm->id < 0)  in tee_shm_get_fd()
419 	refcount_inc(&shm->refcount);  in tee_shm_get_fd()
420 	fd = anon_inode_getfd("tee_shm", &tee_shm_fops, shm, O_RDWR);  in tee_shm_get_fd()
422 		tee_shm_put(shm);  in tee_shm_get_fd()
430 void tee_shm_free(struct tee_shm *shm)  in tee_shm_free()  argument
432 	tee_shm_put(shm);  in tee_shm_free()
443 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)  in tee_shm_get_va()  argument
445 	if (!shm->kaddr)  in tee_shm_get_va()
447 	if (offs >= shm->size)  in tee_shm_get_va()
449 	return (char *)shm->kaddr + offs;  in tee_shm_get_va()
461 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)  in tee_shm_get_pa()  argument
463 	if (offs >= shm->size)  in tee_shm_get_pa()
466 		*pa = shm->paddr + offs;  in tee_shm_get_pa()
481 	struct tee_shm *shm;  in tee_shm_get_from_id()  local
488 	shm = idr_find(&teedev->idr, id);  in tee_shm_get_from_id()
494 	if (!shm || shm->ctx != ctx)  in tee_shm_get_from_id()
495 		shm = ERR_PTR(-EINVAL);  in tee_shm_get_from_id()
497 		refcount_inc(&shm->refcount);  in tee_shm_get_from_id()
499 	return shm;  in tee_shm_get_from_id()
507 void tee_shm_put(struct tee_shm *shm)  in tee_shm_put()  argument
509 	struct tee_device *teedev = shm->ctx->teedev;  in tee_shm_put()
513 	if (refcount_dec_and_test(&shm->refcount)) {  in tee_shm_put()
520 		if (shm->id >= 0)  in tee_shm_put()
521 			idr_remove(&teedev->idr, shm->id);  in tee_shm_put()
527 		tee_shm_release(teedev, shm);  in tee_shm_put()