15 static void release_registered_pages(struct tee_shm *shm)  in release_registered_pages()  argument
17 	if (shm->pages) {  in release_registered_pages()
18 		if (shm->flags & TEE_SHM_USER_MAPPED) {  in release_registered_pages()
19 			unpin_user_pages(shm->pages, shm->num_pages);  in release_registered_pages()
23 			for (n = 0; n < shm->num_pages; n++)  in release_registered_pages()
24 				put_page(shm->pages[n]);  in release_registered_pages()
27 		kfree(shm->pages);  in release_registered_pages()
31 static void tee_shm_release(struct tee_shm *shm)  in tee_shm_release()  argument
33 	struct tee_device *teedev = shm->ctx->teedev;  in tee_shm_release()
35 	if (shm->flags & TEE_SHM_DMA_BUF) {  in tee_shm_release()
37 		idr_remove(&teedev->idr, shm->id);  in tee_shm_release()
41 	if (shm->flags & TEE_SHM_POOL) {  in tee_shm_release()
44 		if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_release()
49 		poolm->ops->free(poolm, shm);  in tee_shm_release()
50 	} else if (shm->flags & TEE_SHM_REGISTER) {  in tee_shm_release()
51 		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);  in tee_shm_release()
55 				"unregister shm %p failed: %d", shm, rc);  in tee_shm_release()
57 		release_registered_pages(shm);  in tee_shm_release()
60 	teedev_ctx_put(shm->ctx);  in tee_shm_release()
62 	kfree(shm);  in tee_shm_release()
81 	struct tee_shm *shm = dmabuf->priv;  in tee_shm_op_release()  local
83 	tee_shm_release(shm);  in tee_shm_op_release()
88 	struct tee_shm *shm = dmabuf->priv;  in tee_shm_op_mmap()  local
92 	if (shm->flags & TEE_SHM_USER_MAPPED)  in tee_shm_op_mmap()
95 	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,  in tee_shm_op_mmap()
110 	struct tee_shm *shm;  in tee_shm_alloc()  local
121 		dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);  in tee_shm_alloc()
134 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in tee_shm_alloc()
135 	if (!shm) {  in tee_shm_alloc()
140 	shm->flags = flags | TEE_SHM_POOL;  in tee_shm_alloc()
141 	shm->ctx = ctx;  in tee_shm_alloc()
147 	rc = poolm->ops->alloc(poolm, shm, size);  in tee_shm_alloc()
158 		shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in tee_shm_alloc()
160 		if (shm->id < 0) {  in tee_shm_alloc()
161 			ret = ERR_PTR(shm->id);  in tee_shm_alloc()
166 		exp_info.size = shm->size;  in tee_shm_alloc()
168 		exp_info.priv = shm;  in tee_shm_alloc()
170 		shm->dmabuf = dma_buf_export(&exp_info);  in tee_shm_alloc()
171 		if (IS_ERR(shm->dmabuf)) {  in tee_shm_alloc()
172 			ret = ERR_CAST(shm->dmabuf);  in tee_shm_alloc()
179 	return shm;  in tee_shm_alloc()
183 		idr_remove(&teedev->idr, shm->id);  in tee_shm_alloc()
187 	poolm->ops->free(poolm, shm);  in tee_shm_alloc()
189 	kfree(shm);  in tee_shm_alloc()
220 	struct tee_shm *shm;  in tee_shm_register()  local
240 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in tee_shm_register()
241 	if (!shm) {  in tee_shm_register()
246 	shm->flags = flags | TEE_SHM_REGISTER;  in tee_shm_register()
247 	shm->ctx = ctx;  in tee_shm_register()
248 	shm->id = -1;  in tee_shm_register()
251 	shm->offset = addr - start;  in tee_shm_register()
252 	shm->size = length;  in tee_shm_register()
254 	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);  in tee_shm_register()
255 	if (!shm->pages) {  in tee_shm_register()
262 					 shm->pages);  in tee_shm_register()
278 		rc = get_kernel_pages(kiov, num_pages, 0, shm->pages);  in tee_shm_register()
282 		shm->num_pages = rc;  in tee_shm_register()
291 	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in tee_shm_register()
294 	if (shm->id < 0) {  in tee_shm_register()
295 		ret = ERR_PTR(shm->id);  in tee_shm_register()
299 	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,  in tee_shm_register()
300 					     shm->num_pages, start);  in tee_shm_register()
310 		exp_info.size = shm->size;  in tee_shm_register()
312 		exp_info.priv = shm;  in tee_shm_register()
314 		shm->dmabuf = dma_buf_export(&exp_info);  in tee_shm_register()
315 		if (IS_ERR(shm->dmabuf)) {  in tee_shm_register()
316 			ret = ERR_CAST(shm->dmabuf);  in tee_shm_register()
317 			teedev->desc->ops->shm_unregister(ctx, shm);  in tee_shm_register()
322 	return shm;  in tee_shm_register()
324 	if (shm) {  in tee_shm_register()
325 		if (shm->id >= 0) {  in tee_shm_register()
327 			idr_remove(&teedev->idr, shm->id);  in tee_shm_register()
330 		release_registered_pages(shm);  in tee_shm_register()
332 	kfree(shm);  in tee_shm_register()
341  * @shm:	Shared memory handle
344 int tee_shm_get_fd(struct tee_shm *shm)  in tee_shm_get_fd()  argument
348 	if (!(shm->flags & TEE_SHM_DMA_BUF))  in tee_shm_get_fd()
351 	get_dma_buf(shm->dmabuf);  in tee_shm_get_fd()
352 	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);  in tee_shm_get_fd()
354 		dma_buf_put(shm->dmabuf);  in tee_shm_get_fd()
360  * @shm:	Handle to shared memory to free
362 void tee_shm_free(struct tee_shm *shm)  in tee_shm_free()  argument
371 	if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_free()
372 		dma_buf_put(shm->dmabuf);  in tee_shm_free()
374 		tee_shm_release(shm);  in tee_shm_free()
380  * @shm:	Shared memory handle
385 int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)  in tee_shm_va2pa()  argument
387 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_va2pa()
389 	/* Check that we're in the range of the shm */  in tee_shm_va2pa()
390 	if ((char *)va < (char *)shm->kaddr)  in tee_shm_va2pa()
392 	if ((char *)va >= ((char *)shm->kaddr + shm->size))  in tee_shm_va2pa()
396 			shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);  in tee_shm_va2pa()
402  * @shm:	Shared memory handle
407 int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)  in tee_shm_pa2va()  argument
409 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_pa2va()
411 	/* Check that we're in the range of the shm */  in tee_shm_pa2va()
412 	if (pa < shm->paddr)  in tee_shm_pa2va()
414 	if (pa >= (shm->paddr + shm->size))  in tee_shm_pa2va()
418 		void *v = tee_shm_get_va(shm, pa - shm->paddr);  in tee_shm_pa2va()
430  * @shm:	Shared memory handle
435 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)  in tee_shm_get_va()  argument
437 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_get_va()
439 	if (offs >= shm->size)  in tee_shm_get_va()
441 	return (char *)shm->kaddr + offs;  in tee_shm_get_va()
447  * @shm:	Shared memory handle
453 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)  in tee_shm_get_pa()  argument
455 	if (offs >= shm->size)  in tee_shm_get_pa()
458 		*pa = shm->paddr + offs;  in tee_shm_get_pa()
473 	struct tee_shm *shm;  in tee_shm_get_from_id()  local
480 	shm = idr_find(&teedev->idr, id);  in tee_shm_get_from_id()
481 	if (!shm || shm->ctx != ctx)  in tee_shm_get_from_id()
482 		shm = ERR_PTR(-EINVAL);  in tee_shm_get_from_id()
483 	else if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_get_from_id()
484 		get_dma_buf(shm->dmabuf);  in tee_shm_get_from_id()
486 	return shm;  in tee_shm_get_from_id()
492  * @shm:	Shared memory handle
494 void tee_shm_put(struct tee_shm *shm)  in tee_shm_put()  argument
496 	if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_put()
497 		dma_buf_put(shm->dmabuf);  in tee_shm_put()