14 static void tee_shm_release(struct tee_shm *shm)  in tee_shm_release()  argument
16 	struct tee_device *teedev = shm->teedev;  in tee_shm_release()
19 	idr_remove(&teedev->idr, shm->id);  in tee_shm_release()
20 	if (shm->ctx)  in tee_shm_release()
21 		list_del(&shm->link);  in tee_shm_release()
24 	if (shm->flags & TEE_SHM_POOL) {  in tee_shm_release()
27 		if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_release()
32 		poolm->ops->free(poolm, shm);  in tee_shm_release()
33 	} else if (shm->flags & TEE_SHM_REGISTER) {  in tee_shm_release()
35 		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);  in tee_shm_release()
39 				"unregister shm %p failed: %d", shm, rc);  in tee_shm_release()
41 		for (n = 0; n < shm->num_pages; n++)  in tee_shm_release()
42 			put_page(shm->pages[n]);  in tee_shm_release()
44 		kfree(shm->pages);  in tee_shm_release()
47 	if (shm->ctx)  in tee_shm_release()
48 		teedev_ctx_put(shm->ctx);  in tee_shm_release()
50 	kfree(shm);  in tee_shm_release()
69 	struct tee_shm *shm = dmabuf->priv;  in tee_shm_op_release()  local
71 	tee_shm_release(shm);  in tee_shm_op_release()
81 	struct tee_shm *shm = dmabuf->priv;  in tee_shm_op_mmap()  local
85 	if (shm->flags & TEE_SHM_REGISTER)  in tee_shm_op_mmap()
88 	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,  in tee_shm_op_mmap()
105 	struct tee_shm *shm;  in __tee_shm_alloc()  local
134 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in __tee_shm_alloc()
135 	if (!shm) {  in __tee_shm_alloc()
140 	shm->flags = flags | TEE_SHM_POOL;  in __tee_shm_alloc()
141 	shm->teedev = teedev;  in __tee_shm_alloc()
142 	shm->ctx = ctx;  in __tee_shm_alloc()
148 	rc = poolm->ops->alloc(poolm, shm, size);  in __tee_shm_alloc()
155 	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in __tee_shm_alloc()
157 	if (shm->id < 0) {  in __tee_shm_alloc()
158 		ret = ERR_PTR(shm->id);  in __tee_shm_alloc()
166 		exp_info.size = shm->size;  in __tee_shm_alloc()
168 		exp_info.priv = shm;  in __tee_shm_alloc()
170 		shm->dmabuf = dma_buf_export(&exp_info);  in __tee_shm_alloc()
171 		if (IS_ERR(shm->dmabuf)) {  in __tee_shm_alloc()
172 			ret = ERR_CAST(shm->dmabuf);  in __tee_shm_alloc()
180 		list_add_tail(&shm->link, &ctx->list_shm);  in __tee_shm_alloc()
184 	return shm;  in __tee_shm_alloc()
187 	idr_remove(&teedev->idr, shm->id);  in __tee_shm_alloc()
190 	poolm->ops->free(poolm, shm);  in __tee_shm_alloc()
192 	kfree(shm);  in __tee_shm_alloc()
227 	struct tee_shm *shm;  in tee_shm_register()  local
247 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in tee_shm_register()
248 	if (!shm) {  in tee_shm_register()
253 	shm->flags = flags | TEE_SHM_REGISTER;  in tee_shm_register()
254 	shm->teedev = teedev;  in tee_shm_register()
255 	shm->ctx = ctx;  in tee_shm_register()
256 	shm->id = -1;  in tee_shm_register()
259 	shm->offset = addr - start;  in tee_shm_register()
260 	shm->size = length;  in tee_shm_register()
262 	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);  in tee_shm_register()
263 	if (!shm->pages) {  in tee_shm_register()
268 	rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);  in tee_shm_register()
270 		shm->num_pages = rc;  in tee_shm_register()
279 	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in tee_shm_register()
282 	if (shm->id < 0) {  in tee_shm_register()
283 		ret = ERR_PTR(shm->id);  in tee_shm_register()
287 	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,  in tee_shm_register()
288 					     shm->num_pages, start);  in tee_shm_register()
298 		exp_info.size = shm->size;  in tee_shm_register()
300 		exp_info.priv = shm;  in tee_shm_register()
302 		shm->dmabuf = dma_buf_export(&exp_info);  in tee_shm_register()
303 		if (IS_ERR(shm->dmabuf)) {  in tee_shm_register()
304 			ret = ERR_CAST(shm->dmabuf);  in tee_shm_register()
305 			teedev->desc->ops->shm_unregister(ctx, shm);  in tee_shm_register()
311 	list_add_tail(&shm->link, &ctx->list_shm);  in tee_shm_register()
314 	return shm;  in tee_shm_register()
316 	if (shm) {  in tee_shm_register()
319 		if (shm->id >= 0) {  in tee_shm_register()
321 			idr_remove(&teedev->idr, shm->id);  in tee_shm_register()
324 		if (shm->pages) {  in tee_shm_register()
325 			for (n = 0; n < shm->num_pages; n++)  in tee_shm_register()
326 				put_page(shm->pages[n]);  in tee_shm_register()
327 			kfree(shm->pages);  in tee_shm_register()
330 	kfree(shm);  in tee_shm_register()
342 int tee_shm_get_fd(struct tee_shm *shm)  in tee_shm_get_fd()  argument
346 	if (!(shm->flags & TEE_SHM_DMA_BUF))  in tee_shm_get_fd()
349 	get_dma_buf(shm->dmabuf);  in tee_shm_get_fd()
350 	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);  in tee_shm_get_fd()
352 		dma_buf_put(shm->dmabuf);  in tee_shm_get_fd()
360 void tee_shm_free(struct tee_shm *shm)  in tee_shm_free()  argument
369 	if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_free()
370 		dma_buf_put(shm->dmabuf);  in tee_shm_free()
372 		tee_shm_release(shm);  in tee_shm_free()
383 int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)  in tee_shm_va2pa()  argument
385 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_va2pa()
388 	if ((char *)va < (char *)shm->kaddr)  in tee_shm_va2pa()
390 	if ((char *)va >= ((char *)shm->kaddr + shm->size))  in tee_shm_va2pa()
394 			shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);  in tee_shm_va2pa()
405 int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)  in tee_shm_pa2va()  argument
407 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_pa2va()
410 	if (pa < shm->paddr)  in tee_shm_pa2va()
412 	if (pa >= (shm->paddr + shm->size))  in tee_shm_pa2va()
416 		void *v = tee_shm_get_va(shm, pa - shm->paddr);  in tee_shm_pa2va()
433 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)  in tee_shm_get_va()  argument
435 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_get_va()
437 	if (offs >= shm->size)  in tee_shm_get_va()
439 	return (char *)shm->kaddr + offs;  in tee_shm_get_va()
451 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)  in tee_shm_get_pa()  argument
453 	if (offs >= shm->size)  in tee_shm_get_pa()
456 		*pa = shm->paddr + offs;  in tee_shm_get_pa()
471 	struct tee_shm *shm;  in tee_shm_get_from_id()  local
478 	shm = idr_find(&teedev->idr, id);  in tee_shm_get_from_id()
479 	if (!shm || shm->ctx != ctx)  in tee_shm_get_from_id()
480 		shm = ERR_PTR(-EINVAL);  in tee_shm_get_from_id()
481 	else if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_get_from_id()
482 		get_dma_buf(shm->dmabuf);  in tee_shm_get_from_id()
484 	return shm;  in tee_shm_get_from_id()
492 void tee_shm_put(struct tee_shm *shm)  in tee_shm_put()  argument
494 	if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_put()
495 		dma_buf_put(shm->dmabuf);  in tee_shm_put()