Lines Matching +full:shared +full:- +full:memory
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2015-2021, Linaro Limited
9 #include <linux/arm-smccc.h>
35 * OP-TEE OS via raw SMCs.
38 * 2. Low level support functions to register shared memory in secure world
39 * 3. Dynamic shared memory pool based on alloc_pages()
46 * A typical OP-TEE private shm allocation is 224 bytes (argument struct
49 * have a handful of these structs allocated at a time. Most memory will
69 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT + in from_msg_param_tmp_mem()
70 attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT; in from_msg_param_tmp_mem()
71 p->u.memref.size = mp->u.tmem.size; in from_msg_param_tmp_mem()
72 shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref; in from_msg_param_tmp_mem()
74 p->u.memref.shm_offs = 0; in from_msg_param_tmp_mem()
75 p->u.memref.shm = NULL; in from_msg_param_tmp_mem()
83 p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa; in from_msg_param_tmp_mem()
84 p->u.memref.shm = shm; in from_msg_param_tmp_mem()
94 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT + in from_msg_param_reg_mem()
95 attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; in from_msg_param_reg_mem()
96 p->u.memref.size = mp->u.rmem.size; in from_msg_param_reg_mem()
97 shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref; in from_msg_param_reg_mem()
100 p->u.memref.shm_offs = mp->u.rmem.offs; in from_msg_param_reg_mem()
101 p->u.memref.shm = shm; in from_msg_param_reg_mem()
103 p->u.memref.shm_offs = 0; in from_msg_param_reg_mem()
104 p->u.memref.shm = NULL; in from_msg_param_reg_mem()
109 * optee_from_msg_param() - convert from OPTEE_MSG parameters to
127 u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK; in optee_from_msg_param()
131 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; in optee_from_msg_param()
132 memset(&p->u, 0, sizeof(p->u)); in optee_from_msg_param()
153 return -EINVAL; in optee_from_msg_param()
165 mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr - in to_msg_param_tmp_mem()
168 mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm; in to_msg_param_tmp_mem()
169 mp->u.tmem.size = p->u.memref.size; in to_msg_param_tmp_mem()
171 if (!p->u.memref.shm) { in to_msg_param_tmp_mem()
172 mp->u.tmem.buf_ptr = 0; in to_msg_param_tmp_mem()
176 rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa); in to_msg_param_tmp_mem()
180 mp->u.tmem.buf_ptr = pa; in to_msg_param_tmp_mem()
181 mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED << in to_msg_param_tmp_mem()
190 mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr - in to_msg_param_reg_mem()
193 mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm; in to_msg_param_reg_mem()
194 mp->u.rmem.size = p->u.memref.size; in to_msg_param_reg_mem()
195 mp->u.rmem.offs = p->u.memref.shm_offs; in to_msg_param_reg_mem()
200 * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
218 switch (p->attr) { in optee_to_msg_param()
220 mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; in optee_to_msg_param()
221 memset(&mp->u, 0, sizeof(mp->u)); in optee_to_msg_param()
231 if (tee_shm_is_dynamic(p->u.memref.shm)) in optee_to_msg_param()
239 return -EINVAL; in optee_to_msg_param()
246 * 2. Low level support functions to register shared memory in secure world
248 * Functions to enable/disable shared memory caching in secure world, that
249 * is, lazy freeing of previously allocated shared memory. Freeing is
252 * Functions to register and unregister shared memory both for normal
253 * clients and for tee-supplicant.
257 * optee_enable_shm_cache() - Enables caching of some shared memory allocation
258 * in OP-TEE
266 optee_cq_wait_init(&optee->call_queue, &w); in optee_enable_shm_cache()
270 optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE, in optee_enable_shm_cache()
274 optee_cq_wait_for_completion(&optee->call_queue, &w); in optee_enable_shm_cache()
276 optee_cq_wait_final(&optee->call_queue, &w); in optee_enable_shm_cache()
280 * __optee_disable_shm_cache() - Disables caching of some shared memory
281 * allocation in OP-TEE
283 * @is_mapped: true if the cached shared memory addresses were mapped by this
291 optee_cq_wait_init(&optee->call_queue, &w); in __optee_disable_shm_cache()
298 optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE, in __optee_disable_shm_cache()
306 * Shared memory references that were not mapped by in __optee_disable_shm_cache()
316 optee_cq_wait_for_completion(&optee->call_queue, &w); in __optee_disable_shm_cache()
319 optee_cq_wait_final(&optee->call_queue, &w); in __optee_disable_shm_cache()
323 * optee_disable_shm_cache() - Disables caching of mapped shared memory
324 * allocations in OP-TEE
333 * optee_disable_unmapped_shm_cache() - Disables caching of shared memory
334 * allocations in OP-TEE which are not
344 ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
368 * optee_fill_pages_list() - write list of user pages to given shared
371 * @dst: page-aligned buffer where list of pages will be stored
372 * @pages: array of pages that represents shared buffer
394 * Currently OP-TEE uses 4k page size and it does not looks in optee_fill_pages_list()
407 * because they bear no value data for OP-TEE. in optee_fill_pages_list()
413 pages_data->pages_list[n++] = optee_page; in optee_fill_pages_list()
416 pages_data->next_page_data = in optee_fill_pages_list()
424 if (!--num_pages) in optee_fill_pages_list()
436 struct optee *optee = tee_get_drvdata(ctx->teedev); in optee_shm_register()
444 return -EINVAL; in optee_shm_register()
452 return -ENOMEM; in optee_shm_register()
455 * We're about to register shared memory we can't register shared in optee_shm_register()
456 * memory for this request or there's a catch-22. in optee_shm_register()
461 sz = optee_msg_arg_size(optee->rpc_param_count); in optee_shm_register()
477 msg_arg->num_params = 1; in optee_shm_register()
478 msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM; in optee_shm_register()
479 msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | in optee_shm_register()
481 msg_arg->params->u.tmem.shm_ref = (unsigned long)shm; in optee_shm_register()
482 msg_arg->params->u.tmem.size = tee_shm_get_size(shm); in optee_shm_register()
484 * In the least bits of msg_arg->params->u.tmem.buf_ptr we in optee_shm_register()
485 * store buffer offset from 4k page, as described in OP-TEE ABI. in optee_shm_register()
487 msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) | in optee_shm_register()
488 (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1)); in optee_shm_register()
490 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) || in optee_shm_register()
491 msg_arg->ret != TEEC_SUCCESS) in optee_shm_register()
492 rc = -EINVAL; in optee_shm_register()
502 struct optee *optee = tee_get_drvdata(ctx->teedev); in optee_shm_unregister()
509 * We're about to unregister shared memory and we may not be able in optee_shm_unregister()
510 * register shared memory for this request in case we're called in optee_shm_unregister()
517 sz = optee_msg_arg_size(optee->rpc_param_count); in optee_shm_unregister()
528 msg_arg->num_params = 1; in optee_shm_unregister()
529 msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM; in optee_shm_unregister()
530 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; in optee_shm_unregister()
531 msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm; in optee_shm_unregister()
533 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) || in optee_shm_unregister()
534 msg_arg->ret != TEEC_SUCCESS) in optee_shm_unregister()
535 rc = -EINVAL; in optee_shm_unregister()
546 * We don't want to register supplicant memory in OP-TEE. in optee_shm_register_supp()
559 * 3. Dynamic shared memory pool based on alloc_pages()
561 * Implements an OP-TEE specific shared memory pool which is used
562 * when dynamic shared memory is supported by secure world.
571 * Shared memory private to the OP-TEE driver doesn't need in pool_op_alloc()
572 * to be registered with OP-TEE. in pool_op_alloc()
574 if (shm->flags & TEE_SHM_PRIV) in pool_op_alloc()
584 if (!(shm->flags & TEE_SHM_PRIV)) in pool_op_free()
602 * optee_shm_pool_alloc_pages() - create page-based allocator pool
604 * This pool is used when OP-TEE supports dymanic SHM. In this case
605 * command buffers and such are allocated from kernel's own memory.
612 return ERR_PTR(-ENOMEM); in optee_shm_pool_alloc_pages()
614 pool->ops = &pool_ops; in optee_shm_pool_alloc_pages()
625 * delivery of non-secure interrupts to for instance allow rescheduling of
634 arg->ret_origin = TEEC_ORIGIN_COMMS; in handle_rpc_func_cmd_shm_free()
636 if (arg->num_params != 1 || in handle_rpc_func_cmd_shm_free()
637 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) { in handle_rpc_func_cmd_shm_free()
638 arg->ret = TEEC_ERROR_BAD_PARAMETERS; in handle_rpc_func_cmd_shm_free()
642 shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b; in handle_rpc_func_cmd_shm_free()
643 switch (arg->params[0].u.value.a) { in handle_rpc_func_cmd_shm_free()
651 arg->ret = TEEC_ERROR_BAD_PARAMETERS; in handle_rpc_func_cmd_shm_free()
653 arg->ret = TEEC_SUCCESS; in handle_rpc_func_cmd_shm_free()
666 arg->ret_origin = TEEC_ORIGIN_COMMS; in handle_rpc_func_cmd_shm_alloc()
668 if (!arg->num_params || in handle_rpc_func_cmd_shm_alloc()
669 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) { in handle_rpc_func_cmd_shm_alloc()
670 arg->ret = TEEC_ERROR_BAD_PARAMETERS; in handle_rpc_func_cmd_shm_alloc()
674 for (n = 1; n < arg->num_params; n++) { in handle_rpc_func_cmd_shm_alloc()
675 if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) { in handle_rpc_func_cmd_shm_alloc()
676 arg->ret = TEEC_ERROR_BAD_PARAMETERS; in handle_rpc_func_cmd_shm_alloc()
681 sz = arg->params[0].u.value.b; in handle_rpc_func_cmd_shm_alloc()
682 switch (arg->params[0].u.value.a) { in handle_rpc_func_cmd_shm_alloc()
687 shm = tee_shm_alloc_priv_buf(optee->ctx, sz); in handle_rpc_func_cmd_shm_alloc()
690 arg->ret = TEEC_ERROR_BAD_PARAMETERS; in handle_rpc_func_cmd_shm_alloc()
695 arg->ret = TEEC_ERROR_OUT_OF_MEMORY; in handle_rpc_func_cmd_shm_alloc()
700 arg->ret = TEEC_ERROR_BAD_PARAMETERS; in handle_rpc_func_cmd_shm_alloc()
713 arg->ret = TEEC_ERROR_OUT_OF_MEMORY; in handle_rpc_func_cmd_shm_alloc()
719 arg->ret = TEEC_ERROR_OUT_OF_MEMORY; in handle_rpc_func_cmd_shm_alloc()
723 call_ctx->pages_list = pages_list; in handle_rpc_func_cmd_shm_alloc()
724 call_ctx->num_entries = page_num; in handle_rpc_func_cmd_shm_alloc()
726 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | in handle_rpc_func_cmd_shm_alloc()
730 * from 4k page, as described in OP-TEE ABI. in handle_rpc_func_cmd_shm_alloc()
732 arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) | in handle_rpc_func_cmd_shm_alloc()
734 (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1)); in handle_rpc_func_cmd_shm_alloc()
735 arg->params[0].u.tmem.size = tee_shm_get_size(shm); in handle_rpc_func_cmd_shm_alloc()
736 arg->params[0].u.tmem.shm_ref = (unsigned long)shm; in handle_rpc_func_cmd_shm_alloc()
741 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT; in handle_rpc_func_cmd_shm_alloc()
742 arg->params[0].u.tmem.buf_ptr = pa; in handle_rpc_func_cmd_shm_alloc()
743 arg->params[0].u.tmem.size = sz; in handle_rpc_func_cmd_shm_alloc()
744 arg->params[0].u.tmem.shm_ref = (unsigned long)shm; in handle_rpc_func_cmd_shm_alloc()
747 arg->ret = TEEC_SUCCESS; in handle_rpc_func_cmd_shm_alloc()
755 if (call_ctx->pages_list) { in free_pages_list()
756 optee_free_pages_list(call_ctx->pages_list, in free_pages_list()
757 call_ctx->num_entries); in free_pages_list()
758 call_ctx->pages_list = NULL; in free_pages_list()
759 call_ctx->num_entries = 0; in free_pages_list()
773 switch (arg->cmd) { in handle_rpc_func_cmd()
787 * optee_handle_rpc() - handle RPC from secure world
790 * @call_ctx: call context. Preserved during one OP-TEE invocation
799 struct tee_device *teedev = ctx->teedev; in optee_handle_rpc()
805 switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) { in optee_handle_rpc()
807 shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1); in optee_handle_rpc()
809 reg_pair_from_64(¶m->a1, ¶m->a2, pa); in optee_handle_rpc()
810 reg_pair_from_64(¶m->a4, ¶m->a5, in optee_handle_rpc()
813 param->a1 = 0; in optee_handle_rpc()
814 param->a2 = 0; in optee_handle_rpc()
815 param->a4 = 0; in optee_handle_rpc()
816 param->a5 = 0; in optee_handle_rpc()
821 shm = reg_pair_to_ptr(param->a1, param->a2); in optee_handle_rpc()
836 shm = reg_pair_to_ptr(param->a1, param->a2); in optee_handle_rpc()
849 (u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)); in optee_handle_rpc()
853 param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC; in optee_handle_rpc()
857 * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
859 * @shm: shared memory holding the message to pass to secure world
862 * Does and SMC to OP-TEE in secure world and handles eventual resulting
863 * Remote Procedure Calls (RPC) from OP-TEE.
870 struct optee *optee = tee_get_drvdata(ctx->teedev); in optee_smc_do_call_with_arg()
877 if (optee->rpc_param_count) { in optee_smc_do_call_with_arg()
885 rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params); in optee_smc_do_call_with_arg()
909 optee_cq_wait_init(&optee->call_queue, &w); in optee_smc_do_call_with_arg()
914 optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3, in optee_smc_do_call_with_arg()
924 optee_cq_wait_for_completion(&optee->call_queue, &w); in optee_smc_do_call_with_arg()
943 optee_cq_wait_final(&optee->call_queue, &w); in optee_smc_do_call_with_arg()
959 msg_arg->cmd = cmd; in simple_call_with_arg()
1003 value = get_async_notif_value(optee->smc.invoke_fn, in notif_irq_handler()
1023 optee_smc_do_bottom_half(optee->ctx); in notif_irq_thread_fn()
1038 optee->smc.notif_irq = irq; in optee_smc_notif_init_irq()
1045 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) { in optee_smc_notif_uninit_irq()
1046 optee_smc_stop_async_notif(optee->ctx); in optee_smc_notif_uninit_irq()
1047 if (optee->smc.notif_irq) { in optee_smc_notif_uninit_irq()
1048 free_irq(optee->smc.notif_irq, optee); in optee_smc_notif_uninit_irq()
1049 irq_dispose_mapping(optee->smc.notif_irq); in optee_smc_notif_uninit_irq()
1059 * configuration. This involves for instance support for dynamic shared
1060 * memory instead of a static memory carvout.
1073 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) in optee_get_version()
1075 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL) in optee_get_version()
1082 struct optee *optee = tee_get_drvdata(ctx->teedev); in optee_smc_open()
1083 u32 sec_caps = optee->smc.sec_caps; in optee_smc_open()
1101 .name = DRIVER_NAME "-clnt",
1117 .name = DRIVER_NAME "-supp",
1136 return -EINVAL; in enable_async_notif()
1243 return ERR_PTR(-ENOENT); in optee_config_shm_memremap()
1247 pr_err("only normal cached shared memory supported\n"); in optee_config_shm_memremap()
1248 return ERR_PTR(-EINVAL); in optee_config_shm_memremap()
1254 size = end - begin; in optee_config_shm_memremap()
1258 pr_err("shared memory ioremap failed\n"); in optee_config_shm_memremap()
1259 return ERR_PTR(-EINVAL); in optee_config_shm_memremap()
1300 return ERR_PTR(-ENXIO); in get_invoke_func()
1309 return ERR_PTR(-EINVAL); in get_invoke_func()
1312 /* optee_remove - Device Removal Routine
1323 * Ask OP-TEE to free all cached shared memory objects to decrease in optee_smc_remove()
1325 * into the old shared memory range. in optee_smc_remove()
1327 if (!optee->rpc_param_count) in optee_smc_remove()
1334 if (optee->smc.memremaped_shm) in optee_smc_remove()
1335 memunmap(optee->smc.memremaped_shm); in optee_smc_remove()
1342 /* optee_shutdown - Device Removal Routine
1353 if (!optee->rpc_param_count) in optee_shutdown()
1360 struct tee_shm_pool *pool = ERR_PTR(-EINVAL); in optee_probe()
1371 invoke_fn = get_invoke_func(&pdev->dev); in optee_probe()
1377 return -EINVAL; in optee_probe()
1384 return -EINVAL; in optee_probe()
1391 return -EINVAL; in optee_probe()
1395 * Try to use dynamic shared memory if possible in optee_probe()
1400 * optee_get_msg_arg() to pre-register (by having in optee_probe()
1404 * With the page is pre-registered we can use a non-zero in optee_probe()
1409 * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages. in optee_probe()
1420 * If dynamic shared memory is not available or failed - try static one in optee_probe()
1424 * The static memory pool can use non-zero page offsets so in optee_probe()
1427 * optee_get_msg_arg() should not pre-register the in optee_probe()
1445 rc = -ENOMEM; in optee_probe()
1449 optee->ops = &optee_ops; in optee_probe()
1450 optee->smc.invoke_fn = invoke_fn; in optee_probe()
1451 optee->smc.sec_caps = sec_caps; in optee_probe()
1452 optee->rpc_param_count = rpc_param_count; in optee_probe()
1459 optee->teedev = teedev; in optee_probe()
1466 optee->supp_teedev = teedev; in optee_probe()
1468 rc = tee_device_register(optee->teedev); in optee_probe()
1472 rc = tee_device_register(optee->supp_teedev); in optee_probe()
1476 mutex_init(&optee->call_queue.mutex); in optee_probe()
1477 INIT_LIST_HEAD(&optee->call_queue.waiters); in optee_probe()
1478 optee_supp_init(&optee->supp); in optee_probe()
1479 optee->smc.memremaped_shm = memremaped_shm; in optee_probe()
1480 optee->pool = pool; in optee_probe()
1484 ctx = teedev_open(optee->teedev); in optee_probe()
1489 optee->ctx = ctx; in optee_probe()
1509 enable_async_notif(optee->smc.invoke_fn); in optee_probe()
1514 * Ensure that there are no pre-existing shm objects before enabling in optee_probe()
1526 if (!optee->rpc_param_count) in optee_probe()
1529 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) in optee_probe()
1530 pr_info("dynamic shared memory is enabled\n"); in optee_probe()
1540 if (!optee->rpc_param_count) in optee_probe()
1550 optee_supp_uninit(&optee->supp); in optee_probe()
1551 mutex_destroy(&optee->call_queue.mutex); in optee_probe()
1553 tee_device_unregister(optee->supp_teedev); in optee_probe()
1555 tee_device_unregister(optee->teedev); in optee_probe()
1566 { .compatible = "linaro,optee-tz" },