Add an OP-TEE transport for RISC-V using the RPMI TEE service group over the SBI MPXY mailbox framework.

Request one mailbox channel per hart and use the channel corresponding to the current CPU when issuing a TEE request. Probe the RPMI TEE service group and required memory-sharing capabilities before registering the transport.

This provides the basic transport and discovery support needed by the following patches.

Signed-off-by: Amirreza Zarrabi amirreza.zarrabi@oss.qualcomm.com --- drivers/tee/optee/Makefile | 1 + drivers/tee/optee/core.c | 8 +- drivers/tee/optee/optee_private.h | 34 ++++ drivers/tee/optee/optee_riscv.c | 312 +++++++++++++++++++++++++++++ drivers/tee/optee/optee_riscv.h | 141 +++++++++++++ include/linux/mailbox/riscv-rpmi-message.h | 1 + 6 files changed, 495 insertions(+), 2 deletions(-)

diff --git a/drivers/tee/optee/Makefile b/drivers/tee/optee/Makefile index ad7049c1c107..925b8ec7ef68 100644 --- a/drivers/tee/optee/Makefile +++ b/drivers/tee/optee/Makefile @@ -9,6 +9,7 @@ optee-objs += supp.o optee-objs += device.o optee-objs += smc_abi.o optee-objs += ffa_abi.o +optee-$(CONFIG_RISCV_SBI_MPXY_MBOX) += optee_riscv.o

# for tracing framework to find optee_trace.h CFLAGS_smc_abi.o := -I$(src) diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c index a52c1f498b99..63f1725e646e 100644 --- a/drivers/tee/optee/core.c +++ b/drivers/tee/optee/core.c @@ -220,6 +220,7 @@ void optee_remove_common(struct optee *optee)

static int smc_abi_rc; static int ffa_abi_rc; +static int riscv_abi_rc; static bool intf_is_regged;

static int __init optee_core_init(void) @@ -245,9 +246,10 @@ static int __init optee_core_init(void)

smc_abi_rc = optee_smc_abi_register(); ffa_abi_rc = optee_ffa_abi_register(); + riscv_abi_rc = optee_riscv_abi_register();

- /* If both failed there's no point with this module */ - if (smc_abi_rc && ffa_abi_rc) { + /* If all failed there's no point with this module */ + if (smc_abi_rc && ffa_abi_rc && riscv_abi_rc) { if (IS_REACHABLE(CONFIG_RPMB)) { rpmb_interface_unregister(&rpmb_class_intf); intf_is_regged = false; @@ -270,6 +272,8 @@ static void __exit optee_core_exit(void) optee_smc_abi_unregister(); if (!ffa_abi_rc) optee_ffa_abi_unregister(); + if (!riscv_abi_rc) + optee_riscv_abi_unregister(); } module_exit(optee_core_exit);

diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h index aefe1e6f5689..8d22d65e087b 100644 --- a/drivers/tee/optee/optee_private.h +++ b/drivers/tee/optee/optee_private.h @@ -171,6 +171,31 @@ struct optee_ffa { struct work_struct notif_work; };

+/** + * struct optee_riscv - RPMI TEE communication struct + * @chan: per-hart RPMI TEE service group mailbox channels + * @client: RPMI mailbox client used to request @chan + * @dev: device backing the RPMI TEE mailbox client + * @nr_chan: number of entries in @chan + * @max_msg_data_size: maximum RPMI message data size of the TEE channel + * @mutex: serializes access to @global_ids + * @global_ids: memory parcel id to tee_shm translation table + * + * This is the RISC-V analog of struct optee_ffa: communication with secure + * world OP-TEE OS rides the RPMI TEE service group (RPMI spec section 4.16) + * over the SBI MPXY mailbox instead of Arm FF-A. + */ +struct optee_riscv { + struct mbox_chan **chan; + struct mbox_client *client; + struct device *dev; + unsigned int nr_chan; + u32 max_msg_data_size; + /* Serializes access to @global_ids */ + struct mutex mutex; + struct rhashtable global_ids; +}; + struct optee;

/** @@ -231,6 +256,7 @@ struct optee_ops { * @ctx: driver internal TEE context * @smc: specific to SMC ABI * @ffa: specific to FF-A ABI + * @riscv: specific to RPMI TEE ABI * @shm_arg_cache: shared memory cache argument * @call_queue: queue of threads waiting to call @invoke_fn * @notif: notification synchronization struct @@ -259,6 +285,7 @@ struct optee { union { struct optee_smc smc; struct optee_ffa ffa; + struct optee_riscv riscv; }; struct optee_shm_arg_cache shm_arg_cache; struct optee_call_queue call_queue; @@ -426,5 +453,12 @@ int optee_smc_abi_register(void); void optee_smc_abi_unregister(void); int optee_ffa_abi_register(void); void optee_ffa_abi_unregister(void); +#ifdef CONFIG_RISCV_SBI_MPXY_MBOX +int optee_riscv_abi_register(void); +void optee_riscv_abi_unregister(void); +#else +static inline int optee_riscv_abi_register(void) { return -EOPNOTSUPP; } +static inline void optee_riscv_abi_unregister(void) { } +#endif

#endif /*OPTEE_PRIVATE_H*/ diff --git a/drivers/tee/optee/optee_riscv.c b/drivers/tee/optee/optee_riscv.c new file mode 100644 index 000000000000..0fe4edf92fc9 --- /dev/null +++ b/drivers/tee/optee/optee_riscv.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * This file implements the ABI used when communicating with secure world + * OP-TEE OS over the RPMI TEE service group (RPMI spec section 4.16). It is + * the RISC-V analog of ffa_abi.c: OP-TEE and Linux are peer endpoints of the + * RPMI framework (OpenSBI), and shared memory follows the FF-A memory-donation + * model through the RPMI memory parcel services. + * + * This file is divided into the following sections: + * 1. Low level RPMI TEE service group transport over the SBI MPXY mailbox + * 2. Feature discovery and notification handshake + * 3. Driver initialization + * + * The remaining FF-A-equivalent sections (parcel id hash table, tee_param + * marshalling, dynamic shared memory pool and the scheduled call into secure + * world) are added on top of this transport layer. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/errno.h> +#include <linux/mailbox_client.h> +#include <linux/mailbox/riscv-rpmi-message.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/smp.h> +#include <linux/tee_core.h> +#include <linux/types.h> + +#include "optee_private.h" +#include "optee_riscv.h" + +/* + * 1. Low level RPMI TEE service group transport over the SBI MPXY mailbox + * + * The RPMI TEE service group is reached through the SBI MPXY mailbox. Each + * hart owns a dedicated MPXY channel so that a call issued on a given hart is + * serviced by the OP-TEE context bound to it; optee_riscv_send() therefore + * selects the channel of the running hart. All RPMI messages are exchanged + * synchronously with rpmi_mbox_send_message(). + */ + +static int optee_riscv_send(struct optee *optee, struct rpmi_mbox_message *msg) +{ + int cpu, ret; + + cpu = get_cpu(); + if (cpu >= optee->riscv.nr_chan || !optee->riscv.chan[cpu]) { + put_cpu(); + return -ENODEV; + } + ret = rpmi_mbox_send_message(optee->riscv.chan[cpu], msg); + put_cpu(); + + return ret; +} + +/* + * 2. Feature discovery and notification handshake + * + * TEE_PROBE_FEATURES (0x02) reports which framework features are available; + * TEE_ENABLE_NOTIFICATION (0x01) subscribes to TEE service group events. Both + * are mandatory services (RPMI spec section 4.16), so probing them also + * confirms that the framework speaks the TEE service group on this channel. + */ + +static int optee_riscv_probe_feature(struct optee *optee, u32 feature_id, + u32 *value) +{ + struct rpmi_tee_probe_features_req tx = { + .feature_id = cpu_to_le32(feature_id), + }; + struct rpmi_tee_probe_features_resp rx = { }; + struct rpmi_mbox_message msg; + int ret; + + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_PROBE_FEATURES, + &tx, sizeof(tx), &rx, sizeof(rx)); + ret = optee_riscv_send(optee, &msg); + if (ret) + return ret; + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + if (value) + *value = le32_to_cpu(rx.value); + + return 0; +} + +static int optee_riscv_features(struct optee *optee) +{ + u32 share = RPMI_TEE_MEMORY_SHARE_NONE; + int ret; + + /* + * Memory parcels carry normal-world shared memory to OP-TEE, so the + * framework must support sharing memory between the REE and a TEE. + */ + ret = optee_riscv_probe_feature(optee, RPMI_TEE_FEAT_MEMORY_SHARE, + &share); + if (ret) { + pr_err("Failed to probe MEMORY_SHARE feature: %d\n", ret); + return ret; + } + if (share != RPMI_TEE_MEMORY_SHARE_FULL) { + pr_err("Framework cannot share memory between REE and TEE (%u)\n", + share); + return -EOPNOTSUPP; + } + + return 0; +} + +static int optee_riscv_enable_notif(struct optee *optee) +{ + struct rpmi_tee_probe_features_resp rx = { }; + struct rpmi_mbox_message msg; + int ret; + + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_ENABLE_NOTIFICATION, + NULL, 0, &rx, sizeof(rx)); + ret = optee_riscv_send(optee, &msg); + if (ret) + return ret; + + /* + * The TEE service group defines no notification events on this + * platform, so RPMI_ERR_NOTSUPP is expected and not fatal. + */ + if (rx.status && le32_to_cpu(rx.status) != (u32)RPMI_ERR_NOTSUPP) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + return 0; +} + +/* + * 3. Driver initialization + * + * The RPMI TEE service group is described in the device tree by a single + * node whose "mboxes" property lists one SBI MPXY channel per hart, in hart + * order. The driver requests each list entry by index and validates the + * transport before building the OP-TEE device. + */ + +static int optee_riscv_request_channels(struct optee *optee) +{ + struct device *dev = optee->riscv.dev; + int nr_mboxes; + unsigned int cpuid; + + nr_mboxes = of_count_phandle_with_args(dev->of_node, "mboxes", + "#mbox-cells"); + if (nr_mboxes != optee->riscv.nr_chan) + return dev_err_probe(dev, -EINVAL, + "Expected %u mailbox channels, got %d\n", + optee->riscv.nr_chan, nr_mboxes); + + for (cpuid = 0; cpuid < optee->riscv.nr_chan; cpuid++) { + optee->riscv.chan[cpuid] = + mbox_request_channel(optee->riscv.client, cpuid); + if (IS_ERR(optee->riscv.chan[cpuid])) { + int ret = PTR_ERR(optee->riscv.chan[cpuid]); + + optee->riscv.chan[cpuid] = NULL; + return dev_err_probe(dev, ret, + "Failed to request channel %u\n", + cpuid); + } + } + + return 0; +} + +static void optee_riscv_free_channels(struct optee *optee) +{ + unsigned int i; + + for (i = 0; i < optee->riscv.nr_chan; i++) { + if (optee->riscv.chan[i]) + mbox_free_channel(optee->riscv.chan[i]); + } +} + +static int optee_riscv_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rpmi_mbox_message msg; + struct mbox_client *client; + struct optee *optee; + u32 servicegroup_id; + unsigned int nr_cpus; + int ret; + + nr_cpus = num_possible_cpus(); + if (!nr_cpus) + return dev_err_probe(dev, -ENODEV, "No harts found\n"); + + optee = kzalloc_obj(*optee); + if (!optee) + return -ENOMEM; + + client = devm_kzalloc(dev, sizeof(*client), GFP_KERNEL); + if (!client) { + ret = -ENOMEM; + goto err_free_optee; + } + client->dev = dev; + client->rx_callback = NULL; + client->tx_block = false; + client->knows_txdone = true; + client->tx_tout = 0; + + optee->riscv.dev = dev; + optee->riscv.client = client; + optee->riscv.nr_chan = nr_cpus; + optee->riscv.chan = kcalloc(nr_cpus, sizeof(*optee->riscv.chan), + GFP_KERNEL); + if (!optee->riscv.chan) { + ret = -ENOMEM; + goto err_free_optee; + } + + ret = optee_riscv_request_channels(optee); + if (ret) + goto err_free_channels; + + /* Confirm the channel really speaks the TEE service group. */ + rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_SERVICEGROUP_ID); + ret = optee_riscv_send(optee, &msg); + if (ret) { + dev_err_probe(dev, ret, "Failed to get service group id\n"); + goto err_free_channels; + } + servicegroup_id = msg.attr.value; + if (servicegroup_id != RPMI_SRVGRP_TEE) { + ret = -ENODEV; + dev_err_probe(dev, ret, "Not a TEE service group channel (0x%x)\n", + servicegroup_id); + goto err_free_channels; + } + + rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_MAX_MSG_DATA_SIZE); + ret = optee_riscv_send(optee, &msg); + if (ret) { + dev_err_probe(dev, ret, "Failed to get max msg data size\n"); + goto err_free_channels; + } + optee->riscv.max_msg_data_size = msg.attr.value; + + ret = optee_riscv_features(optee); + if (ret) { + dev_err_probe(dev, ret, "Missing required TEE features\n"); + goto err_free_channels; + } + + ret = optee_riscv_enable_notif(optee); + if (ret) { + dev_err_probe(dev, ret, "Failed to enable notifications\n"); + goto err_free_channels; + } + + platform_set_drvdata(pdev, optee); + dev_info(dev, "initialized driver\n"); + + return 0; + +err_free_channels: + optee_riscv_free_channels(optee); + kfree(optee->riscv.chan); +err_free_optee: + kfree(optee); + return ret; +} + +static void optee_riscv_remove(struct platform_device *pdev) +{ + struct optee *optee = platform_get_drvdata(pdev); + + optee_riscv_free_channels(optee); + kfree(optee->riscv.chan); + kfree(optee); +} + +static const struct of_device_id optee_riscv_match[] = { + { .compatible = "riscv,rpmi-mpxy-tee" }, + { } +}; +MODULE_DEVICE_TABLE(of, optee_riscv_match); + +static struct platform_driver optee_riscv_driver = { + .driver = { + .name = DRIVER_NAME "-riscv", + .of_match_table = optee_riscv_match, + }, + .probe = optee_riscv_probe, + .remove = optee_riscv_remove, +}; + +int optee_riscv_abi_register(void) +{ + return platform_driver_register(&optee_riscv_driver); +} + +void optee_riscv_abi_unregister(void) +{ + platform_driver_unregister(&optee_riscv_driver); +} diff --git a/drivers/tee/optee/optee_riscv.h b/drivers/tee/optee/optee_riscv.h new file mode 100644 index 000000000000..d87298faa6a2 --- /dev/null +++ b/drivers/tee/optee/optee_riscv.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/* + * This file is exported by OP-TEE and is kept in sync between secure world + * and normal world drivers. It describes the wire contract used when + * communicating with secure world OP-TEE OS over the RPMI TEE service group + * (RPMI specification section 4.16, SERVICEGROUP_ID 0x0010). + * + * The RPMI TEE service group is the RISC-V analog of Arm FF-A: OP-TEE and the + * rich execution environment (REE, i.e. Linux) are peer endpoints, and the + * RPMI framework (OpenSBI in M-mode) mediates every message. Memory sharing + * follows the FF-A memory-donation model through the memory parcel services: + * the REE creates a parcel describing its pages, OP-TEE accepts it lazily by + * parcel id, and teardown is two phased (OP-TEE releases, the REE reclaims). + * + * All request and response payloads are little-endian uint32 words as defined + * by the RPMI specification. These definitions MUST byte-match the OpenSBI + * framework definitions in <sbi_utils/mailbox/rpmi_msgprot.h>. + */ + +#ifndef __OPTEE_RISCV_H +#define __OPTEE_RISCV_H + +#include <linux/mailbox/riscv-rpmi-message.h> +#include <linux/types.h> + +/* + * RPMI TEE service ids (RPMI spec section 4.16, Table 181). + * + * Only TEE_ENABLE_NOTIFICATION, TEE_PROBE_FEATURES and TEE_CALL are mandated; + * the remaining services are optional and may return RPMI_ERR_NOTSUPP. + */ +enum rpmi_tee_service_id { + RPMI_TEE_SRV_ENABLE_NOTIFICATION = 0x01, + RPMI_TEE_SRV_PROBE_FEATURES = 0x02, + RPMI_TEE_SRV_PROBE_SYSTEM = 0x03, + RPMI_TEE_SRV_EXIT = 0x04, + RPMI_TEE_SRV_SIGNAL_BUS_SETUP = 0x05, + RPMI_TEE_SRV_SIGNAL_BUS_TEARDOWN = 0x06, + RPMI_TEE_SRV_SIGNAL_RAISE = 0x07, + RPMI_TEE_SRV_SIGNAL_RETRIEVE = 0x08, + RPMI_TEE_SRV_MEM_PARCEL_CREATE = 0x09, + RPMI_TEE_SRV_MEM_PARCEL_ACCEPT = 0x0a, + RPMI_TEE_SRV_MEM_PARCEL_RELEASE = 0x0b, + RPMI_TEE_SRV_MEM_PARCEL_RECLAIM = 0x0c, + RPMI_TEE_SRV_MEM_PARCEL_SEGMENT_SEND = 0x0d, + RPMI_TEE_SRV_MEM_PARCEL_SEGMENT_RECEIVE = 0x0e, + RPMI_TEE_SRV_CALL = 0x13, + RPMI_TEE_SRV_MAX_COUNT, +}; + +/* + * RPMI TEE endpoint identities. + * + * The RPMI specification does not fix numeric endpoint ids; they are assigned + * by the framework at runtime. These values match the OpenSBI framework + * assignment used on this platform: the REE is endpoint 0 and OP-TEE is + * endpoint 1. + */ +#define RPMI_TEE_ENDPOINT_REE 0 +#define RPMI_TEE_ENDPOINT_OPTEE 1 + +/* + * RPMI TEE feature ids for TEE_PROBE_FEATURES (RPMI spec section 4.16.4, + * Table 182). + */ +enum rpmi_tee_feature_id { + RPMI_TEE_FEAT_MEMORY_DONATE = 1, + RPMI_TEE_FEAT_MEMORY_LEND = 2, + RPMI_TEE_FEAT_MEMORY_SHARE = 3, + RPMI_TEE_FEAT_SIGNAL_BUS = 4, + RPMI_TEE_FEAT_MULTISEGMENT_OPS = 5, + RPMI_TEE_FEAT_SYSINFO_FORMAT = 6, +}; + +/* MEMORY_SHARE feature values (RPMI spec Table 182). */ +#define RPMI_TEE_MEMORY_SHARE_NONE 0 +#define RPMI_TEE_MEMORY_SHARE_TEE_ONLY 1 +#define RPMI_TEE_MEMORY_SHARE_FULL 2 + +/* TEE_PROBE_FEATURES request (Table 183) / response (Table 184). */ +struct rpmi_tee_probe_features_req { + __le32 feature_id; +}; + +struct rpmi_tee_probe_features_resp { + __le32 status; + __le32 value; +}; + +/* + * TEE_CALL wire encoding (RPMI spec section 4.16.21, Tables 218 and 219). + * + * TEE_CALL is the mandatory doorbell service used to enter OP-TEE. The + * request carries a fixed REE->OP-TEE identity, the well-known OP-TEE service + * UUID and a SERVICE_DATA payload; the response carries a STATUS word, a + * SERVICE_RSP_LEN word and the SERVICE_RSP payload. + * + * The SERVICE_DATA/SERVICE_RSP registers are XLEN-sized little-endian values. + * The structures are __packed so the 16-byte UUID does not force padding + * before the length word. + */ +#define RPMI_TEE_UUID_LEN 16 + +/* OP-TEE communicate service UUID: 5be1b1a0-7e11-4e7a-9b10-0010c0ffee00 */ +#define RPMI_TEE_OPTEE_UUID \ + { 0x5b, 0xe1, 0xb1, 0xa0, 0x7e, 0x11, 0x4e, 0x7a, \ + 0x9b, 0x10, 0x00, 0x10, 0xc0, 0xff, 0xee, 0x00 } + +/* OP-TEE SMC-style call convention carried inside SERVICE_DATA. */ +#define RPMI_TEE_OPTEE_CALL_REGS 8 /* a0-a7 */ +#define RPMI_TEE_OPTEE_RESP_REGS 4 /* a0-a3 */ + +#if __riscv_xlen == 64 +typedef __le64 rpmi_xlen_t; +#define cpu_to_rpmi_xlen(x) cpu_to_le64(x) +#define rpmi_xlen_to_cpu(x) le64_to_cpu(x) +#else +typedef __le32 rpmi_xlen_t; +#define cpu_to_rpmi_xlen(x) cpu_to_le32(x) +#define rpmi_xlen_to_cpu(x) le32_to_cpu(x) +#endif + +struct rpmi_tee_call_req { + __le32 sender_id; + __le32 target_id; + u8 service[RPMI_TEE_UUID_LEN]; + __le32 service_data_len; + rpmi_xlen_t reg[RPMI_TEE_OPTEE_CALL_REGS]; +} __packed; + +struct rpmi_tee_call_resp { + __le32 status; + __le32 service_rsp_len; + rpmi_xlen_t reg[RPMI_TEE_OPTEE_RESP_REGS]; +} __packed; + +#endif /* __OPTEE_RISCV_H */ diff --git a/include/linux/mailbox/riscv-rpmi-message.h b/include/linux/mailbox/riscv-rpmi-message.h index e135c6564d0c..47a540bd81c9 100644 --- a/include/linux/mailbox/riscv-rpmi-message.h +++ b/include/linux/mailbox/riscv-rpmi-message.h @@ -93,6 +93,7 @@ static inline int rpmi_to_linux_error(int rpmi_error) /* RPMI service group IDs */ #define RPMI_SRVGRP_SYSTEM_MSI 0x00002 #define RPMI_SRVGRP_CLOCK 0x00008 +#define RPMI_SRVGRP_TEE 0x00010

/* RPMI clock service IDs */ enum rpmi_clock_service_id {

Add the shared-memory and call support required by the RISC-V RPMI transport.

Use RPMI memory parcels to share memory between Linux and OP-TEE and keep a mapping between parcel identifiers and struct tee_shm. Pass the parcel identifier and offset when referencing shared memory from OP-TEE message parameters.

Implement scheduled calls using RPMI TEE_CALL, including yielding calls, RPC handling and shared-memory allocation. Also add OP-TEE version and capability negotiation and complete registration of the OP-TEE device.

Signed-off-by: Amirreza Zarrabi amirreza.zarrabi@oss.qualcomm.com --- drivers/tee/optee/optee_private.h | 2 + drivers/tee/optee/optee_riscv.c | 1198 ++++++++++++++++++++++++++++++++++--- drivers/tee/optee/optee_riscv.h | 135 ++++- 3 files changed, 1257 insertions(+), 78 deletions(-)

diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h index 8d22d65e087b..cf878b8178f9 100644 --- a/drivers/tee/optee/optee_private.h +++ b/drivers/tee/optee/optee_private.h @@ -178,6 +178,7 @@ struct optee_ffa { * @dev: device backing the RPMI TEE mailbox client * @nr_chan: number of entries in @chan * @max_msg_data_size: maximum RPMI message data size of the TEE channel + * @next_nonce: monotonic nonce source for memory parcel creation * @mutex: serializes access to @global_ids * @global_ids: memory parcel id to tee_shm translation table * @@ -191,6 +192,7 @@ struct optee_riscv { struct device *dev; unsigned int nr_chan; u32 max_msg_data_size; + atomic_t next_nonce; /* Serializes access to @global_ids */ struct mutex mutex; struct rhashtable global_ids; diff --git a/drivers/tee/optee/optee_riscv.c b/drivers/tee/optee/optee_riscv.c index 0fe4edf92fc9..36115326486d 100644 --- a/drivers/tee/optee/optee_riscv.c +++ b/drivers/tee/optee/optee_riscv.c @@ -8,34 +8,46 @@ * RPMI framework (OpenSBI), and shared memory follows the FF-A memory-donation * model through the RPMI memory parcel services. * - * This file is divided into the following sections: - * 1. Low level RPMI TEE service group transport over the SBI MPXY mailbox - * 2. Feature discovery and notification handshake - * 3. Driver initialization + * This file is structured exactly like ffa_abi.c: + * 1. Maintain a hash table for lookup of a memory parcel id + * 2. Convert between struct tee_param and struct optee_msg_param + * 3. Low level support functions to register shared memory in secure world + * 4. Dynamic shared memory pool based on alloc_pages() + * 5. Do a normal scheduled call into secure world + * 6. Driver initialization * - * The remaining FF-A-equivalent sections (parcel id hash table, tee_param - * marshalling, dynamic shared memory pool and the scheduled call into secure - * world) are added on top of this transport layer. + * Every FF-A memory operation has a direct RPMI TEE service group analog: + * FFA_MEM_SHARE -> MEM_PARCEL_CREATE (0x09), issued by the REE + * FFA_MEM_RECLAIM -> MEM_PARCEL_RECLAIM (0x0c), issued by the REE + * direct message -> TEE_CALL (0x13), the call doorbell + * and the FF-A g_handle is replaced by a memory parcel id folded together + * with a caller-supplied nonce. */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

+#include <linux/atomic.h> #include <linux/errno.h> #include <linux/mailbox_client.h> #include <linux/mailbox/riscv-rpmi-message.h> +#include <linux/mm.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/rhashtable.h> +#include <linux/sched.h> #include <linux/slab.h> #include <linux/smp.h> +#include <linux/string.h> #include <linux/tee_core.h> #include <linux/types.h>

#include "optee_private.h" #include "optee_riscv.h" +#include "optee_rpc_cmd.h"

/* - * 1. Low level RPMI TEE service group transport over the SBI MPXY mailbox + * Low level RPMI TEE service group transport over the SBI MPXY mailbox. * * The RPMI TEE service group is reached through the SBI MPXY mailbox. Each * hart owns a dedicated MPXY channel so that a call issued on a given hart is @@ -60,25 +72,43 @@ static int optee_riscv_send(struct optee *optee, struct rpmi_mbox_message *msg) }

/* - * 2. Feature discovery and notification handshake + * optee_riscv_tee_call() - issue a TEE_CALL (RPMI service 0x13) + * @optee: main service struct + * @in: the command words carried in SERVICE_DATA, the RISC-V analog + * of struct ffa_send_direct_data's data0-data4 (w3-w7) + * @out: the response words returned in SERVICE_RSP, the RISC-V analog + * of the same data0-data4 set on the return path * - * TEE_PROBE_FEATURES (0x02) reports which framework features are available; - * TEE_ENABLE_NOTIFICATION (0x01) subscribes to TEE service group events. Both - * are mandatory services (RPMI spec section 4.16), so probing them also - * confirms that the framework speaks the TEE service group on this channel. + * TEE_CALL is the RISC-V analog of the FF-A direct message: it is the single + * doorbell used both for the blocking (fast) calls of section 6 and for the + * yielding call of section 5. The struct optee_msg_arg itself is never + * carried here, only its parcel handle and offset, exactly as FF-A carries + * only w4-w6. + * + * Returns 0 on success or <0 on failure. */ - -static int optee_riscv_probe_feature(struct optee *optee, u32 feature_id, - u32 *value) +static int optee_riscv_tee_call(struct optee *optee, + const u64 in[RPMI_TEE_OPTEE_CALL_REGS], + u64 out[RPMI_TEE_OPTEE_RESP_REGS]) { - struct rpmi_tee_probe_features_req tx = { - .feature_id = cpu_to_le32(feature_id), + static const u8 optee_uuid[RPMI_TEE_UUID_LEN] = RPMI_TEE_OPTEE_UUID; + struct rpmi_tee_call_req tx = { + .sender_id = cpu_to_le32(RPMI_TEE_ENDPOINT_REE), + .target_id = cpu_to_le32(RPMI_TEE_ENDPOINT_OPTEE), + .service_data_len = + cpu_to_le32(RPMI_TEE_OPTEE_CALL_REGS * + sizeof(rpmi_xlen_t)), }; - struct rpmi_tee_probe_features_resp rx = { }; + struct rpmi_tee_call_resp rx = { }; struct rpmi_mbox_message msg; + unsigned int i; int ret;

- rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_PROBE_FEATURES, + memcpy(tx.service, optee_uuid, sizeof(tx.service)); + for (i = 0; i < RPMI_TEE_OPTEE_CALL_REGS; i++) + tx.reg[i] = cpu_to_rpmi_xlen(in[i]); + + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_CALL, &tx, sizeof(tx), &rx, sizeof(rx)); ret = optee_riscv_send(optee, &msg); if (ret) @@ -86,61 +116,887 @@ static int optee_riscv_probe_feature(struct optee *optee, u32 feature_id, if (rx.status) return rpmi_to_linux_error(le32_to_cpu(rx.status));

- if (value) - *value = le32_to_cpu(rx.value); + for (i = 0; i < RPMI_TEE_OPTEE_RESP_REGS; i++) + out[i] = rpmi_xlen_to_cpu(rx.reg[i]);

return 0; }

-static int optee_riscv_features(struct optee *optee) +/* + * 1. Maintain a hash table for lookup of a memory parcel id + * + * The RPMI framework assigns a memory parcel id for each piece of shared + * memory. Together with a caller-supplied nonce it forms the wire identity + * used when communicating with secure world, playing the exact role of the + * FF-A global memory handle. + * + * Main functions are optee_shm_add_riscv_handle() and + * optee_shm_rem_riscv_handle(). + */ +struct shm_rhash { + struct tee_shm *shm; + u64 global_id; + struct rhash_head linkage; +}; + +static void rh_free_fn(void *ptr, void *arg) { - u32 share = RPMI_TEE_MEMORY_SHARE_NONE; + kfree(ptr); +} + +static const struct rhashtable_params shm_rhash_params = { + .head_offset = offsetof(struct shm_rhash, linkage), + .key_len = sizeof(u64), + .key_offset = offsetof(struct shm_rhash, global_id), + .automatic_shrinking = true, +}; + +static struct tee_shm *optee_shm_from_riscv_handle(struct optee *optee, + u64 global_id) +{ + struct tee_shm *shm = NULL; + struct shm_rhash *r; + + mutex_lock(&optee->riscv.mutex); + r = rhashtable_lookup_fast(&optee->riscv.global_ids, &global_id, + shm_rhash_params); + if (r) + shm = r->shm; + mutex_unlock(&optee->riscv.mutex); + + return shm; +} + +static int optee_shm_add_riscv_handle(struct optee *optee, struct tee_shm *shm, + u64 global_id) +{ + struct shm_rhash *r; + int rc; + + r = kmalloc_obj(*r); + if (!r) + return -ENOMEM; + r->shm = shm; + r->global_id = global_id; + + mutex_lock(&optee->riscv.mutex); + rc = rhashtable_lookup_insert_fast(&optee->riscv.global_ids, + &r->linkage, shm_rhash_params); + mutex_unlock(&optee->riscv.mutex); + + if (rc) + kfree(r); + + return rc; +} + +static int optee_shm_rem_riscv_handle(struct optee *optee, u64 global_id) +{ + struct shm_rhash *r; + int rc = -ENOENT; + + mutex_lock(&optee->riscv.mutex); + r = rhashtable_lookup_fast(&optee->riscv.global_ids, &global_id, + shm_rhash_params); + if (r) + rc = rhashtable_remove_fast(&optee->riscv.global_ids, + &r->linkage, shm_rhash_params); + mutex_unlock(&optee->riscv.mutex); + + if (!rc) + kfree(r); + + return rc; +} + +/* + * 2. Convert between struct tee_param and struct optee_msg_param + * + * optee_riscv_from_msg_param() and optee_riscv_to_msg_param() are the main + * functions. They are identical to their FF-A counterparts: the memref + * carries only the parcel handle (stored in fmem.global_id, the same slot + * FF-A uses for its g_handle), an offset and a size, never a page list. + */ + +static void from_msg_param_riscv_mem(struct optee *optee, struct tee_param *p, + u32 attr, const struct optee_msg_param *mp) +{ + struct tee_shm *shm = NULL; + u64 offs_high = 0; + u64 offs_low = 0; + + p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT + + attr - OPTEE_MSG_ATTR_TYPE_FMEM_INPUT; + p->u.memref.size = mp->u.fmem.size; + + if (mp->u.fmem.global_id != OPTEE_MSG_FMEM_INVALID_GLOBAL_ID) + shm = optee_shm_from_riscv_handle(optee, mp->u.fmem.global_id); + p->u.memref.shm = shm; + + if (shm) { + offs_low = mp->u.fmem.offs_low; + offs_high = mp->u.fmem.offs_high; + } + p->u.memref.shm_offs = offs_low | offs_high << 32; +} + +/** + * optee_riscv_from_msg_param() - convert from OPTEE_MSG parameters to + * struct tee_param + * @optee: main service struct + * @params: subsystem internal parameter representation + * @num_params: number of elements in the parameter arrays + * @msg_params: OPTEE_MSG parameters + * + * Returns 0 on success or <0 on failure + */ +static int optee_riscv_from_msg_param(struct optee *optee, + struct tee_param *params, + size_t num_params, + const struct optee_msg_param *msg_params) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + struct tee_param *p = params + n; + const struct optee_msg_param *mp = msg_params + n; + u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK; + + switch (attr) { + case OPTEE_MSG_ATTR_TYPE_NONE: + p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; + memset(&p->u, 0, sizeof(p->u)); + break; + case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT: + case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT: + optee_from_msg_param_value(p, attr, mp); + break; + case OPTEE_MSG_ATTR_TYPE_FMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_FMEM_INOUT: + from_msg_param_riscv_mem(optee, p, attr, mp); + break; + default: + return -EINVAL; + } + } + + return 0; +} + +static int to_msg_param_riscv_mem(struct optee_msg_param *mp, + const struct tee_param *p) +{ + struct tee_shm *shm = p->u.memref.shm; + + mp->attr = OPTEE_MSG_ATTR_TYPE_FMEM_INPUT + p->attr - + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; + + if (shm) { + u64 shm_offs = p->u.memref.shm_offs; + + mp->u.fmem.internal_offs = shm->offset; + + mp->u.fmem.offs_low = shm_offs; + mp->u.fmem.offs_high = shm_offs >> 32; + /* Check that the entire offset could be stored. */ + if (mp->u.fmem.offs_high != shm_offs >> 32) + return -EINVAL; + + mp->u.fmem.global_id = shm->sec_world_id; + } else { + memset(&mp->u, 0, sizeof(mp->u)); + mp->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID; + } + mp->u.fmem.size = p->u.memref.size; + + return 0; +} + +/** + * optee_riscv_to_msg_param() - convert from struct tee_params to OPTEE_MSG + * parameters + * @optee: main service struct + * @msg_params: OPTEE_MSG parameters + * @num_params: number of elements in the parameter arrays + * @params: subsystem internal parameter representation + * + * Returns 0 on success or <0 on failure + */ +static int optee_riscv_to_msg_param(struct optee *optee, + struct optee_msg_param *msg_params, + size_t num_params, + const struct tee_param *params) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + const struct tee_param *p = params + n; + struct optee_msg_param *mp = msg_params + n; + + switch (p->attr) { + case TEE_IOCTL_PARAM_ATTR_TYPE_NONE: + mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; + memset(&mp->u, 0, sizeof(mp->u)); + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + optee_to_msg_param_value(mp, p); + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + if (to_msg_param_riscv_mem(mp, p)) + return -EINVAL; + break; + default: + return -EINVAL; + } + } + + return 0; +} + +/* + * 3. Low level support functions to register shared memory in secure world + * + * Functions to register and unregister shared memory both for normal + * clients and for tee-supplicant. Registration creates an RPMI memory + * parcel (MEM_PARCEL_CREATE), which is the analog of FFA_MEM_SHARE; + * unregistration reclaims it (MEM_PARCEL_RECLAIM), the analog of + * FFA_MEM_RECLAIM, after a synchronous handshake with OP-TEE. + */ + +/* + * Coalesce a page array into RPMI block-list entries (Table 198). Each entry + * spans a run of physically contiguous pages, up to RPMI_TEE_PARCEL_BLOCK_MAX_ + * PAGES. When @block_high / @block_low are NULL only the entry count is + * computed, so the caller can size the request buffer first. + */ +static u32 optee_riscv_build_blocks(struct page **pages, size_t num_pages, + __le32 *block_high, __le32 *block_low) +{ + u32 nblocks = 0; + size_t i = 0; + + while (i < num_pages) { + u64 pfn = page_to_pfn(pages[i]); + u32 run = 1; + + while (i + run < num_pages && + run < RPMI_TEE_PARCEL_BLOCK_MAX_PAGES && + page_to_pfn(pages[i + run]) == pfn + run) + run++; + + if (block_high && block_low) { + block_high[nblocks] = rpmi_tee_block_high(pfn); + block_low[nblocks] = rpmi_tee_block_low(pfn, run); + } + nblocks++; + i += run; + } + + return nblocks; +} + +/* + * Issue MEM_PARCEL_CREATE (RPMI service 0x09) for @pages with the REE as the + * creator and OP-TEE as the sole read/write receiver. Returns the framework + * assigned parcel id (>= 0) or a negative errno. + */ +static int optee_riscv_parcel_create(struct optee *optee, struct page **pages, + size_t num_pages, u32 nonce) +{ + struct rpmi_tee_mem_parcel_create_req *req; + struct rpmi_tee_mem_parcel_create_resp rx = { }; + struct rpmi_mbox_message msg; + __le32 *block_high, *block_low; + size_t req_len; + u32 block_cnt; + __le32 *data; int ret;

+ block_cnt = optee_riscv_build_blocks(pages, num_pages, NULL, NULL); + /* - * Memory parcels carry normal-world shared memory to OP-TEE, so the - * framework must support sharing memory between the REE and a TEE. + * Layout of the trailing data[] array (Table 200): one receiver_id and + * one access word (receiver_cnt == 1), then block_high[block_cnt] and + * block_low[block_cnt]. */ - ret = optee_riscv_probe_feature(optee, RPMI_TEE_FEAT_MEMORY_SHARE, - &share); - if (ret) { - pr_err("Failed to probe MEMORY_SHARE feature: %d\n", ret); + req_len = struct_size(req, data, 2 + 2 * block_cnt); + if (optee->riscv.max_msg_data_size && + req_len > optee->riscv.max_msg_data_size) + return -E2BIG; + + req = kzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + + req->creator_id = cpu_to_le32(RPMI_TEE_ENDPOINT_REE); + req->creator_access = cpu_to_le32(RPMI_TEE_PARCEL_ACCESS_R | + RPMI_TEE_PARCEL_ACCESS_W); + req->receiver_cnt = cpu_to_le32(1); + req->flags = 0; + req->nonce = cpu_to_le32(nonce); + req->block_cnt = cpu_to_le32(block_cnt); + + data = req->data; + data[0] = cpu_to_le32(RPMI_TEE_ENDPOINT_OPTEE); + data[1] = cpu_to_le32(RPMI_TEE_PARCEL_ACCESS_R | + RPMI_TEE_PARCEL_ACCESS_W); + block_high = &data[2]; + block_low = &data[2 + block_cnt]; + optee_riscv_build_blocks(pages, num_pages, block_high, block_low); + + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_MEM_PARCEL_CREATE, + req, req_len, &rx, sizeof(rx)); + ret = optee_riscv_send(optee, &msg); + kfree(req); + if (ret) return ret; - } - if (share != RPMI_TEE_MEMORY_SHARE_FULL) { - pr_err("Framework cannot share memory between REE and TEE (%u)\n", - share); - return -EOPNOTSUPP; - } + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status));

- return 0; + return le32_to_cpu(rx.mem_parcel_id); }

-static int optee_riscv_enable_notif(struct optee *optee) +/* + * Issue MEM_PARCEL_RECLAIM (RPMI service 0x0c). OpenSBI fails the reclaim + * while any receiver still holds the parcel, so this is only called after the + * OPTEE_ABI_UNREGISTER_SHM handshake below has confirmed OP-TEE released it. + */ +static int optee_riscv_parcel_reclaim(struct optee *optee, u32 parcel_id) { - struct rpmi_tee_probe_features_resp rx = { }; + struct rpmi_tee_mem_parcel_reclaim_req tx = { + .mem_parcel_id = cpu_to_le32(parcel_id), + }; + struct rpmi_tee_mem_parcel_reclaim_resp rx = { }; struct rpmi_mbox_message msg; int ret;

- rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_ENABLE_NOTIFICATION, - NULL, 0, &rx, sizeof(rx)); + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_MEM_PARCEL_RECLAIM, + &tx, sizeof(tx), &rx, sizeof(rx)); ret = optee_riscv_send(optee, &msg); if (ret) return ret; + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + return 0; +} + +static int optee_riscv_shm_register(struct tee_context *ctx, + struct tee_shm *shm, struct page **pages, + size_t num_pages, unsigned long start) +{ + struct optee *optee = tee_get_drvdata(ctx->teedev); + u64 global_id; + u32 nonce; + int rc; + + rc = optee_check_mem_type(start, num_pages); + if (rc) + return rc;

/* - * The TEE service group defines no notification events on this - * platform, so RPMI_ERR_NOTSUPP is expected and not fatal. + * MEM_PARCEL_CREATE returns only a parcel id; the nonce is + * caller-supplied. Fold them into the FF-A style 64-bit handle: + * parcel id in the low word, nonce in the high word. */ - if (rx.status && le32_to_cpu(rx.status) != (u32)RPMI_ERR_NOTSUPP) - return rpmi_to_linux_error(le32_to_cpu(rx.status)); + nonce = (u32)atomic_inc_return(&optee->riscv.next_nonce); + rc = optee_riscv_parcel_create(optee, pages, num_pages, nonce); + if (rc < 0) + return rc; + global_id = (u32)rc | ((u64)nonce << 32); + + rc = optee_shm_add_riscv_handle(optee, shm, global_id); + if (rc) { + optee_riscv_parcel_reclaim(optee, (u32)global_id); + return rc; + } + + shm->sec_world_id = global_id;

return 0; }

+static int optee_riscv_shm_unregister(struct tee_context *ctx, + struct tee_shm *shm) +{ + struct optee *optee = tee_get_drvdata(ctx->teedev); + u64 global_id = shm->sec_world_id; + u64 in[RPMI_TEE_OPTEE_CALL_REGS] = { + OPTEE_ABI_UNREGISTER_SHM, + (u32)global_id, + global_id >> 32, + 0, + }; + u64 out[RPMI_TEE_OPTEE_RESP_REGS] = { }; + int rc; + + optee_shm_rem_riscv_handle(optee, global_id); + shm->sec_world_id = 0; + + /* + * Synchronous teardown handshake, the analog of the FF-A + * OPTEE_FFA_UNREGISTER_SHM blocking call: OP-TEE releases the parcel on + * its own TEE channel before we reclaim it. Only reclaim once OP-TEE + * has acknowledged, so we never race the release. + */ + rc = optee_riscv_tee_call(optee, in, out); + if (rc) + pr_err("Unregister SHM id 0x%llx rc %d\n", global_id, rc); + + rc = optee_riscv_parcel_reclaim(optee, (u32)global_id); + if (rc) + pr_err("parcel_reclaim: 0x%llx %d\n", global_id, rc); + + return rc; +} + +static int optee_riscv_shm_unregister_supp(struct tee_context *ctx, + struct tee_shm *shm) +{ + struct optee *optee = tee_get_drvdata(ctx->teedev); + u64 global_id = shm->sec_world_id; + int rc; + + /* + * We're skipping the OPTEE_ABI_UNREGISTER_SHM handshake since this is + * OP-TEE freeing via RPC, so it has already retired this parcel. + */ + optee_shm_rem_riscv_handle(optee, global_id); + shm->sec_world_id = 0; + + rc = optee_riscv_parcel_reclaim(optee, (u32)global_id); + if (rc) + pr_err("parcel_reclaim: 0x%llx %d\n", global_id, rc); + + return rc; +} + +/* + * 4. Dynamic shared memory pool based on alloc_pages() + * + * Implements an OP-TEE specific shared memory pool. + * The main function is optee_riscv_shm_pool_alloc_pages(). + */ + +static int pool_riscv_op_alloc(struct tee_shm_pool *pool, + struct tee_shm *shm, size_t size, size_t align) +{ + return tee_dyn_shm_alloc_helper(shm, size, align, + optee_riscv_shm_register); +} + +static void pool_riscv_op_free(struct tee_shm_pool *pool, struct tee_shm *shm) +{ + tee_dyn_shm_free_helper(shm, optee_riscv_shm_unregister); +} + +static void pool_riscv_op_destroy_pool(struct tee_shm_pool *pool) +{ + kfree(pool); +} + +static const struct tee_shm_pool_ops pool_riscv_ops = { + .alloc = pool_riscv_op_alloc, + .free = pool_riscv_op_free, + .destroy_pool = pool_riscv_op_destroy_pool, +}; + +/** + * optee_riscv_shm_pool_alloc_pages() - create page-based allocator pool + * + * This pool is used with OP-TEE over the RPMI TEE service group. In this case + * command buffers and such are allocated from kernel's own memory. + */ +static struct tee_shm_pool *optee_riscv_shm_pool_alloc_pages(void) +{ + struct tee_shm_pool *pool = kzalloc_obj(*pool); + + if (!pool) + return ERR_PTR(-ENOMEM); + + pool->ops = &pool_riscv_ops; + + return pool; +} + /* - * 3. Driver initialization + * 5. Do a normal scheduled call into secure world * + * The function optee_riscv_do_call_with_arg() performs a normal scheduled + * call into secure world. During this call secure world may request help + * from normal world using RPCs, Remote Procedure Calls. This includes + * delivery of non-secure interrupts to for instance allow rescheduling of + * the current task. + */ + +static void handle_riscv_rpc_func_cmd_shm_alloc(struct tee_context *ctx, + struct optee *optee, + struct optee_msg_arg *arg) +{ + struct tee_shm *shm; + + if (arg->num_params != 1 || + arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) { + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + return; + } + + switch (arg->params[0].u.value.a) { + case OPTEE_RPC_SHM_TYPE_APPL: + shm = optee_rpc_cmd_alloc_suppl(ctx, arg->params[0].u.value.b); + break; + case OPTEE_RPC_SHM_TYPE_KERNEL: + shm = tee_shm_alloc_priv_buf(optee->ctx, + arg->params[0].u.value.b); + break; + default: + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + return; + } + + if (IS_ERR(shm)) { + arg->ret = TEEC_ERROR_OUT_OF_MEMORY; + return; + } + + arg->params[0] = (struct optee_msg_param){ + .attr = OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT, + .u.fmem.size = tee_shm_get_size(shm), + .u.fmem.global_id = shm->sec_world_id, + .u.fmem.internal_offs = shm->offset, + }; + + arg->ret = TEEC_SUCCESS; +} + +static void handle_riscv_rpc_func_cmd_shm_free(struct tee_context *ctx, + struct optee *optee, + struct optee_msg_arg *arg) +{ + struct tee_shm *shm; + + if (arg->num_params != 1 || + arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) + goto err_bad_param; + + shm = optee_shm_from_riscv_handle(optee, arg->params[0].u.value.b); + if (!shm) + goto err_bad_param; + switch (arg->params[0].u.value.a) { + case OPTEE_RPC_SHM_TYPE_APPL: + optee_rpc_cmd_free_suppl(ctx, shm); + break; + case OPTEE_RPC_SHM_TYPE_KERNEL: + tee_shm_free(shm); + break; + default: + goto err_bad_param; + } + arg->ret = TEEC_SUCCESS; + return; + +err_bad_param: + arg->ret = TEEC_ERROR_BAD_PARAMETERS; +} + +static void handle_riscv_rpc_func_cmd(struct tee_context *ctx, + struct optee *optee, + struct optee_msg_arg *arg) +{ + arg->ret_origin = TEEC_ORIGIN_COMMS; + switch (arg->cmd) { + case OPTEE_RPC_CMD_SHM_ALLOC: + handle_riscv_rpc_func_cmd_shm_alloc(ctx, optee, arg); + break; + case OPTEE_RPC_CMD_SHM_FREE: + handle_riscv_rpc_func_cmd_shm_free(ctx, optee, arg); + break; + default: + optee_rpc_cmd(ctx, optee, arg); + } +} + +static void optee_handle_riscv_rpc(struct tee_context *ctx, + struct optee *optee, u32 cmd, + struct optee_msg_arg *arg) +{ + switch (cmd) { + case OPTEE_ABI_YIELDING_CALL_RETURN_RPC_CMD: + handle_riscv_rpc_func_cmd(ctx, optee, arg); + break; + case OPTEE_ABI_YIELDING_CALL_RETURN_INTERRUPT: + /* Interrupt delivered by now */ + break; + default: + pr_warn("Unknown RPC func 0x%x\n", cmd); + break; + } +} + +static int optee_riscv_yielding_call(struct tee_context *ctx, + u64 in[RPMI_TEE_OPTEE_CALL_REGS], + struct optee_msg_arg *rpc_arg, + bool system_thread) +{ + struct optee *optee = tee_get_drvdata(ctx->teedev); + struct optee_call_waiter w; + u64 out[RPMI_TEE_OPTEE_RESP_REGS] = { }; + int rc; + + /* Initialize waiter */ + optee_cq_wait_init(&optee->call_queue, &w, system_thread); + while (true) { + rc = optee_riscv_tee_call(optee, in, out); + if (rc) + goto done; + + switch ((int)out[0]) { + case TEEC_SUCCESS: + break; + case TEEC_ERROR_BUSY: + if (in[0] == OPTEE_ABI_YIELDING_CALL_RESUME) { + rc = -EIO; + goto done; + } + + /* + * Out of threads in secure world, wait for a thread + * to become available. + */ + optee_cq_wait_for_completion(&optee->call_queue, &w); + continue; + default: + rc = -EIO; + goto done; + } + + if (out[1] == OPTEE_ABI_YIELDING_CALL_RETURN_DONE) + goto done; + + /* + * OP-TEE has returned with an RPC request. + * + * Note that out[4] (returned in reg[4]) is already filled in + * by optee_riscv_tee_call() returning above. + */ + cond_resched(); + optee_handle_riscv_rpc(ctx, optee, out[1], rpc_arg); + in[0] = OPTEE_ABI_YIELDING_CALL_RESUME; + in[1] = 0; + in[2] = 0; + in[3] = 0; + in[4] = out[4]; /* resume info */ + } +done: + /* + * We're done with our thread in secure world, if there are any + * thread waiters wake up one. + */ + optee_cq_wait_final(&optee->call_queue, &w); + + return rc; +} + +/** + * optee_riscv_do_call_with_arg() - enter OP-TEE in secure world + * @ctx: calling context + * @shm: shared memory holding the message to pass to secure world + * @offs: offset of the message in @shm + * @system_thread: true if caller requests TEE system thread support + * + * Does a TEE_CALL to OP-TEE in secure world and handles the resulting + * Remote Procedure Calls (RPC) from OP-TEE. The struct optee_msg_arg is + * passed by its parcel handle plus @offs, exactly as FF-A passes it by + * shared memory handle. + * + * Returns return code from OP-TEE, 0 is OK + */ +static int optee_riscv_do_call_with_arg(struct tee_context *ctx, + struct tee_shm *shm, u_int offs, + bool system_thread) +{ + u64 in[RPMI_TEE_OPTEE_CALL_REGS] = { + OPTEE_ABI_YIELDING_CALL_WITH_ARG, + (u32)shm->sec_world_id, + shm->sec_world_id >> 32, + offs, + }; + struct optee_msg_arg *arg; + unsigned int rpc_arg_offs; + struct optee_msg_arg *rpc_arg; + + /* + * The shared memory object has to start on a page when passed as + * an argument struct. This is also what the shm pool allocator + * returns, but check this before calling secure world to catch + * eventual errors early in case something changes. + */ + if (shm->offset) + return -EINVAL; + + arg = tee_shm_get_va(shm, offs); + if (IS_ERR(arg)) + return PTR_ERR(arg); + + rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params); + rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs); + if (IS_ERR(rpc_arg)) + return PTR_ERR(rpc_arg); + + return optee_riscv_yielding_call(ctx, in, rpc_arg, system_thread); +} + +/* + * 6. Driver initialization + * + * During driver initialization the OP-TEE Trusted OS is probed over TEE_CALL + * to find out which features it supports so the driver can be initialized + * with a matching configuration. These blocking calls mirror the FF-A + * OPTEE_FFA_GET_API_VERSION / GET_OS_VERSION / EXCHANGE_CAPABILITIES probes. + */ + +static bool optee_riscv_api_is_compatible(struct optee *optee) +{ + u64 in[RPMI_TEE_OPTEE_CALL_REGS] = { OPTEE_ABI_GET_API_VERSION }; + u64 out[RPMI_TEE_OPTEE_RESP_REGS] = { }; + int rc; + + rc = optee_riscv_tee_call(optee, in, out); + if (rc) { + pr_err("Unexpected error %d\n", rc); + return false; + } + if (out[0] != OPTEE_ABI_VERSION_MAJOR || + out[1] < OPTEE_ABI_VERSION_MINOR) { + pr_err("Incompatible OP-TEE API version %llu.%llu\n", + out[0], out[1]); + return false; + } + + return true; +} + +static bool optee_riscv_get_os_revision(struct optee *optee) +{ + u64 in[RPMI_TEE_OPTEE_CALL_REGS] = { OPTEE_ABI_GET_OS_VERSION }; + u64 out[RPMI_TEE_OPTEE_RESP_REGS] = { }; + int rc; + + rc = optee_riscv_tee_call(optee, in, out); + if (rc) { + pr_err("Unexpected error %d\n", rc); + return false; + } + + optee->revision.os_major = out[0]; + optee->revision.os_minor = out[1]; + optee->revision.os_build_id = out[2]; + + if (out[2]) + pr_info("revision %llu.%llu (%08llx)\n", out[0], out[1], + out[2]); + else + pr_info("revision %llu.%llu\n", out[0], out[1]); + + return true; +} + +static bool optee_riscv_exchange_caps(struct optee *optee, u32 *sec_caps, + unsigned int *rpc_param_count, + unsigned int *max_notif_value) +{ + u64 in[RPMI_TEE_OPTEE_CALL_REGS] = { OPTEE_ABI_EXCHANGE_CAPABILITIES }; + u64 out[RPMI_TEE_OPTEE_RESP_REGS] = { }; + int rc; + + rc = optee_riscv_tee_call(optee, in, out); + if (rc) { + pr_err("Unexpected error %d\n", rc); + return false; + } + if (out[0]) { + pr_err("Unexpected exchange error %llu\n", out[0]); + return false; + } + + *rpc_param_count = (u8)out[1]; + *sec_caps = out[2]; + if (out[3]) + *max_notif_value = out[3]; + else + *max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE; + + return true; +} + +static void optee_riscv_get_version(struct tee_device *teedev, + struct tee_ioctl_version_data *vers) +{ + struct tee_ioctl_version_data v = { + .impl_id = TEE_IMPL_ID_OPTEE, + .impl_caps = TEE_OPTEE_CAP_TZ, + .gen_caps = TEE_GEN_CAP_GP | TEE_GEN_CAP_REG_MEM | + TEE_GEN_CAP_MEMREF_NULL, + }; + + *vers = v; +} + +static int optee_riscv_open(struct tee_context *ctx) +{ + return optee_open(ctx, true); +} + +static const struct tee_driver_ops optee_riscv_clnt_ops = { + .get_version = optee_riscv_get_version, + .get_tee_revision = optee_get_revision, + .open = optee_riscv_open, + .release = optee_release, + .open_session = optee_open_session, + .close_session = optee_close_session, + .invoke_func = optee_invoke_func, + .cancel_req = optee_cancel_req, + .shm_register = optee_riscv_shm_register, + .shm_unregister = optee_riscv_shm_unregister, +}; + +static const struct tee_desc optee_riscv_clnt_desc = { + .name = DRIVER_NAME "-riscv-clnt", + .ops = &optee_riscv_clnt_ops, + .owner = THIS_MODULE, +}; + +static const struct tee_driver_ops optee_riscv_supp_ops = { + .get_version = optee_riscv_get_version, + .get_tee_revision = optee_get_revision, + .open = optee_riscv_open, + .release = optee_release_supp, + .supp_recv = optee_supp_recv, + .supp_send = optee_supp_send, + .shm_register = optee_riscv_shm_register, /* same as for clnt ops */ + .shm_unregister = optee_riscv_shm_unregister_supp, +}; + +static const struct tee_desc optee_riscv_supp_desc = { + .name = DRIVER_NAME "-riscv-supp", + .ops = &optee_riscv_supp_ops, + .owner = THIS_MODULE, + .flags = TEE_DESC_PRIVILEGED, +}; + +static const struct optee_ops optee_riscv_ops = { + .do_call_with_arg = optee_riscv_do_call_with_arg, + .to_msg_param = optee_riscv_to_msg_param, + .from_msg_param = optee_riscv_from_msg_param, +}; + +/* * The RPMI TEE service group is described in the device tree by a single * node whose "mboxes" property lists one SBI MPXY channel per hart, in hart * order. The driver requests each list entry by index and validates the @@ -186,15 +1042,117 @@ static void optee_riscv_free_channels(struct optee *optee) } }

+/* Confirm the TEE service group and read its transport attributes. */ +static int optee_riscv_check_transport(struct optee *optee) +{ + struct device *dev = optee->riscv.dev; + struct rpmi_mbox_message msg; + int ret; + + rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_SERVICEGROUP_ID); + ret = optee_riscv_send(optee, &msg); + if (ret) + return dev_err_probe(dev, ret, + "Failed to get service group id\n"); + if (msg.attr.value != RPMI_SRVGRP_TEE) + return dev_err_probe(dev, -ENODEV, + "Not a TEE service group channel (0x%x)\n", + msg.attr.value); + + rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_MAX_MSG_DATA_SIZE); + ret = optee_riscv_send(optee, &msg); + if (ret) + return dev_err_probe(dev, ret, + "Failed to get max msg data size\n"); + optee->riscv.max_msg_data_size = msg.attr.value; + + return 0; +} + +/* + * TEE_PROBE_FEATURES (0x02) reports which framework features are available; + * memory parcels carry normal-world shared memory to OP-TEE, so the framework + * must support sharing memory between the REE and a TEE. + */ +static int optee_riscv_probe_feature(struct optee *optee, u32 feature_id, + u32 *value) +{ + struct rpmi_tee_probe_features_req tx = { + .feature_id = cpu_to_le32(feature_id), + }; + struct rpmi_tee_probe_features_resp rx = { }; + struct rpmi_mbox_message msg; + int ret; + + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_PROBE_FEATURES, + &tx, sizeof(tx), &rx, sizeof(rx)); + ret = optee_riscv_send(optee, &msg); + if (ret) + return ret; + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + if (value) + *value = le32_to_cpu(rx.value); + + return 0; +} + +static int optee_riscv_features(struct optee *optee) +{ + u32 share = RPMI_TEE_MEMORY_SHARE_NONE; + int ret; + + ret = optee_riscv_probe_feature(optee, RPMI_TEE_FEAT_MEMORY_SHARE, + &share); + if (ret) { + pr_err("Failed to probe MEMORY_SHARE feature: %d\n", ret); + return ret; + } + if (share != RPMI_TEE_MEMORY_SHARE_FULL) { + pr_err("Framework cannot share memory between REE and TEE (%u)\n", + share); + return -EOPNOTSUPP; + } + + return 0; +} + +static int optee_riscv_enable_notif(struct optee *optee) +{ + struct rpmi_tee_probe_features_resp rx = { }; + struct rpmi_mbox_message msg; + int ret; + + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_ENABLE_NOTIFICATION, + NULL, 0, &rx, sizeof(rx)); + ret = optee_riscv_send(optee, &msg); + if (ret) + return ret; + + /* + * The TEE service group defines no notification events on this + * platform, so RPMI_ERR_NOTSUPP is expected and not fatal. + */ + if (rx.status && le32_to_cpu(rx.status) != (u32)RPMI_ERR_NOTSUPP) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + return 0; +} + static int optee_riscv_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct rpmi_mbox_message msg; + unsigned int rpc_param_count; + unsigned int max_notif_value; + struct tee_shm_pool *pool; + struct tee_device *teedev; + struct tee_context *ctx; struct mbox_client *client; struct optee *optee; - u32 servicegroup_id; + u32 sec_caps; unsigned int nr_cpus; - int ret; + int rc;

nr_cpus = num_possible_cpus(); if (!nr_cpus) @@ -206,7 +1164,7 @@ static int optee_riscv_probe(struct platform_device *pdev)

client = devm_kzalloc(dev, sizeof(*client), GFP_KERNEL); if (!client) { - ret = -ENOMEM; + rc = -ENOMEM; goto err_free_optee; } client->dev = dev; @@ -221,66 +1179,156 @@ static int optee_riscv_probe(struct platform_device *pdev) optee->riscv.chan = kcalloc(nr_cpus, sizeof(*optee->riscv.chan), GFP_KERNEL); if (!optee->riscv.chan) { - ret = -ENOMEM; + rc = -ENOMEM; goto err_free_optee; }

- ret = optee_riscv_request_channels(optee); - if (ret) + rc = optee_riscv_request_channels(optee); + if (rc) goto err_free_channels;

- /* Confirm the channel really speaks the TEE service group. */ - rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_SERVICEGROUP_ID); - ret = optee_riscv_send(optee, &msg); - if (ret) { - dev_err_probe(dev, ret, "Failed to get service group id\n"); + rc = optee_riscv_check_transport(optee); + if (rc) + goto err_free_channels; + + rc = optee_riscv_features(optee); + if (rc) { + dev_err_probe(dev, rc, "Missing required TEE features\n"); goto err_free_channels; } - servicegroup_id = msg.attr.value; - if (servicegroup_id != RPMI_SRVGRP_TEE) { - ret = -ENODEV; - dev_err_probe(dev, ret, "Not a TEE service group channel (0x%x)\n", - servicegroup_id); + + rc = optee_riscv_enable_notif(optee); + if (rc) { + dev_err_probe(dev, rc, "Failed to enable notifications\n"); goto err_free_channels; }

- rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_MAX_MSG_DATA_SIZE); - ret = optee_riscv_send(optee, &msg); - if (ret) { - dev_err_probe(dev, ret, "Failed to get max msg data size\n"); + if (!optee_riscv_api_is_compatible(optee)) { + rc = -EINVAL; goto err_free_channels; } - optee->riscv.max_msg_data_size = msg.attr.value;

- ret = optee_riscv_features(optee); - if (ret) { - dev_err_probe(dev, ret, "Missing required TEE features\n"); + if (!optee_riscv_get_os_revision(optee)) { + rc = -EINVAL; goto err_free_channels; }

- ret = optee_riscv_enable_notif(optee); - if (ret) { - dev_err_probe(dev, ret, "Failed to enable notifications\n"); + if (!optee_riscv_exchange_caps(optee, &sec_caps, &rpc_param_count, + &max_notif_value)) { + rc = -EINVAL; + goto err_free_channels; + } + + pool = optee_riscv_shm_pool_alloc_pages(); + if (IS_ERR(pool)) { + rc = PTR_ERR(pool); goto err_free_channels; } + optee->pool = pool; + + optee->ops = &optee_riscv_ops; + optee->rpc_param_count = rpc_param_count; + + if (IS_REACHABLE(CONFIG_RPMB) && + (sec_caps & OPTEE_ABI_SEC_CAP_RPMB_PROBE)) + optee->in_kernel_rpmb_routing = true; + + teedev = tee_device_alloc(&optee_riscv_clnt_desc, NULL, optee->pool, + optee); + if (IS_ERR(teedev)) { + rc = PTR_ERR(teedev); + goto err_free_shm_pool; + } + optee->teedev = teedev; + + teedev = tee_device_alloc(&optee_riscv_supp_desc, NULL, optee->pool, + optee); + if (IS_ERR(teedev)) { + rc = PTR_ERR(teedev); + goto err_unreg_teedev; + } + optee->supp_teedev = teedev;

+ optee_set_dev_group(optee); + + rc = tee_device_register(optee->teedev); + if (rc) + goto err_unreg_supp_teedev; + + rc = tee_device_register(optee->supp_teedev); + if (rc) + goto err_unreg_supp_teedev; + + rc = rhashtable_init(&optee->riscv.global_ids, &shm_rhash_params); + if (rc) + goto err_unreg_supp_teedev; + mutex_init(&optee->riscv.mutex); + atomic_set(&optee->riscv.next_nonce, 0); + optee_cq_init(&optee->call_queue, 0); + optee_supp_init(&optee->supp); + optee_shm_arg_cache_init(optee, 0); + mutex_init(&optee->rpmb_dev_mutex); platform_set_drvdata(pdev, optee); + + ctx = teedev_open(optee->teedev); + if (IS_ERR(ctx)) { + rc = PTR_ERR(ctx); + goto err_rhashtable_free; + } + optee->ctx = ctx; + + rc = optee_notif_init(optee, max_notif_value); + if (rc) + goto err_close_ctx; + + rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES); + if (rc) + goto err_unregister_devices; + + INIT_WORK(&optee->rpmb_scan_bus_work, optee_bus_scan_rpmb); + optee->rpmb_intf.notifier_call = optee_rpmb_intf_rdev; + blocking_notifier_chain_register(&optee_rpmb_intf_added, + &optee->rpmb_intf); + dev_info(dev, "initialized driver\n");

return 0;

+err_unregister_devices: + optee_unregister_devices(); + optee_notif_uninit(optee); +err_close_ctx: + teedev_close_context(ctx); +err_rhashtable_free: + rhashtable_free_and_destroy(&optee->riscv.global_ids, rh_free_fn, NULL); + rpmb_dev_put(optee->rpmb_dev); + mutex_destroy(&optee->rpmb_dev_mutex); + optee_supp_uninit(&optee->supp); + mutex_destroy(&optee->call_queue.mutex); + mutex_destroy(&optee->riscv.mutex); +err_unreg_supp_teedev: + tee_device_unregister(optee->supp_teedev); +err_unreg_teedev: + tee_device_unregister(optee->teedev); +err_free_shm_pool: + tee_shm_pool_free(pool); err_free_channels: optee_riscv_free_channels(optee); kfree(optee->riscv.chan); err_free_optee: kfree(optee); - return ret; + return rc; }

static void optee_riscv_remove(struct platform_device *pdev) { struct optee *optee = platform_get_drvdata(pdev);

+ optee_remove_common(optee); + + mutex_destroy(&optee->riscv.mutex); + rhashtable_free_and_destroy(&optee->riscv.global_ids, rh_free_fn, NULL); + optee_riscv_free_channels(optee); kfree(optee->riscv.chan); kfree(optee); diff --git a/drivers/tee/optee/optee_riscv.h b/drivers/tee/optee/optee_riscv.h index d87298faa6a2..2cdbb1fb4eab 100644 --- a/drivers/tee/optee/optee_riscv.h +++ b/drivers/tee/optee/optee_riscv.h @@ -24,6 +24,7 @@ #ifndef __OPTEE_RISCV_H #define __OPTEE_RISCV_H

+#include <linux/bits.h> #include <linux/mailbox/riscv-rpmi-message.h> #include <linux/types.h>

@@ -110,9 +111,13 @@ struct rpmi_tee_probe_features_resp { { 0x5b, 0xe1, 0xb1, 0xa0, 0x7e, 0x11, 0x4e, 0x7a, \ 0x9b, 0x10, 0x00, 0x10, 0xc0, 0xff, 0xee, 0x00 }

-/* OP-TEE SMC-style call convention carried inside SERVICE_DATA. */ -#define RPMI_TEE_OPTEE_CALL_REGS 8 /* a0-a7 */ -#define RPMI_TEE_OPTEE_RESP_REGS 4 /* a0-a3 */ +/* + * OP-TEE FF-A direct message convention carried inside SERVICE_DATA: + * five command words each way, the RISC-V analog of the FF-A data0-data4 + * (w3-w7) set of struct ffa_send_direct_data. + */ +#define RPMI_TEE_OPTEE_CALL_REGS 5 +#define RPMI_TEE_OPTEE_RESP_REGS 5

#if __riscv_xlen == 64 typedef __le64 rpmi_xlen_t; @@ -138,4 +143,128 @@ struct rpmi_tee_call_resp { rpmi_xlen_t reg[RPMI_TEE_OPTEE_RESP_REGS]; } __packed;

+/* + * OP-TEE message ABI carried inside the TEE_CALL SERVICE_DATA words. + * + * This mirrors the FF-A message ABI in <optee_ffa.h>: OP-TEE and the REE are + * peer endpoints and the argument struct optee_msg_arg is passed by shared + * memory handle (a parcel id) plus an offset, never by a register block. The + * SERVICE_DATA registers carry a small command word set that is the RISC-V + * analog of the FF-A w3-w7 register usage: + * + * reg[0]: command / service id (OPTEE_ABI_YIELDING_CALL_* below) + * reg[1]: shared memory handle, lower 32 bits (parcel id) + * reg[2]: shared memory handle, upper 32 bits (parcel nonce) + * reg[3]: offset into the shared memory to the struct optee_msg_arg + * reg[4]: not used on this call, resume info on OPTEE_ABI_YIELDING_CALL_RESUME + * + * On return the SERVICE_RSP registers carry: + * reg[0]: error code, 0 on success + * reg[1]: return code (OPTEE_ABI_YIELDING_CALL_RETURN_* below) + * reg[2..3]: not used + * reg[4]: RPC resume info + * + * These MUST byte-match the secure world OP-TEE header. + */ +#define OPTEE_ABI_BLOCKING_CALL(id) (id) +#define OPTEE_ABI_YIELDING_CALL_BIT 31 +#define OPTEE_ABI_YIELDING_CALL(id) ((id) | BIT(OPTEE_ABI_YIELDING_CALL_BIT)) + +/* Blocking (fast) calls, mirror of OPTEE_FFA_BLOCKING_CALL ids. */ +#define OPTEE_ABI_GET_API_VERSION OPTEE_ABI_BLOCKING_CALL(0) +#define OPTEE_ABI_GET_OS_VERSION OPTEE_ABI_BLOCKING_CALL(1) +#define OPTEE_ABI_EXCHANGE_CAPABILITIES OPTEE_ABI_BLOCKING_CALL(2) +#define OPTEE_ABI_UNREGISTER_SHM OPTEE_ABI_BLOCKING_CALL(3) +#define OPTEE_ABI_ENABLE_ASYNC_NOTIF OPTEE_ABI_BLOCKING_CALL(5) + +/* OP-TEE ABI version, mirror of OPTEE_FFA_VERSION_*. */ +#define OPTEE_ABI_VERSION_MAJOR 1 +#define OPTEE_ABI_VERSION_MINOR 0 + +/* Capabilities returned by EXCHANGE_CAPABILITIES (OPTEE_FFA_SEC_CAP_* analog). */ +#define OPTEE_ABI_SEC_CAP_ARG_OFFSET BIT(0) +#define OPTEE_ABI_SEC_CAP_ASYNC_NOTIF BIT(1) +#define OPTEE_ABI_SEC_CAP_RPMB_PROBE BIT(2) + +#define OPTEE_ABI_MAX_ASYNC_NOTIF_VALUE 64 + +/* Yielding calls, mirror of OPTEE_FFA_YIELDING_CALL_*. */ +#define OPTEE_ABI_YIELDING_CALL_WITH_ARG OPTEE_ABI_YIELDING_CALL(0) +#define OPTEE_ABI_YIELDING_CALL_RESUME OPTEE_ABI_YIELDING_CALL(1) + +#define OPTEE_ABI_YIELDING_CALL_RETURN_DONE 0 +#define OPTEE_ABI_YIELDING_CALL_RETURN_RPC_CMD 1 +#define OPTEE_ABI_YIELDING_CALL_RETURN_INTERRUPT 2 + +/* + * Memory parcel wire encodings (RPMI spec section 4.16, Tables 198-207). + * + * A memory parcel is the RISC-V analog of an FF-A memory-share handle: the REE + * creates a parcel describing its pages and OP-TEE accepts it lazily by parcel + * id. All fields are little-endian uint32 words; block-list addresses are + * expressed in units of 4kB pages. + */ + +/* Memory access encoding (Table 199). */ +#define RPMI_TEE_PARCEL_ACCESS_R BIT(29) +#define RPMI_TEE_PARCEL_ACCESS_W BIT(30) +#define RPMI_TEE_PARCEL_ACCESS_X BIT(31) + +/* MEM_PARCEL_CREATE flags (Table 200). */ +#define RPMI_TEE_PARCEL_CREATE_FLAG_MULTI_SEGMENT BIT(31) +#define RPMI_TEE_PARCEL_CREATE_FLAG_OWNER_XFER BIT(30) + +/* Length of the parcel LABEL field (Table 200). */ +#define RPMI_TEE_PARCEL_LABEL_LEN 16 + +/* + * A block list entry covers a run of physically contiguous 4kB pages + * (Table 198): + * BLOCK_HIGH = page-frame number [51:20] + * BLOCK_LOW = (page-frame number [19:0] << 12) | (page count - 1) + * so a single block spans at most 4096 pages (16MB). + */ +#define RPMI_TEE_PARCEL_BLOCK_MAX_PAGES 4096 + +static inline __le32 rpmi_tee_block_high(u64 pfn) +{ + return cpu_to_le32((u32)(pfn >> 20)); +} + +static inline __le32 rpmi_tee_block_low(u64 pfn, u32 npages) +{ + return cpu_to_le32(((u32)(pfn & 0xfffff) << 12) | (npages - 1)); +} + +/* + * MEM_PARCEL_CREATE request (Table 200): a fixed header followed by + * receiver_id[receiver_cnt], access[receiver_cnt], block_high[block_cnt] and + * block_low[block_cnt]. + */ +struct rpmi_tee_mem_parcel_create_req { + __le32 creator_id; + __le32 creator_access; + __le32 receiver_cnt; + __le32 flags; + __le32 nonce; + __le32 block_cnt; + u8 label[RPMI_TEE_PARCEL_LABEL_LEN]; + __le32 data[]; +}; + +struct rpmi_tee_mem_parcel_create_resp { + __le32 status; + __le32 mem_parcel_id; +}; + +/* MEM_PARCEL_RECLAIM request (Table 206) / response (Table 207). */ +struct rpmi_tee_mem_parcel_reclaim_req { + __le32 mem_parcel_id; +}; + +struct rpmi_tee_mem_parcel_reclaim_resp { + __le32 status; + __le32 flags; +}; + #endif /* __OPTEE_RISCV_H */

Add asynchronous notification support using the RPMI TEE signal bus.

Set up a signal bus between OP-TEE and Linux and use the platform interrupt as the availability doorbell. When signalled, retrieve pending signals and pass normal notification values to optee_notif_send(). Reserve one signal value for requesting the OP-TEE bottom-half handler.

Enable this only when secure world advertises asynchronous notification support.

Signed-off-by: Amirreza Zarrabi amirreza.zarrabi@oss.qualcomm.com --- drivers/tee/optee/optee_private.h | 11 ++ drivers/tee/optee/optee_riscv.c | 251 ++++++++++++++++++++++++++++++++++++++ drivers/tee/optee/optee_riscv.h | 64 ++++++++++ 3 files changed, 326 insertions(+)

diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h index cf878b8178f9..2ec53bd330a1 100644 --- a/drivers/tee/optee/optee_private.h +++ b/drivers/tee/optee/optee_private.h @@ -181,6 +181,12 @@ struct optee_ffa { * @next_nonce: monotonic nonce source for memory parcel creation * @mutex: serializes access to @global_ids * @global_ids: memory parcel id to tee_shm translation table + * @notif_wq: workqueue for signal-bus asynchronous notification + * @notif_work: work for signal-bus asynchronous notification + * @signal_irq: availability doorbell IRQ, or 0 if async notif unused + * @sender_signals: number of signals OP-TEE may raise to the REE + * @bottom_half_value: signal value that requests an RPC bottom half, or + * U32_MAX if async notif is unused * * This is the RISC-V analog of struct optee_ffa: communication with secure * world OP-TEE OS rides the RPMI TEE service group (RPMI spec section 4.16) @@ -196,6 +202,11 @@ struct optee_riscv { /* Serializes access to @global_ids */ struct mutex mutex; struct rhashtable global_ids; + struct workqueue_struct *notif_wq; + struct work_struct notif_work; + unsigned int signal_irq; + u32 sender_signals; + u32 bottom_half_value; };

struct optee; diff --git a/drivers/tee/optee/optee_riscv.c b/drivers/tee/optee/optee_riscv.c index 72c9eb85f4fe..b9bb813adef5 100644 --- a/drivers/tee/optee/optee_riscv.c +++ b/drivers/tee/optee/optee_riscv.c @@ -28,6 +28,7 @@

#include <linux/atomic.h> #include <linux/errno.h> +#include <linux/interrupt.h> #include <linux/mailbox_client.h> #include <linux/mailbox/riscv-rpmi-message.h> #include <linux/mm.h> @@ -46,6 +47,9 @@ #include "optee_riscv.h" #include "optee_rpc_cmd.h"

+static int optee_riscv_probe_feature(struct optee *optee, u32 feature_id, + u32 *value); + /* * Low level RPMI TEE service group transport over the SBI MPXY mailbox. * @@ -852,6 +856,242 @@ static int optee_riscv_do_call_with_arg(struct tee_context *ctx, return optee_riscv_yielding_call(ctx, in, rpc_arg, system_thread); }

+/* + * 5b. Asynchronous notification over the signal bus + * + * The TEE service group defines no framework notification events (RPMI spec + * section 4.16.2), so OP-TEE signals the REE asynchronously over the signal + * bus (services 0x05-0x08). This is the RISC-V analog of the FF-A + * notification path (optee_ffa_async_notif_init / notif_callback): OP-TEE + * raises a signal, the framework rings an availability doorbell delivered as + * a System MSI or System IRQ, and the REE retrieves the pending signals with + * TEE_SIGNAL_RETRIEVE. A retrieved signal value is the OP-TEE async + * notification key; the reserved top value requests an RPC bottom half. + */ + +static void notif_work_fn(struct work_struct *work) +{ + struct optee_riscv *optee_riscv = container_of(work, struct optee_riscv, + notif_work); + struct optee *optee = container_of(optee_riscv, struct optee, riscv); + + optee_do_bottom_half(optee->ctx); +} + +/* + * Drain all pending signals from the framework and dispatch them. Returns + * true if an RPC bottom half was requested by OP-TEE. TEE_SIGNAL_RETRIEVE + * returns the signals of one bus per call and sets MORE_AVAILABLE while other + * buses still have pending signals, so loop until it is clear. + */ +static bool optee_riscv_retrieve_signals(struct optee *optee) +{ + bool do_bottom_half = false; + size_t max_signals = optee->riscv.sender_signals; + struct rpmi_tee_signal_retrieve_resp *rx; + struct rpmi_mbox_message msg; + size_t rx_len; + u32 flags; + + rx_len = struct_size(rx, signal, max_signals); + rx = kzalloc(rx_len, GFP_KERNEL); + if (!rx) + return false; + + do { + u32 status, n, i; + + rpmi_mbox_init_send_with_response(&msg, + RPMI_TEE_SRV_SIGNAL_RETRIEVE, + NULL, 0, rx, rx_len); + if (optee_riscv_send(optee, &msg)) + break; + + status = le32_to_cpu(rx->status); + if (status == (u32)RPMI_ERR_NO_DATA) + break; + if (status) + break; + + n = min_t(u32, le32_to_cpu(rx->signal_len), max_signals); + for (i = 0; i < n; i++) { + u32 value = le32_to_cpu(rx->signal[i]); + + if (value == OPTEE_ABI_ASYNC_NOTIF_BOTTOM_HALF) + do_bottom_half = true; + else + optee_notif_send(optee, value); + } + + flags = le32_to_cpu(rx->flags); + } while (flags & RPMI_TEE_SIGNAL_RETRIEVE_MORE_AVAILABLE); + + kfree(rx); + + return do_bottom_half; +} + +static irqreturn_t notif_irq_handler(int irq, void *dev_id) +{ + struct optee *optee = dev_id; + + if (optee_riscv_retrieve_signals(optee)) + queue_work(optee->riscv.notif_wq, &optee->riscv.notif_work); + + return IRQ_HANDLED; +} + +/* + * Arm the OP-TEE asynchronous notification subsystem (OPTEE_ABI_ENABLE_ASYNC_NOTIF + * blocking call, the mirror of FF-A's OPTEE_FFA_ENABLE_ASYNC_NOTIF). The reserved + * bottom-half signal value is handed to OP-TEE so that a raise of that value is + * understood as a request to run the driver bottom half rather than as a plain + * notification key. + */ +static int optee_riscv_enable_async_notif(struct optee *optee) +{ + u64 in[RPMI_TEE_OPTEE_CALL_REGS] = { OPTEE_ABI_ENABLE_ASYNC_NOTIF, + optee->riscv.bottom_half_value }; + u64 out[RPMI_TEE_OPTEE_RESP_REGS] = { }; + int rc; + + rc = optee_riscv_tee_call(optee, in, out); + if (rc) + return rc; + if (out[0]) + return -EINVAL; + + return 0; +} + +/* + * Set up the signal bus with OP-TEE (TEE_SIGNAL_BUS_SETUP, service 0x05) and + * request the availability doorbell IRQ. The bus must be set up by the REE + * (RPMI spec section 4.16.7) and is sized so every OP-TEE async notification + * value, plus the reserved bottom-half value, maps to a distinct signal that + * OP-TEE may raise. + */ +static int optee_riscv_setup_signal_bus(struct optee *optee) +{ + struct rpmi_tee_signal_bus_setup_req tx = { + .target_id = cpu_to_le32(RPMI_TEE_ENDPOINT_OPTEE), + .bus_width = cpu_to_le32(OPTEE_ABI_ASYNC_NOTIF_BUS_WIDTH), + /* + * SENDER_SIGNALS (RPMI spec Table 190) is the number of signals + * reserved for us, the sender, to receive: signals 0 <= x < N + * are raised by the target (OP-TEE) and read by us. We only + * ever receive notifications from OP-TEE and never raise any, so + * reserve the whole bus for OP-TEE to raise. + */ + .sender_signals = cpu_to_le32(OPTEE_ABI_ASYNC_NOTIF_BUS_WIDTH), + }; + struct rpmi_tee_signal_bus_setup_resp rx = { }; + struct rpmi_mbox_message msg; + int ret; + + rpmi_mbox_init_send_with_response(&msg, RPMI_TEE_SRV_SIGNAL_BUS_SETUP, + &tx, sizeof(tx), &rx, sizeof(rx)); + ret = optee_riscv_send(optee, &msg); + if (ret) + return ret; + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + return 0; +} + +static void optee_riscv_teardown_signal_bus(struct optee *optee) +{ + struct rpmi_tee_signal_bus_teardown_req tx = { + .target_id = cpu_to_le32(RPMI_TEE_ENDPOINT_OPTEE), + }; + struct rpmi_tee_signal_bus_teardown_resp rx = { }; + struct rpmi_mbox_message msg; + + rpmi_mbox_init_send_with_response(&msg, + RPMI_TEE_SRV_SIGNAL_BUS_TEARDOWN, + &tx, sizeof(tx), &rx, sizeof(rx)); + optee_riscv_send(optee, &msg); +} + +/* + * Discover and enable asynchronous notification. Probe the SIGNAL_BUS + * feature word: bits [1:0] give the doorbell transport (System MSI or System + * IRQ), [11:2] the maximum bus width and [31:12] the doorbell index. On this + * platform the doorbell is wired to the platform device as its interrupt, so + * the index is resolved through the DT and requested with platform_get_irq(). + */ +static int optee_riscv_async_notif_init(struct platform_device *pdev, + struct optee *optee) +{ + u32 feat = 0; + int irq, rc; + + rc = optee_riscv_probe_feature(optee, RPMI_TEE_FEAT_SIGNAL_BUS, &feat); + if (rc) + return rc; + + if (RPMI_TEE_SIGNAL_BUS_TRANSPORT(feat) == RPMI_TEE_SIGNAL_BUS_NONE) + return -EOPNOTSUPP; + if (RPMI_TEE_SIGNAL_BUS_WIDTH(feat) < OPTEE_ABI_ASYNC_NOTIF_BUS_WIDTH) + return -EOPNOTSUPP; + + irq = platform_get_irq_optional(pdev, 0); + if (irq < 0) + return irq; + + INIT_WORK(&optee->riscv.notif_work, notif_work_fn); + optee->riscv.notif_wq = create_workqueue("optee_notification"); + if (!optee->riscv.notif_wq) { + rc = -ENOMEM; + goto err; + } + + optee->riscv.sender_signals = OPTEE_ABI_ASYNC_NOTIF_BUS_WIDTH; + + rc = optee_riscv_setup_signal_bus(optee); + if (rc) + goto err_wq; + + rc = request_threaded_irq(irq, NULL, notif_irq_handler, IRQF_ONESHOT, + "optee_notification", optee); + if (rc) + goto err_bus; + optee->riscv.signal_irq = irq; + optee->riscv.bottom_half_value = OPTEE_ABI_ASYNC_NOTIF_BOTTOM_HALF; + + rc = optee_riscv_enable_async_notif(optee); + if (rc) + goto err_irq; + + return 0; + +err_irq: + free_irq(irq, optee); + optee->riscv.signal_irq = 0; +err_bus: + optee_riscv_teardown_signal_bus(optee); +err_wq: + destroy_workqueue(optee->riscv.notif_wq); + optee->riscv.notif_wq = NULL; +err: + optee->riscv.sender_signals = 0; + optee->riscv.bottom_half_value = U32_MAX; + + return rc; +} + +static void optee_riscv_async_notif_uninit(struct optee *optee) +{ + if (optee->riscv.bottom_half_value == U32_MAX) + return; + + free_irq(optee->riscv.signal_irq, optee); + optee_riscv_teardown_signal_bus(optee); + destroy_workqueue(optee->riscv.notif_wq); + optee->riscv.notif_wq = NULL; +} + /* * 6. Driver initialization * @@ -1238,6 +1478,7 @@ static int optee_riscv_probe(struct platform_device *pdev)

optee->ops = &optee_riscv_ops; optee->rpc_param_count = rpc_param_count; + optee->riscv.bottom_half_value = U32_MAX;

if (IS_REACHABLE(CONFIG_RPMB) && (sec_caps & OPTEE_ABI_SEC_CAP_RPMB_PROBE)) @@ -1291,6 +1532,13 @@ static int optee_riscv_probe(struct platform_device *pdev) if (rc) goto err_close_ctx;

+ if (sec_caps & OPTEE_ABI_SEC_CAP_ASYNC_NOTIF) { + rc = optee_riscv_async_notif_init(pdev, optee); + if (rc) + dev_warn(dev, "Failed to initialize async notifications: %d\n", + rc); + } + rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES); if (rc) goto err_unregister_devices; @@ -1306,6 +1554,7 @@ static int optee_riscv_probe(struct platform_device *pdev)

err_unregister_devices: optee_unregister_devices(); + optee_riscv_async_notif_uninit(optee); optee_notif_uninit(optee); err_close_ctx: teedev_close_context(ctx); @@ -1334,6 +1583,8 @@ static void optee_riscv_remove(struct platform_device *pdev) { struct optee *optee = platform_get_drvdata(pdev);

+ optee_riscv_async_notif_uninit(optee); + optee_remove_common(optee);

mutex_destroy(&optee->riscv.mutex); diff --git a/drivers/tee/optee/optee_riscv.h b/drivers/tee/optee/optee_riscv.h index 2cdbb1fb4eab..33a714b79f98 100644 --- a/drivers/tee/optee/optee_riscv.h +++ b/drivers/tee/optee/optee_riscv.h @@ -82,6 +82,70 @@ enum rpmi_tee_feature_id { #define RPMI_TEE_MEMORY_SHARE_TEE_ONLY 1 #define RPMI_TEE_MEMORY_SHARE_FULL 2

+/* + * SIGNAL_BUS feature word encoding (RPMI spec Table 182, feature id 4). + * + * The TEE service group defines no framework notification events + * (RPMI spec section 4.16.2), so asynchronous notification from OP-TEE rides + * the signal bus (services 0x05-0x08) instead. The SIGNAL_BUS feature word + * describes both how the availability doorbell is delivered and the shape of + * the bus: + * [1:0] transport: 0 unsupported, 1 System MSI, 2 System IRQ + * [11:2] maximum bus width (number of signals per endpoint pair) + * [31:12] System MSI index or System IRQ index of the availability doorbell + */ +#define RPMI_TEE_SIGNAL_BUS_TRANSPORT(v) ((v) & GENMASK(1, 0)) +#define RPMI_TEE_SIGNAL_BUS_WIDTH(v) (((v) & GENMASK(11, 2)) >> 2) +#define RPMI_TEE_SIGNAL_BUS_INDEX(v) (((v) & GENMASK(31, 12)) >> 12) + +#define RPMI_TEE_SIGNAL_BUS_NONE 0 +#define RPMI_TEE_SIGNAL_BUS_MSI 1 +#define RPMI_TEE_SIGNAL_BUS_SYSIRQ 2 + +/* + * Signal bus wire encodings (RPMI spec section 4.16.7-4.16.10, + * Tables 190-197). The bus is always set up by the REE; signals + * 0 <= x < sender_signals are raised by the target (OP-TEE) and read by us. + */ +struct rpmi_tee_signal_bus_setup_req { + __le32 target_id; + __le32 bus_width; + __le32 sender_signals; +}; + +struct rpmi_tee_signal_bus_setup_resp { + __le32 status; +}; + +struct rpmi_tee_signal_bus_teardown_req { + __le32 target_id; +}; + +struct rpmi_tee_signal_bus_teardown_resp { + __le32 status; +}; + +/* TEE_SIGNAL_RETRIEVE response (Table 197); request carries no data. */ +#define RPMI_TEE_SIGNAL_RETRIEVE_MORE_AVAILABLE BIT(31) + +struct rpmi_tee_signal_retrieve_resp { + __le32 status; + __le32 flags; + __le32 target_id; + __le32 signal_len; + __le32 signal[]; +}; + +/* + * Asynchronous notification signal assignment (frozen contract with secure + * world OP-TEE). A raised signal value is the OP-TEE async notification key + * verbatim: values 0 <= x < OPTEE_ABI_MAX_ASYNC_NOTIF_VALUE are delivered to + * optee_notif_send(), and the reserved top value requests an RPC bottom half. + * The bus is therefore sized one wider than the maximum notification value. + */ +#define OPTEE_ABI_ASYNC_NOTIF_BOTTOM_HALF OPTEE_ABI_MAX_ASYNC_NOTIF_VALUE +#define OPTEE_ABI_ASYNC_NOTIF_BUS_WIDTH (OPTEE_ABI_MAX_ASYNC_NOTIF_VALUE + 1) + /* TEE_PROBE_FEATURES request (Table 183) / response (Table 184). */ struct rpmi_tee_probe_features_req { __le32 feature_id;