Split ffa_memory_send into a separate version for TEE.
This refactoring avoids checking for the TEE case several times, and
makes the memory ownership more consistent. The ffa_memory module is now
responsible for forwarding memory send calls on to the TEE.
Bug: 132429380
Change-Id: I32bc216d9201e690c1d90c1349ae53e125969987
diff --git a/inc/hf/api.h b/inc/hf/api.h
index e56a2d2..bfeed7c 100644
--- a/inc/hf/api.h
+++ b/inc/hf/api.h
@@ -62,8 +62,7 @@
const struct vcpu *current, struct vcpu **next);
struct ffa_value api_ffa_mem_send(uint32_t share_func, uint32_t length,
uint32_t fragment_length, ipaddr_t address,
- uint32_t page_count, struct vcpu *current,
- struct vcpu **next);
+ uint32_t page_count, struct vcpu *current);
struct ffa_value api_ffa_mem_retrieve_req(uint32_t length,
uint32_t fragment_length,
ipaddr_t address, uint32_t page_count,
diff --git a/inc/hf/ffa_memory.h b/inc/hf/ffa_memory.h
index 30be4d8..9715918 100644
--- a/inc/hf/ffa_memory.h
+++ b/inc/hf/ffa_memory.h
@@ -21,10 +21,15 @@
#include "vmapi/hf/ffa.h"
-struct ffa_value ffa_memory_send(struct vm *to, struct vm_locked from_locked,
+struct ffa_value ffa_memory_send(struct vm_locked from_locked,
struct ffa_memory_region *memory_region,
uint32_t memory_share_length,
- uint32_t share_func, struct mpool *page_pool);
+ uint32_t fragment_length, uint32_t share_func,
+ struct mpool *page_pool);
+struct ffa_value ffa_memory_tee_send(
+ struct vm_locked from_locked, struct vm_locked to_locked,
+ struct ffa_memory_region *memory_region, uint32_t memory_share_length,
+ uint32_t fragment_length, uint32_t share_func, struct mpool *page_pool);
struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
struct ffa_memory_region *retrieve_request,
uint32_t retrieve_request_length,
diff --git a/src/api.c b/src/api.c
index 11d0fc2..41ccdac 100644
--- a/src/api.c
+++ b/src/api.c
@@ -382,12 +382,6 @@
.arg1 = (receiver->mailbox.recv_sender << 16) |
receiver->id,
.arg3 = receiver->mailbox.recv_size};
- case FFA_MEM_DONATE_32:
- case FFA_MEM_LEND_32:
- case FFA_MEM_SHARE_32:
- return (struct ffa_value){.func = receiver->mailbox.recv_func,
- .arg1 = receiver->mailbox.recv_size,
- .arg2 = receiver->mailbox.recv_size};
default:
/* This should never be reached, but return an error in case. */
dlog_error("Tried to return an invalid message function %#x\n",
@@ -1463,8 +1457,7 @@
struct ffa_value api_ffa_mem_send(uint32_t share_func, uint32_t length,
uint32_t fragment_length, ipaddr_t address,
- uint32_t page_count, struct vcpu *current,
- struct vcpu **next)
+ uint32_t page_count, struct vcpu *current)
{
struct vm *from = current->vm;
struct vm *to;
@@ -1504,7 +1497,8 @@
* pool. This prevents the sender from changing it underneath us, and
* also lets us keep it around in the share state table if needed.
*/
- if (length > HF_MAILBOX_SIZE || length > MM_PPOOL_ENTRY_SIZE) {
+ if (fragment_length > HF_MAILBOX_SIZE ||
+ fragment_length > MM_PPOOL_ENTRY_SIZE) {
return ffa_error(FFA_INVALID_PARAMETERS);
}
memory_region = (struct ffa_memory_region *)mpool_alloc(&api_page_pool);
@@ -1512,7 +1506,7 @@
dlog_verbose("Failed to allocate memory region copy.\n");
return ffa_error(FFA_NO_MEMORY);
}
- memcpy_s(memory_region, MM_PPOOL_ENTRY_SIZE, from_msg, length);
+ memcpy_s(memory_region, MM_PPOOL_ENTRY_SIZE, from_msg, fragment_length);
/* The sender must match the caller. */
if (memory_region->sender != from->id) {
@@ -1553,18 +1547,14 @@
goto out_unlock;
}
- ret = ffa_memory_send(to, vm_to_from_lock.vm2, memory_region,
- length, share_func, &api_page_pool);
- if (ret.func == FFA_SUCCESS_32) {
- /* Forward memory send message on to TEE. */
- memcpy_s(to->mailbox.recv, FFA_MSG_PAYLOAD_MAX,
- memory_region, length);
- to->mailbox.recv_size = length;
- to->mailbox.recv_sender = from->id;
- to->mailbox.recv_func = share_func;
- ret = deliver_msg(vm_to_from_lock.vm1, from->id,
- current, next);
- }
+ ret = ffa_memory_tee_send(
+ vm_to_from_lock.vm2, vm_to_from_lock.vm1, memory_region,
+ length, fragment_length, share_func, &api_page_pool);
+ /*
+ * ffa_tee_memory_send takes ownership of the memory_region, so
+ * make sure we don't free it.
+ */
+ memory_region = NULL;
out_unlock:
vm_unlock(&vm_to_from_lock.vm1);
@@ -1572,8 +1562,9 @@
} else {
struct vm_locked from_locked = vm_lock(from);
- ret = ffa_memory_send(to, from_locked, memory_region, length,
- share_func, &api_page_pool);
+ ret = ffa_memory_send(from_locked, memory_region, length,
+ fragment_length, share_func,
+ &api_page_pool);
/*
* ffa_memory_send takes ownership of the memory_region, so
* make sure we don't free it.
@@ -1721,17 +1712,17 @@
ffa_memory_region_flags_t flags,
struct cpu *cpu)
{
- uint32_t fragment_length;
+ struct ffa_value tee_ret;
uint32_t length;
- uint32_t request_length;
+ uint32_t fragment_length;
struct ffa_memory_region *memory_region =
(struct ffa_memory_region *)cpu_get_buffer(cpu);
uint32_t message_buffer_size = cpu_get_buffer_size(cpu);
- struct ffa_value tee_ret;
-
- request_length = ffa_memory_lender_retrieve_request_init(
+ uint32_t request_length = ffa_memory_lender_retrieve_request_init(
from_locked.vm->mailbox.recv, handle, to_locked.vm->id);
+ CHECK(request_length <= HF_MAILBOX_SIZE);
+
/* Retrieve memory region information from the TEE. */
tee_ret = arch_tee_call(
(struct ffa_value){.func = FFA_MEM_RETRIEVE_REQ_32,
diff --git a/src/arch/aarch64/hypervisor/handler.c b/src/arch/aarch64/hypervisor/handler.c
index 8eb580c..c67ede0 100644
--- a/src/arch/aarch64/hypervisor/handler.c
+++ b/src/arch/aarch64/hypervisor/handler.c
@@ -367,7 +367,7 @@
case FFA_MEM_SHARE_32:
*args = api_ffa_mem_send(func, args->arg1, args->arg2,
ipa_init(args->arg3), args->arg4,
- current(), next);
+ current());
return true;
case FFA_MEM_RETRIEVE_REQ_32:
*args = api_ffa_mem_retrieve_req(args->arg1, args->arg2,
diff --git a/src/ffa_memory.c b/src/ffa_memory.c
index e6ab21e..3bdda92 100644
--- a/src/ffa_memory.c
+++ b/src/ffa_memory.c
@@ -973,7 +973,7 @@
if (ret.func != FFA_SUCCESS_32) {
dlog_verbose(
- "Got %#x (%d) from EL3 in response to "
+ "Got %#x (%d) from TEE in response to "
"FFA_MEM_RECLAIM_32, expected FFA_SUCCESS_32.\n",
ret.func, ret.arg2);
goto out;
@@ -1086,14 +1086,14 @@
* Check that the given `memory_region` represents a valid memory send request
* of the given `share_func` type, return the clear flag and permissions via the
* respective output parameters, and update the permissions if necessary.
+ *
* Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
* not.
*/
static struct ffa_value ffa_memory_send_validate(
- struct vm *to, struct vm_locked from_locked,
- struct ffa_memory_region *memory_region, uint32_t memory_share_length,
- uint32_t share_func, bool *clear,
- ffa_memory_access_permissions_t *permissions)
+ struct vm_locked from_locked, struct ffa_memory_region *memory_region,
+ uint32_t memory_share_length, uint32_t fragment_length,
+ uint32_t share_func, ffa_memory_access_permissions_t *permissions)
{
struct ffa_composite_memory_region *composite;
uint32_t receivers_length;
@@ -1102,21 +1102,20 @@
enum ffa_data_access data_access;
enum ffa_instruction_access instruction_access;
- CHECK(clear != NULL);
CHECK(permissions != NULL);
+ /*
+ * This should already be checked by the caller, just making the
+ * assumption clear here.
+ */
+ CHECK(memory_region->receiver_count == 1);
+
/* The sender must match the message sender. */
if (memory_region->sender != from_locked.vm->id) {
dlog_verbose("Invalid sender %d.\n", memory_region->sender);
return ffa_error(FFA_INVALID_PARAMETERS);
}
- /* We only support a single recipient. */
- if (memory_region->receiver_count != 1) {
- dlog_verbose("Multiple recipients not supported.\n");
- return ffa_error(FFA_INVALID_PARAMETERS);
- }
-
/*
* Ensure that the composite header is within the memory bounds and
* doesn't overlap the first part of the message.
@@ -1127,7 +1126,7 @@
ffa_composite_constituent_offset(memory_region, 0);
if (memory_region->receivers[0].composite_memory_region_offset <
sizeof(struct ffa_memory_region) + receivers_length ||
- constituents_offset >= memory_share_length) {
+ constituents_offset > fragment_length) {
dlog_verbose(
"Invalid composite memory region descriptor offset "
"%d.\n",
@@ -1151,18 +1150,12 @@
return ffa_error(FFA_INVALID_PARAMETERS);
}
- /* The recipient must match the message recipient. */
- if (memory_region->receivers[0].receiver_permissions.receiver !=
- to->id) {
- return ffa_error(FFA_INVALID_PARAMETERS);
- }
-
- *clear = memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
/*
* Clear is not allowed for memory sharing, as the sender still has
* access to the memory.
*/
- if (*clear && share_func == FFA_MEM_SHARE_32) {
+ if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
+ share_func == FFA_MEM_SHARE_32) {
dlog_verbose("Memory can't be cleared while being shared.\n");
return ffa_error(FFA_INVALID_PARAMETERS);
}
@@ -1230,39 +1223,65 @@
return (struct ffa_value){.func = FFA_SUCCESS_32};
}
+/** Forwards a memory send message on to the TEE. */
+static struct ffa_value memory_send_tee_forward(
+ struct vm_locked tee_locked, ffa_vm_id_t sender_vm_id,
+ uint32_t share_func, struct ffa_memory_region *memory_region,
+ uint32_t memory_share_length, uint32_t fragment_length)
+{
+ struct ffa_value ret;
+
+ memcpy_s(tee_locked.vm->mailbox.recv, FFA_MSG_PAYLOAD_MAX,
+ memory_region, fragment_length);
+ tee_locked.vm->mailbox.recv_size = fragment_length;
+ tee_locked.vm->mailbox.recv_sender = sender_vm_id;
+ tee_locked.vm->mailbox.recv_func = share_func;
+ tee_locked.vm->mailbox.state = MAILBOX_STATE_RECEIVED;
+ ret = arch_tee_call((struct ffa_value){.func = share_func,
+ .arg1 = memory_share_length,
+ .arg2 = fragment_length});
+ /*
+ * After the call to the TEE completes it must have finished reading its
+ * RX buffer, so it is ready for another message.
+ */
+ tee_locked.vm->mailbox.state = MAILBOX_STATE_EMPTY;
+
+ return ret;
+}
+
/**
- * Validates a call to donate, lend or share memory and then updates the stage-2
- * page tables. Specifically, check if the message length and number of memory
- * region constituents match, and if the transition is valid for the type of
- * memory sending operation.
+ * Validates a call to donate, lend or share memory to a non-TEE VM and then
+ * updates the stage-2 page tables. Specifically, check if the message length
+ * and number of memory region constituents match, and if the transition is
+ * valid for the type of memory sending operation.
*
- * Assumes that the caller has already found and locked both VMs and ensured
- * that the destination RX buffer is available, and copied the memory region
- * descriptor from the sender's TX buffer to a freshly allocated page from
- * Hafnium's internal pool.
+ * Assumes that the caller has already found and locked the sender VM and copied
+ * the memory region descriptor from the sender's TX buffer to a freshly
+ * allocated page from Hafnium's internal pool. The caller must have also
+ * validated that the receiver VM ID is valid.
*
- * This function takes ownership of the `memory_region` passed in; it must not
- * be freed by the caller.
+ * This function takes ownership of the `memory_region` passed in and will free
+ * it when necessary; it must not be freed by the caller.
*/
-struct ffa_value ffa_memory_send(struct vm *to, struct vm_locked from_locked,
+struct ffa_value ffa_memory_send(struct vm_locked from_locked,
struct ffa_memory_region *memory_region,
uint32_t memory_share_length,
- uint32_t share_func, struct mpool *page_pool)
+ uint32_t fragment_length, uint32_t share_func,
+ struct mpool *page_pool)
{
- struct ffa_composite_memory_region *composite;
- bool clear;
ffa_memory_access_permissions_t permissions;
struct ffa_value ret;
ffa_memory_handle_t handle;
+ struct ffa_composite_memory_region *composite;
/*
* If there is an error validating the `memory_region` then we need to
* free it because we own it but we won't be storing it in a share state
* after all.
*/
- ret = ffa_memory_send_validate(to, from_locked, memory_region,
- memory_share_length, share_func, &clear,
- &permissions);
+ ret = ffa_memory_send_validate(from_locked, memory_region,
+ memory_share_length, fragment_length,
+ share_func, &permissions);
if (ret.func != FFA_SUCCESS_32) {
mpool_free(page_pool, memory_region);
return ret;
@@ -1289,8 +1308,7 @@
* failed then it would leave the memory in a state where nobody could
* get it back.
*/
- if (to->id != HF_TEE_VM_ID &&
- !allocate_share_state(share_func, memory_region, &handle)) {
+ if (!allocate_share_state(share_func, memory_region, &handle)) {
dlog_verbose("Failed to allocate share state.\n");
mpool_free(page_pool, memory_region);
return ffa_error(FFA_NO_MEMORY);
@@ -1300,26 +1318,72 @@
/* Check that state is valid in sender page table and update. */
composite = ffa_memory_region_get_composite(memory_region, 0);
- ret = ffa_send_memory(from_locked, composite->constituents,
- composite->constituent_count, share_func,
- permissions, page_pool, clear);
+ ret = ffa_send_memory(
+ from_locked, composite->constituents,
+ composite->constituent_count, share_func, permissions,
+ page_pool, memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR);
if (ret.func != FFA_SUCCESS_32) {
- if (to->id != HF_TEE_VM_ID) {
- /* Free share state. */
- bool freed = share_state_free_handle(handle, page_pool);
-
- CHECK(freed);
- }
-
+ /* Free share state. */
+ CHECK(share_state_free_handle(handle, page_pool));
return ret;
}
- if (to->id == HF_TEE_VM_ID) {
- /* No share state allocated here so no handle to return. */
- return (struct ffa_value){.func = FFA_SUCCESS_32};
+ return ffa_mem_success(handle);
+}
+
+/**
+ * Validates a call to donate, lend or share memory to the TEE and then updates
+ * the stage-2 page tables. Specifically, check if the message length and number
+ * of memory region constituents match, and if the transition is valid for the
+ * type of memory sending operation.
+ *
+ * Assumes that the caller has already found and locked the sender VM and the
+ * TEE VM, and copied the memory region descriptor from the sender's TX buffer
+ * to a freshly allocated page from Hafnium's internal pool. The caller must
+ * have also validated that the receiver VM ID is valid.
+ *
+ * This function takes ownership of the `memory_region` passed in and will free
+ * it when necessary; it must not be freed by the caller.
+ */
+struct ffa_value ffa_memory_tee_send(
+ struct vm_locked from_locked, struct vm_locked to_locked,
+ struct ffa_memory_region *memory_region, uint32_t memory_share_length,
+ uint32_t fragment_length, uint32_t share_func, struct mpool *page_pool)
+{
+ ffa_memory_access_permissions_t permissions;
+ struct ffa_value ret;
+ struct ffa_composite_memory_region *composite;
+
+ /*
+ * If there is an error validating the `memory_region` then we need to
+ * free it because we own it but we won't be storing it in a share state
+ * after all.
+ */
+ ret = ffa_memory_send_validate(from_locked, memory_region,
+ memory_share_length, fragment_length,
+ share_func, &permissions);
+ if (ret.func != FFA_SUCCESS_32) {
+ goto out;
}
- return ffa_mem_success(handle);
+ /* Check that state is valid in sender page table and update. */
+ composite = ffa_memory_region_get_composite(memory_region, 0);
+ ret = ffa_send_memory(
+ from_locked, composite->constituents,
+ composite->constituent_count, share_func, permissions,
+ page_pool, memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR);
+ if (ret.func != FFA_SUCCESS_32) {
+ goto out;
+ }
+
+ /* Forward memory send message on to TEE. */
+ ret = memory_send_tee_forward(to_locked, from_locked.vm->id, share_func,
+ memory_region, memory_share_length,
+ fragment_length);
+
+out:
+ mpool_free(page_pool, memory_region);
+ return ret;
}
struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
