Allow messages to be sent between all VMs.

Any VM can send a message to any other VM. The hypervisor acts as a
router to deliver the message to the destination but does not track
higher level state about the communications.

When receiving a message, the source VM is reported so access controls
can be applies and replies can be sent back.

Bug: 116705004
Change-Id: Ib83988eb8ddee1753dfd67a3baa3fb991ebd4dd7
diff --git a/driver/linux b/driver/linux
index bb7ae41..0973a2e 160000
--- a/driver/linux
+++ b/driver/linux
@@ -1 +1 @@
-Subproject commit bb7ae41a9456dc4e051468e61320cac0d5248f87
+Subproject commit 0973a2ec678062affef66303b1e057b7c0be555e
diff --git a/inc/hf/api.h b/inc/hf/api.h
index a936558..35aba8d 100644
--- a/inc/hf/api.h
+++ b/inc/hf/api.h
@@ -8,10 +8,9 @@
 int64_t api_vcpu_run(uint32_t vm_id, uint32_t vcpu_idx, struct vcpu **next);
 int64_t api_vm_configure(ipaddr_t send, ipaddr_t recv);
 
-int64_t api_rpc_request(uint32_t vm_id, size_t size);
-int64_t api_rpc_read_request(bool block, struct vcpu **next);
-int64_t api_rpc_reply(size_t size, bool ack, struct vcpu **next);
-int64_t api_rpc_ack(void);
+int64_t api_mailbox_send(uint32_t vm_id, size_t size, struct vcpu **next);
+int64_t api_mailbox_receive(bool block, struct vcpu **next);
+int64_t api_mailbox_clear(void);
 
 struct vcpu *api_wait_for_interrupt(void);
 struct vcpu *api_yield(void);
diff --git a/inc/hf/cpu.h b/inc/hf/cpu.h
index b263e39..d598296 100644
--- a/inc/hf/cpu.h
+++ b/inc/hf/cpu.h
@@ -10,10 +10,19 @@
 #include "hf/spinlock.h"
 
 enum vcpu_state {
+	/* The vcpu is switched off. */
 	vcpu_state_off,
+
+	/* The vcpu is ready to be run. */
 	vcpu_state_ready,
+
+	/* The vcpu is currently running. */
 	vcpu_state_running,
-	vcpu_state_blocked_rpc,
+
+	/* The vcpu is waiting for a message. */
+	vcpu_state_blocked_mailbox,
+
+	/* The vcpu is waiting for an interrupt. */
 	vcpu_state_blocked_interrupt,
 };
 
@@ -21,7 +30,7 @@
 	struct spinlock lock;
 	enum vcpu_state state;
 	struct vm *vm;
-	struct vcpu *rpc_next;
+	struct vcpu *mailbox_next;
 	struct arch_regs regs;
 };
 
diff --git a/inc/hf/vm.h b/inc/hf/vm.h
index 3d7bf2e..64e4ee0 100644
--- a/inc/hf/vm.h
+++ b/inc/hf/vm.h
@@ -3,14 +3,20 @@
 #include "hf/cpu.h"
 #include "hf/mm.h"
 
-enum rpc_state {
-	rpc_state_idle,
-	rpc_state_pending,
-	rpc_state_inflight,
+enum mailbox_state {
+	/* There is no message in the mailbox. */
+	mailbox_state_empty,
+
+	/* There is a message in the mailbox that is waiting for a reader. */
+	mailbox_state_received,
+
+	/* There is a message in the mailbox that has been read. */
+	mailbox_state_read,
 };
 
-struct rpc {
-	enum rpc_state state;
+struct mailbox {
+	enum mailbox_state state;
+	uint32_t recv_from_id;
 	int16_t recv_bytes;
 	void *recv;
 	const void *send;
@@ -23,7 +29,7 @@
 	uint32_t vcpu_count;
 	struct vcpu vcpus[MAX_CPUS];
 	struct mm_ptable ptable;
-	struct rpc rpc;
+	struct mailbox mailbox;
 };
 
 bool vm_init(uint32_t vcpu_count, struct vm **new_vm);
diff --git a/inc/vmapi/hf/call.h b/inc/vmapi/hf/call.h
index 0650868..4f6e05b 100644
--- a/inc/vmapi/hf/call.h
+++ b/inc/vmapi/hf/call.h
@@ -24,10 +24,9 @@
 #define HF_VM_GET_COUNT     0xff01
 #define HF_VCPU_GET_COUNT   0xff02
 #define HF_VM_CONFIGURE     0xff03
-#define HF_RPC_REQUEST      0xff04
-#define HF_RPC_READ_REQUEST 0xff05
-#define HF_RPC_ACK          0xff06
-#define HF_RPC_REPLY        0xff07
+#define HF_MAILBOX_SEND     0xff04
+#define HF_MAILBOX_RECEIVE  0xff05
+#define HF_MAILBOX_CLEAR    0xff06
 
 /* The ID of the primary VM which is responsile for scheduling. */
 #define HF_PRIMARY_VM_ID 0
@@ -36,14 +35,26 @@
 #define HF_VCPU_RUN_YIELD              0x00
 #define HF_VCPU_RUN_WAIT_FOR_INTERRUPT 0x01
 #define HF_VCPU_RUN_WAKE_UP            0x02
-#define HF_VCPU_RUN_RESPONSE_READY     0x03
+#define HF_VCPU_RUN_MESSAGE            0x03
 
 /* Construct and destruct the hf_vcpu_run() response. */
-#define HF_VCPU_RUN_RESPONSE(code, data) ((code & 0xff) | (data << 8))
+#define HF_VCPU_RUN_RESPONSE(code, vm_id, data)               \
+	((code & 0xff) | ((uint64_t)(vm_id & 0xffff) << 16) | \
+	 ((uint64_t)data << 32))
 #define HF_VCPU_RUN_CODE(ret) (ret & 0xff)
-#define HF_VCPU_RUN_DATA(ret) (ret >> 8)
+#define HF_VCPU_RUN_VM_ID(ret) ((ret >> 16) & 0xffff)
+#define HF_VCPU_RUN_DATA(ret) (ret >> 32)
 
-#define HF_RPC_REQUEST_MAX_SIZE 4096
+/* Construct and destruct the hf_mailbox_receive() response. */
+#define HF_MAILBOX_RECEIVE_RESPONSE(vm_id, size) \
+	((vm_id & 0xffff) | ((uint64_t)size << 32))
+#define HF_MAILBOX_RECEIVE_VM_ID(ret) (ret & 0xffff)
+#define HF_MAILBOX_RECEIVE_SIZE(ret) (ret >> 32)
+
+#define HF_MAILBOX_SIZE 4096
+
+#define HF_INVALID_VM_ID 0xffff
+#define HF_INVALID_VCPU  0xffff
 
 /* clang-format on */
 
@@ -87,45 +98,28 @@
 }
 
 /**
- * Called by the primary VM to send an RPC request to a secondary VM. Data is
- * copied from the caller's send buffer to the destination's receive buffer.
+ * Copies data from the sender's send buffer to the recipient's receive buffer.
  */
-static inline int64_t hf_rpc_request(uint32_t vm_id, size_t size)
+static inline int64_t hf_mailbox_send(uint32_t vm_id, size_t size)
 {
-	return hf_call(HF_RPC_REQUEST, vm_id, size, 0);
+	return hf_call(HF_MAILBOX_SEND, vm_id, size, 0);
 }
 
 /**
- * Called by the primary VM to read a request sent from a previous call to
- * api_rpc_request. If one isn't available, this function can optionally block
- * the caller until one becomes available.
+ * Called by secondary VMs to receive a message. The call can optionally block
+ * until a message is received.
  *
- * Once the caller has completed handling a request, it must indicate it by
- * either calling api_rpc_reply or api_rpc_ack. No new requests can be accepted
- * until the current one is acknowledged.
+ * The mailbox must be cleared before a new message can be received.
  */
-static inline int64_t hf_rpc_read_request(bool block)
+static inline int64_t hf_mailbox_receive(bool block)
 {
-	return hf_call(HF_RPC_READ_REQUEST, block, 0, 0);
+	return hf_call(HF_MAILBOX_RECEIVE, block, 0, 0);
 }
 
 /**
- * Acknowledges that either a request or a reply has been received and handled.
- * After this call completes, the caller will be able to receive additional
- * requests or replies.
+ * Clears the mailbox so a new message can be received.
  */
-static inline int64_t hf_rpc_ack(void)
+static inline int64_t hf_mailbox_clear(void)
 {
-	return hf_call(HF_RPC_ACK, 0, 0, 0);
-}
-
-/**
- * Called by a secondary VM to send a reply to the primary VM. Data is copied
- * from the caller's send buffer to the destination's receive buffer.
- *
- * It can optionally acknowledge the pending request.
- */
-static inline int64_t hf_rpc_reply(size_t size, bool ack)
-{
-	return hf_call(HF_RPC_REPLY, size, ack, 0);
+	return hf_call(HF_MAILBOX_CLEAR, 0, 0, 0);
 }
diff --git a/src/api.c b/src/api.c
index 11a2632..cd90205 100644
--- a/src/api.c
+++ b/src/api.c
@@ -7,12 +7,16 @@
 
 #include "vmapi/hf/call.h"
 
-static_assert(HF_RPC_REQUEST_MAX_SIZE == PAGE_SIZE,
+static_assert(HF_MAILBOX_SIZE == PAGE_SIZE,
 	      "Currently, a page is mapped for the send and receive buffers so "
 	      "the maximum request is the size of a page.");
 
 /**
  * Switches the physical CPU back to the corresponding vcpu of the primary VM.
+ *
+ * This triggers the scheduling logic to run. Run in the context of secondary VM
+ * to cause HF_VCPU_RUN to return and the primary VM to regain control of the
+ * cpu.
  */
 static struct vcpu *api_switch_to_primary(size_t primary_retval,
 					  enum vcpu_state secondary_state)
@@ -25,8 +29,7 @@
 	vm_set_current(primary);
 
 	/*
-	 * Inidicate to primary VM that this vcpu blocked waiting for an
-	 * interrupt.
+	 * Set the return valuefor the primary VM's call to HF_VCPU_RUN.
 	 */
 	arch_regs_set_retval(&next->regs, primary_retval);
 
@@ -39,6 +42,28 @@
 }
 
 /**
+ * Returns to the primary vm leaving the current vcpu ready to be scheduled
+ * again.
+ */
+struct vcpu *api_yield(void)
+{
+	return api_switch_to_primary(
+		HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_YIELD, 0, 0),
+		vcpu_state_ready);
+}
+
+/**
+ * Puts the current vcpu in wait for interrupt mode, and returns to the primary
+ * vm.
+ */
+struct vcpu *api_wait_for_interrupt(void)
+{
+	return api_switch_to_primary(
+		HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAIT_FOR_INTERRUPT, 0, 0),
+		vcpu_state_blocked_interrupt);
+}
+
+/**
  * Returns the number of VMs configured to run.
  */
 int64_t api_vm_get_count(void)
@@ -100,19 +125,20 @@
 
 	sl_lock(&vcpu->lock);
 	if (vcpu->state != vcpu_state_ready) {
-		ret = HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAIT_FOR_INTERRUPT, 0);
+		ret = HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAIT_FOR_INTERRUPT, 0,
+					   0);
 	} else {
 		vcpu->state = vcpu_state_running;
 		vm_set_current(vm);
 		*next = vcpu;
-		ret = HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_YIELD, 0);
+		ret = HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_YIELD, 0, 0);
 	}
 	sl_unlock(&vcpu->lock);
 
 	return ret;
 
 fail:
-	return HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAIT_FOR_INTERRUPT, 0);
+	return HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAIT_FOR_INTERRUPT, 0, 0);
 }
 
 /**
@@ -137,7 +163,7 @@
 	sl_lock(&vm->lock);
 
 	/* We only allow these to be setup once. */
-	if (vm->rpc.send || vm->rpc.recv) {
+	if (vm->mailbox.send || vm->mailbox.recv) {
 		ret = -1;
 		goto exit;
 	}
@@ -168,8 +194,9 @@
 	pa_recv_end = pa_add(pa_recv_begin, PAGE_SIZE);
 
 	/* Map the send page as read-only in the hypervisor address space. */
-	vm->rpc.send = mm_identity_map(pa_send_begin, pa_send_end, MM_MODE_R);
-	if (!vm->rpc.send) {
+	vm->mailbox.send =
+		mm_identity_map(pa_send_begin, pa_send_end, MM_MODE_R);
+	if (!vm->mailbox.send) {
 		ret = -1;
 		goto exit;
 	}
@@ -178,9 +205,10 @@
 	 * Map the receive page as writable in the hypervisor address space. On
 	 * failure, unmap the send page before returning.
 	 */
-	vm->rpc.recv = mm_identity_map(pa_recv_begin, pa_recv_end, MM_MODE_W);
-	if (!vm->rpc.recv) {
-		vm->rpc.send = NULL;
+	vm->mailbox.recv =
+		mm_identity_map(pa_recv_begin, pa_recv_end, MM_MODE_W);
+	if (!vm->mailbox.recv) {
+		vm->mailbox.send = NULL;
 		mm_unmap(pa_send_begin, pa_send_end, 0);
 		ret = -1;
 		goto exit;
@@ -196,18 +224,20 @@
 }
 
 /**
- * Sends an RPC request from the primary VM to a secondary VM. Data is copied
- * from the caller's send buffer to the destination's receive buffer.
+ * Copies data from the sender's send buffer to the recipient's receive buffer
+ * and notifies the recipient.
  */
-int64_t api_rpc_request(uint32_t vm_id, size_t size)
+int64_t api_mailbox_send(uint32_t vm_id, size_t size, struct vcpu **next)
 {
 	struct vm *from = cpu()->current->vm;
 	struct vm *to;
 	const void *from_buf;
+	uint16_t vcpu;
 	int64_t ret;
+	int64_t primary_ret;
 
-	/* Basic argument validation. */
-	if (size > HF_RPC_REQUEST_MAX_SIZE) {
+	/* Limit the size of transfer. */
+	if (size > HF_MAILBOX_SIZE) {
 		return -1;
 	}
 
@@ -222,199 +252,169 @@
 		return -1;
 	}
 
-	/* Only the primary VM can make calls. */
-	if (from->id != HF_PRIMARY_VM_ID) {
-		return -1;
-	}
-
 	/*
 	 * Check that the sender has configured its send buffer. It is safe to
 	 * use from_buf after releasing the lock because the buffer cannot be
 	 * modified once it's configured.
 	 */
 	sl_lock(&from->lock);
-	from_buf = from->rpc.send;
+	from_buf = from->mailbox.send;
 	sl_unlock(&from->lock);
-	if (!from_buf) {
+	if (from_buf == NULL) {
 		return -1;
 	}
 
 	sl_lock(&to->lock);
 
-	if (to->rpc.state != rpc_state_idle || !to->rpc.recv) {
+	if (to->mailbox.state != mailbox_state_empty ||
+	    to->mailbox.recv == NULL) {
 		/* Fail if the target isn't currently ready to receive data. */
 		ret = -1;
-	} else {
-		/* Copy data. */
-		memcpy(to->rpc.recv, from_buf, size);
-		to->rpc.recv_bytes = size;
-
-		if (!to->rpc.recv_waiter) {
-			to->rpc.state = rpc_state_pending;
-			ret = to->vcpu_count;
-		} else {
-			struct vcpu *to_vcpu = to->rpc.recv_waiter;
-
-			to->rpc.state = rpc_state_inflight;
-
-			/*
-			 * Take target vcpu out of waiter list and mark as ready
-			 * to run again.
-			 */
-			sl_lock(&to_vcpu->lock);
-			to->rpc.recv_waiter = to_vcpu->rpc_next;
-			to_vcpu->state = vcpu_state_ready;
-			arch_regs_set_retval(&to_vcpu->regs, size);
-			sl_unlock(&to_vcpu->lock);
-
-			ret = to_vcpu - to->vcpus;
-		}
+		goto out;
 	}
 
+	/* Copy data. */
+	memcpy(to->mailbox.recv, from_buf, size);
+	to->mailbox.recv_bytes = size;
+	to->mailbox.recv_from_id = from->id;
+	to->mailbox.state = mailbox_state_read;
+
+	/* Messages for the primary VM are delivered directly. */
+	if (to->id == HF_PRIMARY_VM_ID) {
+		primary_ret =
+			HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_MESSAGE, 0, size);
+		ret = 0;
+		goto out;
+	}
+
+	/*
+	 * Try to find a vcpu to handle the message and tell the scheduler to
+	 * run it.
+	 */
+	if (to->mailbox.recv_waiter == NULL) {
+		/*
+		 * The scheduler must choose a vcpu to interrupt so it can
+		 * handle the message.
+		 */
+		to->mailbox.state = mailbox_state_received;
+		vcpu = HF_INVALID_VCPU;
+	} else {
+		struct vcpu *to_vcpu = to->mailbox.recv_waiter;
+
+		/*
+		 * Take target vcpu out of waiter list and mark as ready
+		 * to run again.
+		 */
+		sl_lock(&to_vcpu->lock);
+		to->mailbox.recv_waiter = to_vcpu->mailbox_next;
+		to_vcpu->state = vcpu_state_ready;
+
+		/* Return from HF_MAILBOX_RECEIVE. */
+		arch_regs_set_retval(&to_vcpu->regs,
+				     HF_MAILBOX_RECEIVE_RESPONSE(
+					     to->mailbox.recv_from_id, size));
+
+		sl_unlock(&to_vcpu->lock);
+
+		vcpu = to_vcpu - to->vcpus;
+	}
+
+	/* Return to the primary VM directly or with a switch. */
+	primary_ret = HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAKE_UP, to->id, vcpu);
+	ret = 0;
+
+out:
+	/*
+	 * Unlock before routing the return values as switching to the primary
+	 * will acquire more locks and nesting the locks is avoidable.
+	 */
 	sl_unlock(&to->lock);
 
-	return ret;
+	/* Report errors to the sender. */
+	if (ret != 0) {
+		return ret;
+	}
+
+	/* If the sender is the primary, return the vcpu to schedule. */
+	if (from->id == HF_PRIMARY_VM_ID) {
+		return vcpu;
+	}
+
+	/* Switch to primary for scheduling and return success to the sender. */
+	*next = api_switch_to_primary(primary_ret, vcpu_state_ready);
+	return 0;
 }
 
 /**
- * Reads a request sent from a previous call to api_rpc_request. If one isn't
- * available, this function can optionally block the caller until one becomes
- * available.
+ * Receives a message from the mailbox. If one isn't available, this function
+ * can optionally block the caller until one becomes available.
  *
- * Once the caller has completed handling a request, it must indicate it by
- * either calling api_rpc_reply or api_rpc_ack. No new requests can be accepted
- * until the current one is acknowledged.
+ * No new messages can be received until the mailbox has been cleared.
  */
-int64_t api_rpc_read_request(bool block, struct vcpu **next)
+int64_t api_mailbox_receive(bool block, struct vcpu **next)
 {
 	struct vcpu *vcpu = cpu()->current;
 	struct vm *vm = vcpu->vm;
-	struct vm *primary = vm_get(HF_PRIMARY_VM_ID);
-	int64_t ret;
+	int64_t ret = 0;
 
-	/* Only the secondary VMs can receive calls. */
+	/*
+	 * The primary VM will receive messages as a status code from running
+	 * vcpus and must not call this function.
+	 */
 	if (vm->id == HF_PRIMARY_VM_ID) {
 		return -1;
 	}
 
 	sl_lock(&vm->lock);
-	if (vm->rpc.state == rpc_state_pending) {
-		ret = vm->rpc.recv_bytes;
-		vm->rpc.state = rpc_state_inflight;
-	} else if (!block) {
-		ret = -1;
-	} else {
-		sl_lock(&vcpu->lock);
-		vcpu->state = vcpu_state_blocked_rpc;
 
-		/* Push vcpu into waiter list. */
-		vcpu->rpc_next = vm->rpc.recv_waiter;
-		vm->rpc.recv_waiter = vcpu;
-		sl_unlock(&vcpu->lock);
-
-		/* Switch back to primary vm. */
-		*next = &primary->vcpus[cpu_index(cpu())];
-		vm_set_current(primary);
-
-		/*
-		 * Inidicate to primary VM that this vcpu blocked waiting for an
-		 * interrupt.
-		 */
-		arch_regs_set_retval(
-			&(*next)->regs,
-			HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAIT_FOR_INTERRUPT,
-					     0));
-		ret = 0;
+	/* Return pending messages without blocking. */
+	if (vm->mailbox.state == mailbox_state_received) {
+		vm->mailbox.state = mailbox_state_read;
+		block = false;
+		ret = HF_MAILBOX_RECEIVE_RESPONSE(vm->mailbox.recv_from_id,
+						  vm->mailbox.recv_bytes);
+		goto out;
 	}
+
+	/* No pending message so fail if not allowed to block. */
+	if (!block) {
+		ret = -1;
+		goto out;
+	}
+
+	sl_lock(&vcpu->lock);
+	vcpu->state = vcpu_state_blocked_mailbox;
+
+	/* Push vcpu into waiter list. */
+	vcpu->mailbox_next = vm->mailbox.recv_waiter;
+	vm->mailbox.recv_waiter = vcpu;
+	sl_unlock(&vcpu->lock);
+
+	/* Switch back to primary vm to block. */
+	*next = api_wait_for_interrupt();
+
+out:
 	sl_unlock(&vm->lock);
 
 	return ret;
 }
 
 /**
- * Sends a reply from a secondary VM to the primary VM. Data is copied from the
- * caller's send buffer to the destination's receive buffer.
- *
- * It can optionally acknowledge the pending request.
+ * Clears the caller's mailbox so that a new message can be received. The caller
+ * must have copied out all data they wish to preserve as new messages will
+ * overwrite the old and will arrive asynchronously.
  */
-int64_t api_rpc_reply(size_t size, bool ack, struct vcpu **next)
-{
-	struct vm *from = cpu()->current->vm;
-	struct vm *to;
-	const void *from_buf;
-
-	/* Basic argument validation. */
-	if (size > PAGE_SIZE) {
-		return -1;
-	}
-
-	/* Only the secondary VM can send responses. */
-	if (from->id == HF_PRIMARY_VM_ID) {
-		return -1;
-	}
-
-	/* Acknowledge the current pending request if requested. */
-	if (ack) {
-		api_rpc_ack();
-	}
-
-	/*
-	 * Check that the sender has configured its send buffer. It is safe to
-	 * use from_buf after releasing the lock because the buffer cannot be
-	 * modified once it's configured.
-	 */
-	sl_lock(&from->lock);
-	from_buf = from->rpc.send;
-	sl_unlock(&from->lock);
-	if (!from_buf) {
-		return -1;
-	}
-
-	to = vm_get(HF_PRIMARY_VM_ID);
-	sl_lock(&to->lock);
-
-	if (to->rpc.state != rpc_state_idle || !to->rpc.recv) {
-		/*
-		 * Fail if the target isn't currently ready to receive a
-		 * response.
-		 */
-		sl_unlock(&to->lock);
-		return -1;
-	}
-
-	/* Copy data. */
-	memcpy(to->rpc.recv, from_buf, size);
-	to->rpc.recv_bytes = size;
-	to->rpc.state = rpc_state_inflight;
-	sl_unlock(&to->lock);
-
-	/*
-	 * Switch back to primary VM so that it is aware that a response was
-	 * received. But we leave the current vcpu still runnable.
-	 */
-	*next = api_switch_to_primary(
-		HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_RESPONSE_READY, size),
-		vcpu_state_ready);
-
-	return 0;
-}
-
-/**
- * Acknowledges that either a request or a reply has been received and handled.
- * After this call completes, the caller will be able to receive additional
- * requests or replies.
- */
-int64_t api_rpc_ack(void)
+int64_t api_mailbox_clear(void)
 {
 	struct vm *vm = cpu()->current->vm;
 	int64_t ret;
 
 	sl_lock(&vm->lock);
-	if (vm->rpc.state != rpc_state_inflight) {
-		ret = -1;
-	} else {
+	if (vm->mailbox.state == mailbox_state_read) {
 		ret = 0;
-		vm->rpc.state = rpc_state_idle;
+		vm->mailbox.state = mailbox_state_empty;
+	} else {
+		ret = -1;
 	}
 	sl_unlock(&vm->lock);
 
@@ -424,24 +424,3 @@
 
 	return ret;
 }
-
-/**
- * Returns to the primary vm leaving the current vcpu ready to be scheduled
- * again.
- */
-struct vcpu *api_yield(void)
-{
-	return api_switch_to_primary(HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_YIELD, 0),
-				     vcpu_state_ready);
-}
-
-/**
- * Puts the current vcpu in wait for interrupt mode, and returns to the primary
- * vm.
- */
-struct vcpu *api_wait_for_interrupt(void)
-{
-	return api_switch_to_primary(
-		HF_VCPU_RUN_RESPONSE(HF_VCPU_RUN_WAIT_FOR_INTERRUPT, 0),
-		vcpu_state_blocked_interrupt);
-}
diff --git a/src/arch/aarch64/handler.c b/src/arch/aarch64/handler.c
index b982bf9..4bdbb04 100644
--- a/src/arch/aarch64/handler.c
+++ b/src/arch/aarch64/handler.c
@@ -189,20 +189,16 @@
 		ret.user_ret = api_vm_configure(ipa_init(arg1), ipa_init(arg2));
 		break;
 
-	case HF_RPC_REQUEST:
-		ret.user_ret = api_rpc_request(arg1, arg2);
+	case HF_MAILBOX_SEND:
+		ret.user_ret = api_mailbox_send(arg1, arg2, &ret.new);
 		break;
 
-	case HF_RPC_READ_REQUEST:
-		ret.user_ret = api_rpc_read_request(arg1, &ret.new);
+	case HF_MAILBOX_RECEIVE:
+		ret.user_ret = api_mailbox_receive(arg1, &ret.new);
 		break;
 
-	case HF_RPC_ACK:
-		ret.user_ret = api_rpc_ack();
-		break;
-
-	case HF_RPC_REPLY:
-		ret.user_ret = api_rpc_reply(arg1, arg2, &ret.new);
+	case HF_MAILBOX_CLEAR:
+		ret.user_ret = api_mailbox_clear();
 		break;
 
 	default:
diff --git a/src/vm.c b/src/vm.c
index 0958682..b62238d 100644
--- a/src/vm.c
+++ b/src/vm.c
@@ -24,7 +24,7 @@
 
 	vm->id = vm_count;
 	vm->vcpu_count = vcpu_count;
-	vm->rpc.state = rpc_state_idle;
+	vm->mailbox.state = mailbox_state_empty;
 
 	/* Do basic initialization of vcpus. */
 	for (i = 0; i < vcpu_count; i++) {