blob: 606848e32b9b892549026392eaa324bcc52180ff [file] [log] [blame] [edit]
/*
* Copyright 2018 The Hafnium Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "hf/load.h"
#include <stdbool.h>
#include "hf/arch/vm.h"
#include "hf/api.h"
#include "hf/boot_params.h"
#include "hf/check.h"
#include "hf/dlog.h"
#include "hf/layout.h"
#include "hf/memiter.h"
#include "hf/mm.h"
#include "hf/plat/console.h"
#include "hf/plat/iommu.h"
#include "hf/static_assert.h"
#include "hf/std.h"
#include "hf/vm.h"
#include "vmapi/hf/call.h"
alignas(PAGE_SIZE) static uint8_t tee_send_buffer[HF_MAILBOX_SIZE];
alignas(PAGE_SIZE) static uint8_t tee_recv_buffer[HF_MAILBOX_SIZE];
/**
* Copies data to an unmapped location by mapping it for write, copying the
* data, then unmapping it.
*
* The data is written so that it is available to all cores with the cache
* disabled. When switching to the partitions, the caching is initially disabled
* so the data must be available without the cache.
*/
static bool copy_to_unmapped(struct mm_stage1_locked stage1_locked, paddr_t to,
struct memiter *from_it, struct mpool *ppool)
{
const void *from = memiter_base(from_it);
size_t size = memiter_size(from_it);
paddr_t to_end = pa_add(to, size);
void *ptr;
ptr = mm_identity_map(stage1_locked, to, to_end, MM_MODE_W, ppool);
if (!ptr) {
return false;
}
memcpy_s(ptr, size, from, size);
arch_mm_flush_dcache(ptr, size);
CHECK(mm_unmap(stage1_locked, to, to_end, ppool));
return true;
}
static bool load_kernel(struct mm_stage1_locked stage1_locked, paddr_t begin,
paddr_t end, const struct manifest_vm *manifest_vm,
const struct memiter *cpio, struct mpool *ppool)
{
struct memiter kernel;
if (!cpio_get_file(cpio, &manifest_vm->kernel_filename, &kernel)) {
dlog_error("Could not find kernel file \"%s\".\n",
string_data(&manifest_vm->kernel_filename));
return false;
}
if (pa_difference(begin, end) < memiter_size(&kernel)) {
dlog_error("Kernel is larger than available memory.\n");
return false;
}
if (!copy_to_unmapped(stage1_locked, begin, &kernel, ppool)) {
dlog_error("Unable to copy kernel.\n");
return false;
}
return true;
}
/**
* Performs VM loading activities that are common between the primary and
* secondaries.
*/
static bool load_common(const struct manifest_vm *manifest_vm, struct vm *vm)
{
vm->smc_whitelist = manifest_vm->smc_whitelist;
/* Initialize architecture-specific features. */
arch_vm_features_set(vm);
return true;
}
/**
* Loads the primary VM.
*/
static bool load_primary(struct mm_stage1_locked stage1_locked,
const struct manifest_vm *manifest_vm,
const struct memiter *cpio,
const struct boot_params *params, struct mpool *ppool)
{
paddr_t primary_begin;
ipaddr_t primary_entry;
struct vm *vm;
struct vm_locked vm_locked;
struct vcpu_locked vcpu_locked;
size_t i;
bool ret;
if (manifest_vm->is_ffa_partition) {
primary_begin = pa_init(manifest_vm->sp.load_addr);
primary_entry = ipa_add(ipa_from_pa(primary_begin),
manifest_vm->sp.ep_offset);
} else {
primary_begin =
(manifest_vm->primary.boot_address ==
MANIFEST_INVALID_ADDRESS)
? layout_primary_begin()
: pa_init(manifest_vm->primary.boot_address);
primary_entry = ipa_from_pa(primary_begin);
}
paddr_t primary_end = pa_add(primary_begin, RSIZE_MAX);
/*
* Load the kernel if a filename is specified in the VM manifest.
* For an FF-A partition, kernel_filename is undefined indicating
* the partition package has already been loaded prior to Hafnium
* booting.
*/
if (!string_is_empty(&manifest_vm->kernel_filename)) {
if (!load_kernel(stage1_locked, primary_begin, primary_end,
manifest_vm, cpio, ppool)) {
dlog_error("Unable to load primary kernel.\n");
return false;
}
}
if (!vm_init_next(MAX_CPUS, ppool, &vm)) {
dlog_error("Unable to initialise primary VM.\n");
return false;
}
if (vm->id != HF_PRIMARY_VM_ID) {
dlog_error("Primary VM was not given correct ID.\n");
return false;
}
vm_locked = vm_lock(vm);
if (!load_common(manifest_vm, vm)) {
ret = false;
goto out;
}
if (params->device_mem_ranges_count == 0) {
/*
* Map 1TB of address space as device memory to, most likely,
* make all devices available to the primary VM.
*
* TODO: remove this once all targets provide valid ranges.
*/
dlog_warning(
"Device memory not provided, defaulting to 1 TB.\n");
if (!vm_identity_map(
vm_locked, pa_init(0),
pa_init(UINT64_C(1024) * 1024 * 1024 * 1024),
MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool, NULL)) {
dlog_error(
"Unable to initialise address space for "
"primary VM.\n");
ret = false;
goto out;
}
}
/* Map normal memory as such to permit caching, execution, etc. */
for (i = 0; i < params->mem_ranges_count; ++i) {
if (!vm_identity_map(vm_locked, params->mem_ranges[i].begin,
params->mem_ranges[i].end,
MM_MODE_R | MM_MODE_W | MM_MODE_X, ppool,
NULL)) {
dlog_error(
"Unable to initialise memory for primary "
"VM.\n");
ret = false;
goto out;
}
}
/* Map device memory as such to prevent execution, speculation etc. */
for (i = 0; i < params->device_mem_ranges_count; ++i) {
if (!vm_identity_map(
vm_locked, params->device_mem_ranges[i].begin,
params->device_mem_ranges[i].end,
MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool, NULL)) {
dlog("Unable to initialise device memory for primary "
"VM.\n");
ret = false;
goto out;
}
}
if (!vm_unmap_hypervisor(vm_locked, ppool)) {
dlog_error("Unable to unmap hypervisor from primary VM.\n");
ret = false;
goto out;
}
if (!plat_iommu_unmap_iommus(vm_locked, ppool)) {
dlog_error("Unable to unmap IOMMUs from primary VM.\n");
ret = false;
goto out;
}
dlog_info("Loaded primary VM with %u vCPUs, entry at %#x.\n",
vm->vcpu_count, pa_addr(primary_begin));
vcpu_locked = vcpu_lock(vm_get_vcpu(vm, 0));
vcpu_on(vcpu_locked, primary_entry, params->kernel_arg);
vcpu_unlock(&vcpu_locked);
ret = true;
out:
vm_unlock(&vm_locked);
return ret;
}
/*
* Loads a secondary VM.
*/
static bool load_secondary(struct mm_stage1_locked stage1_locked,
paddr_t mem_begin, paddr_t mem_end,
const struct manifest_vm *manifest_vm,
const struct memiter *cpio, struct mpool *ppool)
{
struct vm *vm;
struct vm_locked vm_locked;
struct vcpu *vcpu;
ipaddr_t secondary_entry;
bool ret;
/*
* Load the kernel if a filename is specified in the VM manifest.
* For an FF-A partition, kernel_filename is undefined indicating
* the partition package has already been loaded prior to Hafnium
* booting.
*/
if (!string_is_empty(&manifest_vm->kernel_filename)) {
if (!load_kernel(stage1_locked, mem_begin, mem_end, manifest_vm,
cpio, ppool)) {
dlog_error("Unable to load kernel.\n");
return false;
}
}
if (!vm_init_next(manifest_vm->secondary.vcpu_count, ppool, &vm)) {
dlog_error("Unable to initialise VM.\n");
return false;
}
if (!load_common(manifest_vm, vm)) {
return false;
}
vm_locked = vm_lock(vm);
/* Grant the VM access to the memory. */
if (!vm_identity_map(vm_locked, mem_begin, mem_end,
MM_MODE_R | MM_MODE_W | MM_MODE_X, ppool,
&secondary_entry)) {
dlog_error("Unable to initialise memory.\n");
ret = false;
goto out;
}
dlog_info("Loaded with %u vCPUs, entry at %#x.\n",
manifest_vm->secondary.vcpu_count, pa_addr(mem_begin));
if (manifest_vm->is_ffa_partition) {
secondary_entry =
ipa_add(secondary_entry, manifest_vm->sp.ep_offset);
}
vcpu = vm_get_vcpu(vm, 0);
vcpu_secondary_reset_and_start(vcpu, secondary_entry,
pa_difference(mem_begin, mem_end));
ret = true;
out:
vm_unlock(&vm_locked);
return ret;
}
/**
* Try to find a memory range of the given size within the given ranges, and
* remove it from them. Return true on success, or false if no large enough
* contiguous range is found.
*/
static bool carve_out_mem_range(struct mem_range *mem_ranges,
size_t mem_ranges_count, uint64_t size_to_find,
paddr_t *found_begin, paddr_t *found_end)
{
size_t i;
/*
* TODO(b/116191358): Consider being cleverer about how we pack VMs
* together, with a non-greedy algorithm.
*/
for (i = 0; i < mem_ranges_count; ++i) {
if (size_to_find <=
pa_difference(mem_ranges[i].begin, mem_ranges[i].end)) {
/*
* This range is big enough, take some of it from the
* end and reduce its size accordingly.
*/
*found_end = mem_ranges[i].end;
*found_begin = pa_init(pa_addr(mem_ranges[i].end) -
size_to_find);
mem_ranges[i].end = *found_begin;
return true;
}
}
return false;
}
/**
* Given arrays of memory ranges before and after memory was removed for
* secondary VMs, add the difference to the reserved ranges of the given update.
* Return true on success, or false if there would be more than MAX_MEM_RANGES
* reserved ranges after adding the new ones.
* `before` and `after` must be arrays of exactly `mem_ranges_count` elements.
*/
static bool update_reserved_ranges(struct boot_params_update *update,
const struct mem_range *before,
const struct mem_range *after,
size_t mem_ranges_count)
{
size_t i;
for (i = 0; i < mem_ranges_count; ++i) {
if (pa_addr(after[i].begin) > pa_addr(before[i].begin)) {
if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
dlog_error(
"Too many reserved ranges after "
"loading secondary VMs.\n");
return false;
}
update->reserved_ranges[update->reserved_ranges_count]
.begin = before[i].begin;
update->reserved_ranges[update->reserved_ranges_count]
.end = after[i].begin;
update->reserved_ranges_count++;
}
if (pa_addr(after[i].end) < pa_addr(before[i].end)) {
if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
dlog_error(
"Too many reserved ranges after "
"loading secondary VMs.\n");
return false;
}
update->reserved_ranges[update->reserved_ranges_count]
.begin = after[i].end;
update->reserved_ranges[update->reserved_ranges_count]
.end = before[i].end;
update->reserved_ranges_count++;
}
}
return true;
}
/*
* Loads alls VMs from the manifest.
*/
bool load_vms(struct mm_stage1_locked stage1_locked,
const struct manifest *manifest, const struct memiter *cpio,
const struct boot_params *params,
struct boot_params_update *update, struct mpool *ppool)
{
struct vm *primary;
struct vm *tee;
struct mem_range mem_ranges_available[MAX_MEM_RANGES];
struct vm_locked primary_vm_locked;
size_t i;
bool success = true;
if (!load_primary(stage1_locked, &manifest->vm[HF_PRIMARY_VM_INDEX],
cpio, params, ppool)) {
dlog_error("Unable to load primary VM.\n");
return false;
}
/*
* Initialise the dummy VM which represents TrustZone, and set up its
* RX/TX buffers.
*/
tee = vm_init(HF_TEE_VM_ID, 0, ppool);
CHECK(tee != NULL);
tee->mailbox.send = &tee_send_buffer;
tee->mailbox.recv = &tee_recv_buffer;
static_assert(
sizeof(mem_ranges_available) == sizeof(params->mem_ranges),
"mem_range arrays must be the same size for memcpy.");
static_assert(sizeof(mem_ranges_available) < 500,
"This will use too much stack, either make "
"MAX_MEM_RANGES smaller or change this.");
memcpy_s(mem_ranges_available, sizeof(mem_ranges_available),
params->mem_ranges, sizeof(params->mem_ranges));
/* Round the last addresses down to the page size. */
for (i = 0; i < params->mem_ranges_count; ++i) {
mem_ranges_available[i].end = pa_init(align_down(
pa_addr(mem_ranges_available[i].end), PAGE_SIZE));
}
primary = vm_find(HF_PRIMARY_VM_ID);
primary_vm_locked = vm_lock(primary);
for (i = 0; i < manifest->vm_count; ++i) {
const struct manifest_vm *manifest_vm = &manifest->vm[i];
ffa_vm_id_t vm_id = HF_VM_ID_OFFSET + i;
uint64_t mem_size;
paddr_t secondary_mem_begin;
paddr_t secondary_mem_end;
if (vm_id == HF_PRIMARY_VM_ID) {
continue;
}
dlog_info("Loading VM%d: %s.\n", (int)vm_id,
manifest_vm->debug_name);
mem_size = align_up(manifest_vm->secondary.mem_size, PAGE_SIZE);
if (manifest_vm->is_ffa_partition) {
secondary_mem_begin =
pa_init(manifest_vm->sp.load_addr);
secondary_mem_end =
pa_init(manifest_vm->sp.load_addr + mem_size);
} else if (!carve_out_mem_range(mem_ranges_available,
params->mem_ranges_count,
mem_size, &secondary_mem_begin,
&secondary_mem_end)) {
dlog_error("Not enough memory (%u bytes).\n", mem_size);
continue;
}
if (!load_secondary(stage1_locked, secondary_mem_begin,
secondary_mem_end, manifest_vm, cpio,
ppool)) {
dlog_error("Unable to load VM.\n");
continue;
}
/* Deny the primary VM access to this memory. */
if (!vm_unmap(primary_vm_locked, secondary_mem_begin,
secondary_mem_end, ppool)) {
dlog_error(
"Unable to unmap secondary VM from primary "
"VM.\n");
success = false;
break;
}
}
vm_unlock(&primary_vm_locked);
if (!success) {
return false;
}
/*
* Add newly reserved areas to update params by looking at the
* difference between the available ranges from the original params and
* the updated mem_ranges_available. We assume that the number and order
* of available ranges is the same, i.e. we don't remove any ranges
* above only make them smaller.
*/
return update_reserved_ranges(update, params->mem_ranges,
mem_ranges_available,
params->mem_ranges_count);
}