diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000..c83c7f7
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,24 @@
+BasedOnStyle: Google
+
+IndentWidth: 8
+ContinuationIndentWidth: 8
+UseTab: Always
+
+BreakBeforeBraces: Linux
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+IndentCaseLabels: false
+
+IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex:    '^<[[:alnum:].]+>'
+    Priority: 1
+  - Regex:    '^"hf/arch/'
+    Priority: 2
+  - Regex:    '^"hf/'
+    Priority: 3
+  - Regex:    '^"vmapi/'
+    Priority: 4
+  - Regex:    '.*'
+    Priority: 5
diff --git a/.clang-tidy b/.clang-tidy
new file mode 100644
index 0000000..38278ee
--- /dev/null
+++ b/.clang-tidy
@@ -0,0 +1,4 @@
+Checks: 'readability-*,portability-*,performance-*,misc-*,bugprone-*,modernize-*,google-runtime-int,-modernize-deprecated-headers,-clang-analyzer-valist.Uninitialized,-readability-magic-numbers'
+HeaderFilterRegex: '^(?!third_party).+'
+FormatStyle: file
+WarningsAsErrors: '*'
diff --git a/.gitignore b/.gitignore
index 8fd2089..9e90f42 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,9 @@
-*.ko
-*.mod.c
-*.o
-.*o.cmd
-.tmp_versions/
-Module.symvers
-modules.order
+out/
+.repo/
+
+# Ignore project/ subfolders which are not part of the public Hafnium tree.
+# When these are checked out using a `repo` manifest Git otherwise treats them
+# as untracked files and the repository as dirty.
+!project/
+project/*/
+!project/reference/
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..13fb56f
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,16 @@
+[submodule "prebuilts"]
+	path = prebuilts
+	url = https://hafnium.googlesource.com/hafnium/prebuilts
+	shallow = true
+[submodule "project/reference"]
+	path = project/reference
+	url = https://hafnium.googlesource.com/hafnium/project/reference
+	shallow = true
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://hafnium.googlesource.com/hafnium/third_party/googletest
+	shallow = true
+[submodule "third_party/linux"]
+	path = third_party/linux
+	url = https://hafnium.googlesource.com/hafnium/third_party/linux
+	shallow = true
diff --git a/.gn b/.gn
new file mode 100644
index 0000000..b5eecc7
--- /dev/null
+++ b/.gn
@@ -0,0 +1,19 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The location of the build configuration file.
+buildconfig = "//build/BUILDCONFIG.gn"
+
+# Force use of Python 3
+script_executable = "python3"
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json
new file mode 100644
index 0000000..6a99fb2
--- /dev/null
+++ b/.vscode/c_cpp_properties.json
@@ -0,0 +1,17 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**"
+            ],
+            "defines": [],
+            "compilerPath": "${workspaceFolder}/prebuilts/linux-x64/clang/bin/clang",
+            "cStandard": "c11",
+            "cppStandard": "c++17",
+            "intelliSenseMode": "clang-x64",
+            "compileCommands": "${workspaceFolder}/out/reference/compile_commands.json"
+        }
+    ],
+    "version": 4
+}
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000..1e0e217
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,29 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "(gdb) QEMU",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "out/reference/qemu_aarch64_clang/hafnium.elf",
+            "miDebuggerServerAddress": "localhost:1234",
+            "MIMode": "gdb",
+            "miDebuggerPath": "/usr/bin/gdb-multiarch",
+            "cwd": "${workspaceRoot}",
+            "setupCommands": [
+                {
+                "description": "Enable pretty-printing for gdb",
+                "text": "-enable-pretty-printing",
+                "ignoreFailures": true
+                },
+                {
+                    // Update this with whatever VMs you want to debug.
+                    "text": "add-symbol-file ${workspaceRoot}/out/reference/qemu_aarch64_vm_clang/obj/test/vmapi/primary_with_secondaries/services/service_vm3.elf 0x43d00000+0xc4",
+                }
+            ]
+        },
+    ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..3475316
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,24 @@
+{
+    "files.associations": {
+        "constants.h": "c",
+        "hftest.h": "c",
+        "primary_with_secondary.h": "c",
+        "cpu.h": "c",
+        "call.h": "c",
+        "mpool.h": "c",
+        "atomic": "c",
+        "assert.h": "c",
+        "dlog.h": "c",
+        "stdarg.h": "c",
+        "primary_only.h": "c",
+        "fdt_handler.h": "c",
+        "spinlock.h": "c",
+        "offsets.h": "c",
+        "barriers.h": "c",
+        "spci.h": "c",
+        "spci_internal.h": "c",
+        "interrupts_gicv3.h": "c",
+        "interrupts.h": "c"
+    },
+    "C_Cpp.errorSquiggles": "Disabled"
+}
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
new file mode 100644
index 0000000..23d9330
--- /dev/null
+++ b/.vscode/tasks.json
@@ -0,0 +1,51 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "make",
+            "type": "shell",
+            "command": "make",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "problemMatcher": []
+        },
+        {
+            "label": "format",
+            "type": "shell",
+            "command": "make format",
+            "problemMatcher": []
+        },
+        {
+            "label": "check",
+            "type": "shell",
+            "command": "make check",
+            "problemMatcher": []
+        },
+        {
+            "label": "build.sh",
+            "type": "shell",
+            "command": "kokoro/ubuntu/build.sh",
+            "problemMatcher": []
+        },
+        {
+            "label": "test",
+            "type": "shell",
+            "command": "make && kokoro/ubuntu/test.sh"
+        },
+        {
+            "label": "test on FVP",
+            "type": "shell",
+            "command": "make && kokoro/ubuntu/test.sh --fvp"
+        },
+        {
+            "label": "push",
+            "type": "shell",
+            "command": "git push origin HEAD:refs/for/master",
+            "problemMatcher": []
+        }
+    ]
+}
diff --git a/AUTHORS b/AUTHORS
index 7c577d2..2d9ea28 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -5,4 +5,5 @@
 # of contributors, see the revision history in source control.
 #
 # Please keep the list sorted alphabetically.
+Arm Ltd.
 Google LLC
diff --git a/BUILD.gn b/BUILD.gn
index 6c7d688..7b1bb73 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -1,4 +1,4 @@
-# Copyright 2019 The Hafnium Authors.
+# Copyright 2018 The Hafnium Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,10 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import("//build/linux/linux.gni")
+# The root of the build redirects to a project build file so each project can
+# select the artifacts it needs to build.
 
-linux_kernel_module("linux") {
-  module_name = "hafnium"
-  kernel_target = "//third_party/linux:linux"
-  kernel_dir = "//third_party/linux"
+# The root of the build.
+group("root") {
+  deps = [
+    "//project/${project}:root",
+  ]
+}
+
+# The root of the build for test artifacts.
+group("test_root") {
+  testonly = true
+
+  deps = [
+    "//project/${project}:test_root",
+  ]
+}
+
+group("update_prebuilts") {
+  deps = [
+    "//third_party/linux",
+  ]
+}
+
+group("default") {
+  testonly = true
+  deps = [
+    ":root",
+    ":test_root",
+  ]
 }
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..1efa7e5
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,54 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution;
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Style guide
+
+Submissions should follow the Hafnium [style guide](docs/StyleGuide.md).
+
+## Code reviews
+
+All submissions, including submissions by project members, require review. We
+use [Gerrit](https://hafnium-review.googlesource.com) for this purpose.
+
+To submit a change:
+
+1.  Create an account in the
+    [Gerrit UI](https://hafnium-review.googlesource.com).
+2.  Follow the [getting started](docs/GettingStarted.md) instructions to clone
+    the Hafnium repositories and set up the necessary commit hook.
+3.  Make your change.
+4.  Run our autoformatter with `make format`.
+5.  Commit as usual. If you make a change in a submodule you will also need to
+    commit a change in the main repository to update the submodule version.
+6.  Run the [tests](docs/Testing.md) and other presubmit checks with
+    `kokoro/ubuntu/build.sh`, ensure they all pass.
+7.  Upload the change to Gerrit with `git push origin HEAD:refs/for/master`. If
+    you have changed submodules then you'll need to push them as well.
+8.  If you changed submodules, then add a matching 'topic' from the Gerrit UI
+    for all your changes (submodules and the main repository) so that they can
+    be reviewed and submitted together.
+9.  Wait 10-15 minutes for our presubmit tests to run, and make sure a 'Kokoro
+    +1' comment shows up in Gerrit indicating that they have passed. If not,
+    follow the links to find the errors, fix them and try again.
+10. From the Gerrit UI add one or more reviewers. Looking at who has modified
+    the same files frequently recently is usually a good way to pick a reviewer,
+    but if you're not sure then you can add hafnium-team@google.com.
+
+## Community Guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/).
diff --git a/LICENSE b/LICENSE
index d159169..d645695 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,339 +1,202 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-                            Preamble
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
+   1. Definitions.
 
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
 
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
 
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
 
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
 
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
 
-  The precise terms and conditions for copying, distribution and
-modification follow.
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
 
-                    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
 
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
 
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
 
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
 
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
 
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
 
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
 
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
 
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
 
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
 
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
 
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
 
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
 
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
 
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
 
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
 
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
+   END OF TERMS AND CONDITIONS
 
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
+   APPENDIX: How to apply the Apache License to your work.
 
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
 
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
+   Copyright [yyyy] [name of copyright owner]
 
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
 
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
+       http://www.apache.org/licenses/LICENSE-2.0
 
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-                            NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/Makefile b/Makefile
index 0add537..9f39365 100644
--- a/Makefile
+++ b/Makefile
@@ -1,45 +1,125 @@
 # Copyright 2018 The Hafnium Authors.
 #
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# version 2 as published by the Free Software Foundation.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-# By default, assume this was checked out as a submodule of the Hafnium repo
-# and that Linux was checked out along side that checkout. These paths can be
-# overridden if that assumption is incorrect.
-HAFNIUM_PATH ?= $(CURDIR)/../..
+# Select the project to build.
+PROJECT ?= reference
 
-ifneq ($(KERNELRELEASE),)
+# If HAFNIUM_HERMETIC_BUILD is "true" (not default), invoke `make` inside
+# a container. The 'run_in_container.sh' script will set the variable value to
+# 'inside' to avoid recursion.
+ifeq ($(HAFNIUM_HERMETIC_BUILD),true)
 
-obj-m += hafnium.o
+# TODO: This is not ideal as (a) we invoke the container once per command-line
+# target, and (b) we cannot pass `make` arguments to the script. We could
+# consider creating a bash alias for `make` to invoke the script directly.
 
-hafnium-y += main.o
-hafnium-y += vmlib/aarch64/call.o
-hafnium-y += vmlib/spci.o
-
-ccflags-y = -I$(HAFNIUM_PATH)/inc/vmapi -I$(M)/inc
-
-else
-
-KERNEL_PATH ?= $(HAFNIUM_PATH)/third_party/linux
-ARCH ?= arm64
-CROSS_COMPILE ?= aarch64-linux-gnu-
-CHECKPATCH ?= $(KERNEL_PATH)/scripts/checkpatch.pl -q
-
+# Need to define at least one non-default target.
 all:
-	cp -r $(HAFNIUM_PATH)/vmlib/ $(CURDIR)
-	make -C $(KERNEL_PATH) HAFNIUM_PATH=$(HAFNIUM_PATH) M=$(CURDIR) O=$(O) ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) modules
+	@$(CURDIR)/build/run_in_container.sh make PROJECT=$(PROJECT) $@
 
+# Catch-all target.
+.DEFAULT:
+	@$(CURDIR)/build/run_in_container.sh make PROJECT=$(PROJECT) $@
+
+else  # HAFNIUM_HERMETIC_BUILD
+
+# Set path to prebuilts used in the build.
+UNNAME_S := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+PREBUILTS := $(CURDIR)/prebuilts/$(UNNAME_S)-x64
+GN ?= $(PREBUILTS)/gn/gn
+NINJA ?= $(PREBUILTS)/ninja/ninja
+export PATH := $(PREBUILTS)/clang/bin:$(PATH)
+
+
+CHECKPATCH := $(CURDIR)/third_party/linux/scripts/checkpatch.pl \
+	--ignore BRACES,SPDX_LICENSE_TAG,VOLATILE,SPLIT_STRING,AVOID_EXTERNS,USE_SPINLOCK_T,NEW_TYPEDEFS,INITIALISED_STATIC,FILE_PATH_CHANGES,EMBEDDED_FUNCTION_NAME,SINGLE_STATEMENT_DO_WHILE_MACRO,MACRO_WITH_FLOW_CONTROL --quiet
+
+# Specifies the grep pattern for ignoring specific files in checkpatch.
+# C++ headers, *.hh, are automatically excluded.
+# Separate the different items in the list with a grep or (\|).
+# debug_el1.c : uses XMACROS, which checkpatch doesn't understand.
+# perfmon.c : uses XMACROS, which checkpatch doesn't understand.
+# feature_id.c : uses XMACROS, which checkpatch doesn't understand.
+CHECKPATCH_IGNORE := "src/arch/aarch64/hypervisor/debug_el1.c\|src/arch/aarch64/hypervisor/perfmon.c\|src/arch/aarch64/hypervisor/feature_id.c"
+
+OUT ?= out/$(PROJECT)
+OUT_DIR = out/$(PROJECT)
+
+.PHONY: all
+all: $(OUT_DIR)/build.ninja
+	@$(NINJA) -C $(OUT_DIR)
+
+$(OUT_DIR)/build.ninja:
+	@$(GN) --export-compile-commands gen --args='project="$(PROJECT)"' $(OUT_DIR)
+
+.PHONY: clean
 clean:
-	make -C $(KERNEL_PATH) HAFNIUM_PATH=$(HAFNIUM_PATH) M=$(CURDIR) O=$(O) clean
-	rm -rf vmlib
+	@$(NINJA) -C $(OUT_DIR) -t clean
 
+.PHONY: clobber
+clobber:
+	rm -rf $(OUT)
+
+# see .clang-format.
+.PHONY: format
+format:
+	@echo "Formatting..."
+	@find src/ -name \*.c -o -name \*.cc -o -name \*.h | xargs -r clang-format -style file -i
+	@find inc/ -name \*.c -o -name \*.cc -o -name \*.h | xargs -r clang-format -style file -i
+	@find test/ -name \*.c -o -name \*.cc -o -name \*.h | xargs -r clang-format -style file -i
+	@find project/ -name \*.c -o -name \*.cc -o -name \*.h | xargs -r clang-format -style file -i
+	@find . \( -name \*.gn -o -name \*.gni \) | xargs -n1 $(GN) format
+
+.PHONY: checkpatch
 checkpatch:
-	$(CHECKPATCH) -f main.c
+	@find src/ -name \*.c -o -name \*.h | grep -v $(CHECKPATCH_IGNORE) | xargs $(CHECKPATCH) -f
+	@find inc/ -name \*.c -o -name \*.h | grep -v $(CHECKPATCH_IGNORE) | xargs $(CHECKPATCH) -f
+	# TODO: enable for test/
+	@find project/ -name \*.c -o -name \*.h | grep -v $(CHECKPATCH_IGNORE) | xargs $(CHECKPATCH) -f
 
-endif
+# see .clang-tidy.
+.PHONY: tidy
+tidy: $(OUT_DIR)/build.ninja
+	@$(NINJA) -C $(OUT_DIR)
+	@echo "Tidying..."
+	# TODO: enable readability-magic-numbers once there are fewer violations.
+	# TODO: enable for c++ tests as it currently gives spurious errors.
+	@find src/ \( -name \*.c \) | xargs clang-tidy -p $(OUT_DIR) -fix
+	@find test/ \( -name \*.c \) | xargs clang-tidy -p $(OUT_DIR) -fix
+
+.PHONY: check
+check: $(OUT_DIR)/build.ninja
+	@$(NINJA) -C $(OUT_DIR)
+	@echo "Checking..."
+	# TODO: enable for c++ tests as it currently gives spurious errors.
+	@find src/ \( -name \*.c \) | xargs clang-check -p $(OUT_DIR) -analyze -fix-what-you-can
+	@find test/ \( -name \*.c \) | xargs clang-check -p $(OUT_DIR) -analyze -fix-what-you-can
+
+.PHONY: license
+license:
+	@find src/ -name \*.S -o -name \*.c -o -name \*.cc -o -name \*.h -o -name \*.dts | xargs -n1 python build/license.py --style c
+	@find inc/ -name \*.S -o -name \*.c -o -name \*.cc -o -name \*.h -o -name \*.dts | xargs -n1 python build/license.py --style c
+	@find test/ -name \*.S -o -name \*.c -o -name \*.cc -o -name \*.h -o -name \*.dts | xargs -n1 python build/license.py --style c
+	@find build/ -name \*.py| xargs -n1 python build/license.py --style hash
+	@find test/ -name \*.py| xargs -n1 python build/license.py --style hash
+	@find . \( -name \*.gn -o -name \*.gni \) | xargs -n1 python build/license.py --style hash
+
+.PHONY: update-prebuilts
+update-prebuilts: prebuilts/linux-aarch64/linux/vmlinuz
+
+prebuilts/linux-aarch64/linux/vmlinuz: $(OUT_DIR)/build.ninja
+	@$(NINJA) -C $(OUT_DIR) "third_party/linux"
+	cp out/reference/obj/third_party/linux/linux.bin $@
+
+endif  # HAFNIUM_HERMETIC_BUILD
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5ee74bf
--- /dev/null
+++ b/README.md
@@ -0,0 +1,28 @@
+# Hafnium
+
+Hafnium is a hypervisor, initially supporting aarch64 (64-bit Armv8 CPUs).
+
+Get in touch and keep up-to-date at
+[hafnium-discuss@googlegroups.com](https://groups.google.com/forum/#!forum/hafnium-discuss).
+
+## Getting started
+
+To jump in and build Hafnium, follow the
+[getting started](docs/GettingStarted.md) instructions.
+
+If you want to contribute to the project, see details of
+[how we accept contributions](CONTRIBUTING.md).
+
+## Documentation
+
+More documentation is available on:
+
+*   [Hafnium architecture](docs/Architecture.md)
+*   [Code structure](docs/CodeStructure.md)
+*   [Hafnium test infrastructure](docs/Testing.md)
+*   [Running Hafnium under the Arm Fixed Virtual Platform](docs/FVP.md)
+*   [How to build a RAM disk containing VMs for Hafnium to run](docs/HafniumRamDisk.md)
+*   [Building Hafnium hermetically with Docker](docs/HermeticBuild.md)
+*   [The interface Hafnium provides to VMs](docs/VmInterface.md)
+*   [Scheduler VM expectations](docs/SchedulerExpectations.md)
+*   [Hafnium coding style](docs/StyleGuide.md)
diff --git a/build/BUILD.gn b/build/BUILD.gn
new file mode 100644
index 0000000..8ce649c
--- /dev/null
+++ b/build/BUILD.gn
@@ -0,0 +1,58 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/platform.gni")
+
+# Default language and error reporting configuration.
+config("compiler_defaults") {
+  cflags = [
+    "-g",
+    "-Wall",
+    "-O2",
+
+    #"-Wextra",
+    "-Wpedantic",
+    "-Werror",
+
+    "-fstack-protector-all",
+  ]
+
+  cflags_c = [ "-std=c11" ]
+
+  cflags_cc = [ "-std=c++2a" ]
+}
+
+# Platform configuration.
+config("platform") {
+  include_dirs = [
+    "//inc",
+    "//inc/vmapi",
+    "//src/arch/${plat_arch}/inc",
+
+    # Auto-generated headers using the 'offset_size_header' build rule.
+    "${root_gen_dir}/offset_size_header",
+  ]
+
+  defines = [
+    "HEAP_PAGES=${plat_heap_pages}",
+    "MAX_CPUS=${plat_max_cpus}",
+    "MAX_VMS=${plat_max_vms}",
+  ]
+
+  if (is_debug) {
+    defines += [ "DEBUG=1" ]
+  } else {
+    defines += [ "DEBUG=0" ]
+  }
+}
diff --git a/build/BUILDCONFIG.gn b/build/BUILDCONFIG.gn
new file mode 100644
index 0000000..134ddcd
--- /dev/null
+++ b/build/BUILDCONFIG.gn
@@ -0,0 +1,72 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration of the build toolchain.
+declare_args() {
+  # The name of the project being built.
+  project = "reference"
+
+  # Enable extra debugging.
+  is_debug = true
+
+  # Whether to build against the platform for embedded images consisting of
+  # include paths and defines. This is also used for host targets that simulate
+  # an embedded image.
+  use_platform = false
+}
+
+# Check that we support the attempted build.
+assert(host_os == "linux", "Only linux builds are currently supported.")
+
+# Setup the standard variables.
+if (target_os == "") {
+  target_os = host_os
+}
+if (target_cpu == "") {
+  target_cpu = host_cpu
+}
+if (current_os == "") {
+  current_os = target_os
+}
+if (current_cpu == "") {
+  current_cpu = target_cpu
+}
+
+assert(target_os == host_os, "Cross compiles not yet supported.")
+assert(target_cpu == host_cpu, "Cross compiles not yet supported.")
+
+# All binary targets will get this list of configs by default.
+_shared_binary_target_configs = [ "//build:compiler_defaults" ]
+
+# If it's not building a host utility, it's building against the platform so apply the configuration.
+if (use_platform) {
+  _shared_binary_target_configs += [ "//build:platform" ]
+}
+
+# Apply that default list to the binary target types.
+set_defaults("executable") {
+  configs = _shared_binary_target_configs
+}
+set_defaults("static_library") {
+  configs = _shared_binary_target_configs
+}
+set_defaults("shared_library") {
+  configs = _shared_binary_target_configs
+}
+set_defaults("source_set") {
+  configs = _shared_binary_target_configs
+}
+
+# The default toolchain is the target toolchain for building utilities and tests.
+set_default_toolchain("//build/toolchain:host_clang")
diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
new file mode 100644
index 0000000..1eb6b2d
--- /dev/null
+++ b/build/docker/Dockerfile
@@ -0,0 +1,42 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Base container image to be uploaded to Google Cloud Platform as
+# "eu.gcr.io/hafnium-build/hafnium_ci". Each user derives their own container
+# with local user permissions from this base image. It should contain everything
+# needed to build and test Hafnium.
+#
+FROM launcher.gcr.io/google/ubuntu1804
+MAINTAINER Hafnium Team <hafnium-team+build@google.com>
+
+# Install dependencies. Clear APT cache at the end to save space.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN 	apt-get update \
+	&& apt-get install -y \
+		bc                             `# for Linux headers` \
+		bison \
+		build-essential \
+		cpio \
+		flex \
+		git \
+		libpixman-1-0                  `# for QEMU` \
+		libsdl2-2.0-0                  `# for QEMU` \
+		libglib2.0                     `# for QEMU` \
+		libssl-dev                     `# for Linux headers` \
+		python \
+		python-git                     `# for Linux checkpatch` \
+		python-ply                     `# for Linux checkpatch` \
+		strace                         `# for strace_open.sh` \
+	&& rm -rf /var/lib/apt/lists/*
diff --git a/build/docker/Dockerfile.local b/build/docker/Dockerfile.local
new file mode 100644
index 0000000..67eb92f
--- /dev/null
+++ b/build/docker/Dockerfile.local
@@ -0,0 +1,35 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Container derived from the base image hosted on Google Cloud Platform.
+# It sets up a user with the same UID/GID as the local user, so that generated
+# files can be accessed by the host.
+# Please keep the diff between base and local images as small as possible.
+#
+FROM eu.gcr.io/hafnium-build/hafnium_ci
+ARG LOCAL_UID=1000
+ARG LOCAL_GID=1000
+
+RUN	addgroup \
+		--gid "${LOCAL_GID}" \
+		hafnium \
+	&& adduser \
+		-disabled-password \
+		-gecos "" \
+		--uid "${LOCAL_UID}" \
+		--shell "/bin/bash" \
+		--ingroup hafnium \
+		hafnium
+USER hafnium
\ No newline at end of file
diff --git a/build/docker/build.sh b/build/docker/build.sh
new file mode 100755
index 0000000..6ba9467
--- /dev/null
+++ b/build/docker/build.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/common.inc"
+
+${DOCKER} build \
+	--pull \
+	-f "${SCRIPT_DIR}/Dockerfile" \
+	-t "${CONTAINER_TAG}" \
+	"${SCRIPT_DIR}"
diff --git a/build/docker/common.inc b/build/docker/common.inc
new file mode 100644
index 0000000..0d1e1db
--- /dev/null
+++ b/build/docker/common.inc
@@ -0,0 +1,21 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CONTAINER_TAG="eu.gcr.io/hafnium-build/hafnium_ci"
+
+if [[ ! -v DOCKER ]]
+then
+	DOCKER="$(which docker)" \
+		|| (echo "ERROR: Could not find Docker binary" 1>&2; exit 1)
+fi
\ No newline at end of file
diff --git a/build/docker/publish.sh b/build/docker/publish.sh
new file mode 100755
index 0000000..57ad16a
--- /dev/null
+++ b/build/docker/publish.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/common.inc"
+
+# Requires for the user to be an owner of the GCP 'hafnium-build' project and
+# have gcloud SDK installed and authenticated.
+
+${DOCKER} push "${CONTAINER_TAG}"
diff --git a/build/image/check_elf.py b/build/image/check_elf.py
new file mode 100644
index 0000000..1dc0cec
--- /dev/null
+++ b/build/image/check_elf.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Check ELF file for assembly-level regressions.
+
+Objdumps the given ELF file and detects known assembly patterns, checking for
+regressions on bugs such as CPU erratas. Throws an exception if a broken pattern
+is detected.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+HF_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+CLANG_ROOT = os.path.join(HF_ROOT, "prebuilts", "linux-x64", "clang")
+OBJDUMP = os.path.join(CLANG_ROOT, "bin", "llvm-objdump")
+
+def check_eret_speculation_barrier(objdump_stdout):
+	"""
+	Some ARM64 CPUs speculatively execute instructions after ERET.
+	Check that every ERET is followed by DSB NSH and ISB.
+	"""
+	found_eret = False
+
+	STATE_DEFAULT = 1
+	STATE_EXPECT_DSB_NSH = 2
+	STATE_EXPECT_ISB = 3
+
+	REGEX_ERET = re.compile(r"^\s*[0-9a-f]+:\s*e0 03 9f d6\s+eret$")
+	REGEX_DSB_NSH = re.compile(r"^\s*[0-9a-f]+:\s*9f 37 03 d5\s*dsb\s+nsh$")
+	REGEX_ISB = re.compile(r"^\s*[0-9a-f]+:\s*df 3f 03 d5\s+isb$")
+
+	state = STATE_DEFAULT
+	for line in objdump_stdout:
+		if state == STATE_DEFAULT:
+			if re.match(REGEX_ERET, line):
+				found_eret = True
+				state = STATE_EXPECT_DSB_NSH
+		elif state == STATE_EXPECT_DSB_NSH:
+			if re.match(REGEX_DSB_NSH, line):
+				state = STATE_EXPECT_ISB
+			else:
+				raise Exception("ERET not followed by DSB NSH")
+		elif state == STATE_EXPECT_ISB:
+			if re.match(REGEX_ISB, line):
+				state = STATE_DEFAULT
+			else:
+				raise Exception("ERET not followed by ISB")
+
+	# Ensure that at least one instance was found, otherwise the regexes are
+	# probably wrong.
+	if not found_eret:
+		raise Exception("Could not find any ERET instructions")
+
+def Main():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("input_elf",
+		help="ELF file to analyze")
+	parser.add_argument("stamp_file",
+		help="file to be touched if successful")
+	args = parser.parse_args()
+
+	objdump_stdout = subprocess.check_output([
+		OBJDUMP, "-d", args.input_elf ])
+	objdump_stdout = objdump_stdout.decode("utf-8").splitlines()
+
+	check_eret_speculation_barrier(objdump_stdout)
+
+	# Touch `stamp_file`.
+	with open(args.stamp_file, "w"):
+		pass
+
+	return 0
+
+if __name__ == "__main__":
+	sys.exit(Main())
diff --git a/build/image/convert_to_binary.py b/build/image/convert_to_binary.py
new file mode 100644
index 0000000..978d359
--- /dev/null
+++ b/build/image/convert_to_binary.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Convert a file to binary format.
+
+Calls objcopy to convert a file into raw binary format.
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+
+HF_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+CLANG_ROOT = os.path.join(HF_ROOT, "prebuilts", "linux-x64", "clang")
+OBJCOPY = os.path.join(CLANG_ROOT, "bin", "llvm-objcopy")
+
+def Main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+    subprocess.check_call([
+        OBJCOPY, "-O", "binary", args.input, args.output
+    ])
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(Main())
diff --git a/build/image/dtc.py b/build/image/dtc.py
new file mode 100755
index 0000000..c0caff8
--- /dev/null
+++ b/build/image/dtc.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wrapper around Device Tree Compiler (dtc)"""
+
+import argparse
+import os
+import subprocess
+import sys
+
+HF_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+DTC_ROOT = os.path.join(HF_ROOT, "prebuilts", "linux-x64", "dtc")
+DTC = os.path.join(DTC_ROOT, "dtc")
+FDTOVERLAY = os.path.join(DTC_ROOT, "fdtoverlay")
+
+def cmd_compile(args):
+    exec_args = [
+            DTC,
+            "-I", "dts", "-O", "dtb",
+            "--out-version", "17",
+        ]
+
+    if args.output_file:
+        exec_args += [ "-o", args.output_file ]
+    if args.input_file:
+        exec_args += [ args.input_file ]
+
+    return subprocess.call(exec_args)
+
+def cmd_overlay(args):
+    exec_args = [
+            FDTOVERLAY,
+            "-i", args.base_dtb,
+            "-o", args.output_dtb,
+        ] + args.overlay_dtb
+    return subprocess.call(exec_args)
+
+def main():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="command")
+
+    parser_compile = subparsers.add_parser("compile", help="compile DTS to DTB")
+    parser_compile.add_argument("-i", "--input-file")
+    parser_compile.add_argument("-o", "--output-file")
+
+    parser_overlay = subparsers.add_parser("overlay", help="merge DTBs")
+    parser_overlay.add_argument("output_dtb")
+    parser_overlay.add_argument("base_dtb")
+    parser_overlay.add_argument("overlay_dtb", nargs='*')
+
+    args = parser.parse_args()
+
+    if args.command == "compile":
+        return cmd_compile(args)
+    elif args.command == "overlay":
+        return cmd_overlay(args)
+    else:
+        raise Error("Unknown command: {}".format(args.command))
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/build/image/generate_initrd.py b/build/image/generate_initrd.py
new file mode 100644
index 0000000..901bc68
--- /dev/null
+++ b/build/image/generate_initrd.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generate an initial RAM disk for the hypervisor.
+
+Packages the VMs, initrds for the VMs and the list of secondary VMs (vms.txt)
+into an initial RAM disk image.
+"""
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+def Main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--file",
+        action="append", nargs=2,
+        metavar=("NAME", "PATH"),
+        help="File at host location PATH to be added to the RAM disk as NAME")
+    parser.add_argument("-s", "--staging", required=True)
+    parser.add_argument("-o", "--output", required=True)
+    args = parser.parse_args()
+
+    # Create staging folder if needed.
+    if not os.path.isdir(args.staging):
+        os.makedirs(args.staging)
+
+    # Copy files into the staging folder.
+    staged_files = []
+    for name, path in args.file:
+        shutil.copyfile(path, os.path.join(args.staging, name))
+        assert name not in staged_files
+        staged_files.append(name)
+
+    # Package files into an initial RAM disk.
+    with open(args.output, "w") as initrd:
+        # Move into the staging directory so the file names taken by cpio don't
+        # include the path.
+        os.chdir(args.staging)
+        cpio = subprocess.Popen(
+            ["cpio", "--create"],
+            stdin=subprocess.PIPE,
+            stdout=initrd,
+            stderr=subprocess.PIPE)
+        cpio.communicate(input="\n".join(staged_files).encode("utf-8"))
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(Main())
diff --git a/build/image/generate_linux_initrd.py b/build/image/generate_linux_initrd.py
new file mode 100644
index 0000000..c93e19a
--- /dev/null
+++ b/build/image/generate_linux_initrd.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generate an initial RAM disk for a Linux VM."""
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+def Main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--staging", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+    # Package files into an initial RAM disk.
+    with open(args.output, "w") as initrd:
+        # Move into the staging directory so the file names taken by cpio don't
+        # include the path.
+        os.chdir(args.staging)
+        staged_files = [os.path.join(root, filename)
+          for (root, dirs, files) in os.walk(".") for filename in files + dirs]
+        cpio = subprocess.Popen(
+            ["cpio", "--create", "--format=newc"],
+            stdin=subprocess.PIPE,
+            stdout=initrd,
+            stderr=subprocess.PIPE)
+        cpio.communicate(input="\n".join(staged_files).encode("utf-8"))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(Main())
diff --git a/build/image/image.gni b/build/image/image.gni
new file mode 100644
index 0000000..7860784
--- /dev/null
+++ b/build/image/image.gni
@@ -0,0 +1,308 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/embedded.gni")
+
+# Build image, link to an ELF file then convert to plain binary.
+template("image_binary") {
+  assert(defined(invoker.image_name),
+         "image_binary() must specify an \"image_name\" value")
+
+  output_root = ""
+  if (defined(invoker.output_path)) {
+    output_root += "${invoker.output_path}/"
+  }
+  output_root += invoker.image_name
+
+  file_root = "${root_out_dir}/${output_root}"
+  elf_file = "${file_root}.elf"
+  bin_file = "${file_root}.bin"
+
+  elf_target = "${target_name}__elf"
+  checked_elf_target = "${target_name}__checked_elf"
+
+  # Link objects together
+  executable(elf_target) {
+    forward_variables_from(invoker,
+                           [
+                             "cflags",
+                             "cflags_c",
+                             "defines",
+                             "deps",
+                             "include_dirs",
+                             "public_configs",
+                             "public_deps",
+                             "sources",
+                             "testonly",
+                           ])
+    output_name = "${output_root}.elf"
+    inputs = [
+      rebase_path("//build/image/image.ld"),
+    ]
+    ldflags = [
+      "-T",
+      rebase_path("//build/image/image.ld"),
+    ]
+    visibility = [
+      ":${checked_elf_target}",
+      ":${invoker.target_name}",
+    ]
+  }
+
+  # Analyze the generated ELF file and check that assembly-level fixes, e.g.
+  # for CPU errata, have been properly applied.
+  action(checked_elf_target) {
+    forward_variables_from(invoker, [ "testonly" ])
+    stamp_file = elf_file + "_check.stamp"
+
+    script = "//build/image/check_elf.py"
+    deps = [
+      ":${elf_target}",
+    ]
+    args = [
+      rebase_path(elf_file),
+      rebase_path(stamp_file),
+    ]
+    outputs = [
+      stamp_file,
+    ]
+    visibility = [ ":${invoker.target_name}" ]
+  }
+
+  action(target_name) {
+    forward_variables_from(invoker, [ "testonly" ])
+
+    script = "//build/image/convert_to_binary.py"
+
+    if (defined(invoker.check_binary) && invoker.check_binary == true) {
+      deps = [
+        ":${checked_elf_target}",
+      ]
+    } else {
+      deps = [
+        ":${elf_target}",
+      ]
+    }
+    args = [
+      "--input",
+      rebase_path(elf_file),
+      "--output",
+      rebase_path(bin_file),
+    ]
+    outputs = [
+      bin_file,
+    ]
+  }
+}
+
+# Helper to build a hypervisor image
+template("hypervisor") {
+  image_binary(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "cflags",
+                             "cflags_c",
+                             "defines",
+                             "deps",
+                             "public_deps",
+                             "sources",
+                             "testonly",
+                           ])
+    image_name = target_name
+
+    # Perform checks on the generated binary to prevent regressing on some
+    # classes of bugs, typically CPU erratas.
+    check_binary = true
+  }
+}
+
+# Helper to build a virtual machine kernel
+template("vm_kernel") {
+  image_binary(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "cflags",
+                             "cflags_c",
+                             "defines",
+                             "deps",
+                             "include_dirs",
+                             "public_configs",
+                             "public_deps",
+                             "sources",
+                             "testonly",
+                           ])
+    output_path = rebase_path(".", root_out_dir, target_out_dir)
+    image_name = target_name
+  }
+}
+
+# Build the initial RAM disk for the Linux VM.
+template("linux_initrd") {
+  initrd_base = "${target_out_dir}/${target_name}/initrd"
+  initrd_file = "${initrd_base}.img"
+  initrd_staging = "${initrd_base}"
+
+  copy("${target_name}__staging") {
+    forward_variables_from(invoker,
+                           [
+                             "testonly",
+                             "sources",
+                             "deps",
+                           ])
+    outputs = [
+      "${initrd_staging}/{{source_file_part}}",
+    ]
+  }
+
+  action(target_name) {
+    forward_variables_from(invoker, [ "testonly" ])
+    script = "//build/image/generate_linux_initrd.py"
+    args = [
+      "--staging",
+      rebase_path(initrd_staging),
+      "--output",
+      rebase_path(initrd_file),
+    ]
+    deps = [
+      ":${target_name}__staging",
+    ]
+    outputs = [
+      initrd_file,
+    ]
+  }
+}
+
+template("device_tree") {
+  action_foreach(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "testonly",
+                             "sources",
+                             "deps",
+                           ])
+    script = "//build/image/dtc.py"
+
+    if (defined(invoker.output_pattern)) {
+      output_pattern = invoker.output_pattern
+    } else {
+      output_pattern = "${target_out_dir}/{{source_name_part}}.dtb"
+    }
+
+    outputs = [
+      output_pattern,
+    ]
+    args = [
+      "compile",
+      "-i",
+      "{{source}}",
+      "-o",
+      rebase_path(output_pattern),
+    ]
+  }
+}
+
+template("incbin") {
+  source_set(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "testonly",
+                             "deps",
+                           ])
+
+    sources = [
+      "//build/image/incbin.S",
+    ]
+    inputs = invoker.sources
+    defines = [
+      "SECTION_NAME=" + invoker.section,
+      "FILE_PATH=\"" + rebase_path(inputs[0]) + "\"",
+    ]
+  }
+}
+
+# Build the initial RAM disk for the hypervisor.
+template("initrd") {
+  assert(defined(invoker.primary_name),
+         "initrd() must specify a \"primary_name\" value")
+  assert(defined(invoker.primary_vm),
+         "initrd() must specify a \"primary_vm\" value")
+
+  manifest_target = "${target_name}__manifest"
+  base_out_dir = "${target_out_dir}/${target_name}"
+
+  # Generate manifest.dtbo
+  device_tree(manifest_target) {
+    sources = [
+      invoker.manifest,
+    ]
+    output_pattern = "${base_out_dir}/{{source_name_part}}.dtbo"
+  }
+
+  action(target_name) {
+    forward_variables_from(invoker, [ "testonly" ])
+    script = "//build/image/generate_initrd.py"
+
+    initrd_file = "${base_out_dir}/initrd.img"
+    initrd_staging = "${base_out_dir}/initrd"
+
+    # Cannot get target outputs here as they are defined in a different file.
+    primary_vm_image = get_label_info(invoker.primary_vm, "target_out_dir") +
+                       "/" + get_label_info(invoker.primary_vm, "name") + ".bin"
+
+    deps = [
+      ":${manifest_target}",
+      invoker.primary_vm,
+    ]
+    args = [
+      "--file",
+      invoker.primary_name,
+      rebase_path(primary_vm_image),
+      "--staging",
+      rebase_path(initrd_staging),
+      "--output",
+      rebase_path(initrd_file),
+    ]
+
+    if (defined(invoker.primary_initrd)) {
+      deps += [ invoker.primary_initrd ]
+      primary_initrd_outputs = get_target_outputs(invoker.primary_initrd)
+      args += [
+        "--file",
+        "initrd.img",
+        rebase_path(primary_initrd_outputs[0]),
+      ]
+    }
+
+    # Add the info about the secondary VMs. The information about the VMs is
+    # encoded in lists with the following elements:
+    #
+    #    1. File name for the VM image.
+    #    2. Build target for the VM.
+    if (defined(invoker.secondary_vms)) {
+      foreach(vm, invoker.secondary_vms) {
+        deps += [ vm[1] ]
+        args += [
+          "--file",
+          vm[0],
+          rebase_path(get_label_info(vm[1], "target_out_dir") + "/" +
+                      get_label_info(vm[1], "name") + ".bin"),
+        ]
+      }
+    }
+
+    outputs = [
+      initrd_file,
+    ]
+  }
+}
diff --git a/build/image/image.ld b/build/image/image.ld
new file mode 100644
index 0000000..c42ab5f
--- /dev/null
+++ b/build/image/image.ld
@@ -0,0 +1,197 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Code will start running at this symbol which is places at the start of the
+ * image.
+ */
+ENTRY(entry)
+
+/*
+ * The following would be useful to check that .init code is not called back
+ * into once it has completed but it isn't supported by ld.lld.
+ *
+ * NOCROSSREFS_TO(.init .text)
+ */
+
+SECTIONS
+{
+	/*
+	 * Set the image origin to a platform specific address. The images are
+	 * relocatable but some platforms, e.g. QEMU, load to the same address
+	 * and it makes debugging easier if the addresses match the symbols.
+	 */
+	. = ORIGIN_ADDRESS;
+
+	/*
+	 * Collect together the code. This is page aligned so it can be mapped
+	 * as executable-only.
+	 */
+	text_begin = .;
+	.init : {
+		*(.init.entry)
+		*(.init.*)
+	}
+	.text : {
+		*(.text.*)
+	}
+	text_size = ABSOLUTE(. - text_begin);
+	. = ALIGN(4096);
+	text_end = .;
+
+	/*
+	 * Collect together read-only data including relocations at the end
+	 * which are applied by the entry code.  This is page aligned so it can
+	 * be mapped as read-only and non-executable.
+	 */
+	. = ALIGN(4096);
+	rodata_begin = .;
+	.rodata : {
+		*(.rodata.*)
+	}
+	/*
+	 * .rela contains Elf64_Rela entries which contain 8-byte fields so
+	 * should be 8-byte aligned.
+	 */
+	. = ALIGN(8);
+	rela_begin = .;
+	.rela : {
+		*(.rela.*)
+	}
+	rela_end = .;
+	/*
+	 * The linker doesn't allow .dynsym and .dynstr to be discarded, see
+	 * /DISCARD/ below, so make sure they don't get in the way.
+	 */
+	.dynsym : {
+		*(.dynsym.*)
+	}
+	.dynstr : {
+		*(.dynstr.*)
+	}
+	/*
+	 * The hftest framework adds test descriptors in the .hftest section
+	 * which is examined at runtime to discover the available tests. The
+	 * input sections are named after the test they include so sorting here
+	 * means they are stored sorted by the name of the test suite and then
+	 * by test case names. To ensure tests aren't accidentally included in
+	 * images that are not meant to have them, the assertion checks for a
+	 * marker to signal tests are allowed.
+	 */
+	. = ALIGN(8);
+	hftest_begin = .;
+	.hftest : {
+		KEEP(*(SORT(.hftest.*)))
+	}
+	hftest_end = .;
+	ASSERT((SIZEOF(.hftest) == (DEFINED(hftest_enable) ? SIZEOF(.hftest) : 0)),
+	       "Error: Image includes .hftest section but not HFTEST_ENABLE().")
+	rodata_size = ABSOLUTE(. - rodata_begin);
+	. = ALIGN(4096);
+	rodata_end = .;
+
+	/*
+	 * A platform may choose to link blobs such as the FDT or the initrd
+	 * into the image rather than having them loaded separately. These are
+	 * placed at the end of the image and will not be mapped automatically
+	 * on boot so they can be treated as if they were loaded as separate
+	 * blobs. They are page aligned so they can be mapped individually.
+	 *
+	 * TODO: remove this when the loader can reliably deliver both the
+	 * binary and a separate blob for the initrd.
+	 */
+	. = ALIGN(4096);
+	initrd_begin = .;
+	.initrd : {
+		KEEP(*(.plat.initrd))
+	}
+	initrd_end = .;
+	. = ALIGN(4096);
+	fdt_begin = .;
+	.fdt : {
+		KEEP(*(.plat.fdt))
+	}
+	fdt_end = .;
+
+	/*
+	 * Collect together the read-write data including .bss at the end which
+	 * will be zero'd by the entry code. This is page aligned so it can be
+	 * mapped as non-executable.
+	 */
+	. = ALIGN(4096);
+	data_begin = .;
+	.data : {
+		*(.data)
+	}
+	/*
+	 * Global offset table used for relocations. This is where relocation
+	 * fix-ups are applied.
+	 */
+	.got : {
+		*(.got.*)
+	}
+	/*
+	 * The linker doesn't allow .dynamic to be discarded, see /DISCARD/
+	 * below, so make sure it doesn't get in the way.
+	 */
+	.dynamic : {
+		*(.dynamic.*)
+	}
+	/* Everything beyond this point will not be included in the binary. */
+	bin_end = .;
+	/* The entry point code assumes that .bss is 16-byte aligned. */
+	. = ALIGN(16);
+	bss_begin = .;
+	.bss : {
+		*(.bss)
+		*(COMMON)
+	}
+	. = ALIGN(16);
+	bss_end = .;
+	data_size = ABSOLUTE(. - data_begin);
+	. = ALIGN(4096);
+	data_end = .;
+
+	/*
+	 * Remove unused sections from the image.
+	 */
+	/DISCARD/ : {
+		/* The image loads itself so doesn't need these sections. */
+		/* ld.lld doesn't allow these to be discarded.
+		*(.dynsym)
+		*(.dynstr)
+		*(.dynamic)
+		*/
+		*(.gnu.hash)
+		*(.hash)
+		*(.interp)
+	}
+
+	/*
+	 * Make note of some useful values.
+	 */
+
+	/* Note the first page not used in the image. */
+	. = ALIGN(4096);
+	image_end = .;
+
+	/*
+	 * Calculate sizes of the binary file and image loaded into memory as
+	 * well as the text, read-only and read-write data sections.
+	 */
+	bin_size = ABSOLUTE(bin_end - ORIGIN_ADDRESS);
+	image_size = ABSOLUTE(image_end - ORIGIN_ADDRESS);
+}
diff --git a/build/image/incbin.S b/build/image/incbin.S
new file mode 100644
index 0000000..9a19d87
--- /dev/null
+++ b/build/image/incbin.S
@@ -0,0 +1,18 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.section SECTION_NAME, "a"
+.incbin FILE_PATH
diff --git a/build/license.py b/build/license.py
new file mode 100644
index 0000000..ee1b5e9
--- /dev/null
+++ b/build/license.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Add license header to source files.
+
+If the file doesn't have the license header, add it with the appropriate comment
+style.
+"""
+
+import argparse
+import datetime
+import re
+import sys
+
+
+apache2 = """{comment} Copyright {year} The Hafnium Authors.
+{comment}
+{comment} Licensed under the Apache License, Version 2.0 (the "License");
+{comment} you may not use this file except in compliance with the License.
+{comment} You may obtain a copy of the License at
+{comment}
+{comment}     https://www.apache.org/licenses/LICENSE-2.0
+{comment}
+{comment} Unless required by applicable law or agreed to in writing, software
+{comment} distributed under the License is distributed on an "AS IS" BASIS,
+{comment} WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+{comment} See the License for the specific language governing permissions and
+{comment} limitations under the License."""
+
+def Main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file")
+    parser.add_argument("--style", choices=["c", "hash"], required=True)
+    args = parser.parse_args()
+    header = "/*\n" if args.style == "c" else ""
+    year = str(datetime.datetime.now().year)
+    header += apache2.format(comment=" *" if args.style == "c" else "#", year=year)
+    header += "\n */" if args.style == "c" else ""
+    header += "\n\n"
+    header_regex = re.escape(header).replace(year, r"\d\d\d\d")
+    with open(args.file, "r") as f:
+        contents = f.read()
+        if re.search(header_regex, contents):
+            return
+    with open(args.file, "w") as f:
+        f.write(header)
+        f.write(contents)
+
+if __name__ == "__main__":
+    sys.exit(Main())
diff --git a/build/linux/copy_dirs.py b/build/linux/copy_dirs.py
new file mode 100644
index 0000000..3e8ccf6
--- /dev/null
+++ b/build/linux/copy_dirs.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Copies all files inside one folder to another, preserving subfolders."""
+
+import argparse
+import os
+import shutil
+import sys
+
+def main():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("source_folder",
+	                    help="directory to be copied from")
+	parser.add_argument("destination_folder",
+	                    help="directory to be copied into")
+	parser.add_argument("stamp_file",
+	                    help="stamp file to be touched")
+	args = parser.parse_args()
+
+	# Walk the subfolders of the source directory and copy individual files.
+	# Not using shutil.copytree() because it never overwrites files.
+	for root, _, files in os.walk(args.source_folder):
+		for f in files:
+			abs_src_path = os.path.join(root, f)
+			rel_path = os.path.relpath(abs_src_path, args.source_folder)
+			abs_dst_path = os.path.join(args.destination_folder, rel_path)
+			abs_dst_folder = os.path.dirname(abs_dst_path)
+			if not os.path.isdir(abs_dst_folder):
+				os.makedirs(abs_dst_folder)
+			shutil.copyfile(abs_src_path, abs_dst_path)
+
+	# Touch `stamp_file`.
+	with open(args.stamp_file, "w"):
+		pass
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/build/linux/gen_depfile.py b/build/linux/gen_depfile.py
new file mode 100755
index 0000000..12fbd51
--- /dev/null
+++ b/build/linux/gen_depfile.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generate a depfile for a folder."""
+
+import argparse
+import os
+import sys
+
+def main():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("root_dir", help="input directory")
+	parser.add_argument("stamp_file", help="stamp file to be touched")
+	parser.add_argument("dep_file", help="depfile to be written")
+	args = parser.parse_args()
+
+	# Compile list of all files in the folder, relative to `root_dir`.
+	sources = []
+	for root, _, files in os.walk(args.root_dir):
+		sources.extend([ os.path.join(root, f) for f in files ])
+	sources = sorted(sources)
+
+	# Write `dep_file` as a Makefile rule for `stamp_file`.
+	with open(args.dep_file, "w") as f:
+		f.write(args.stamp_file)
+		f.write(":")
+		for source in sources:
+			f.write(' ');
+			f.write(source)
+		f.write(os.linesep)
+
+	# Touch `stamp_file`.
+	with open(args.stamp_file, "w"):
+		pass
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/build/linux/linux.gni b/build/linux/linux.gni
new file mode 100644
index 0000000..f76b291
--- /dev/null
+++ b/build/linux/linux.gni
@@ -0,0 +1,186 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+template("source_dir") {
+  action("${target_name}") {
+    depfile = "${target_out_dir}/${target_name}.d"
+    outputs = [
+      "$target_out_dir/${target_name}.script.stamp",
+    ]
+
+    script = "//build/linux/gen_depfile.py"
+    args = [
+      rebase_path(invoker.path, root_build_dir),
+      rebase_path(outputs[0], root_build_dir),
+      rebase_path(depfile, root_build_dir),
+    ]
+  }
+}
+
+template("source_dir_copy") {
+  source_dir_target = "${target_name}__source_dir"
+
+  source_dir(source_dir_target) {
+    path = invoker.path
+  }
+
+  action("${target_name}") {
+    script = "//build/linux/copy_dirs.py"
+    outputs = [
+      "$target_out_dir/${target_name}.script.stamp",
+    ]
+    args = [
+      rebase_path(invoker.path),
+      rebase_path(target_out_dir),
+      rebase_path(outputs[0]),
+    ]
+    deps = [
+      ":${source_dir_target}",
+    ]
+  }
+}
+
+template("linux_kernel") {
+  source_target = "${target_name}__source"
+  defconfig_target = "${target_name}__defconfig"
+  prebuilt_target = "${target_name}__prebuilt"
+
+  kernel_dir = "./"
+
+  # Args to build/make.py to start the Linux build.
+  shared_args = [
+    "--directory",
+    rebase_path(kernel_dir),
+
+    # TODO: Build with toolchain cc instead of a hardcoded one.
+    "CC=" + rebase_path("//prebuilts/linux-x64/clang/bin/clang"),
+    "LD=" +
+        rebase_path("//prebuilts/linux-x64/gcc/bin/aarch64-linux-android-ld"),
+    "AR=" +
+        rebase_path("//prebuilts/linux-x64/gcc/bin/aarch64-linux-android-ar"),
+    "NM=" + rebase_path("//prebuilts/linux-x64/clang/bin/llvm-nm"),
+    "OBJCOPY=" + rebase_path(
+            "//prebuilts/linux-x64/gcc/bin/aarch64-linux-android-objcopy"),
+    "OBJDUMP=" + rebase_path("//prebuilts/linux-x64/clang/bin/llvm-objdump"),
+    "STRIP=" + rebase_path("//prebuilts/linux-x64/clang/bin/llvm-strip"),
+    "ARCH=arm64",
+    "CROSS_COMPILE=aarch64-linux-gnu-",
+
+    # Build out-of-tree in `target_out_dir`.
+    "O=" + rebase_path(target_out_dir),
+
+    # TODO: Remove/replace.
+    "-j24",
+  ]
+
+  # Subtarget which generates a depfile with all files in the Linux tree
+  # and gets invalidated if any of them change.
+  source_dir(source_target) {
+    path = kernel_dir
+  }
+
+  # Subtarget which runs `defconfig` and `modules_prepare`. Used by targets
+  # which do not require the whole kernel to have been built.
+  action(defconfig_target) {
+    script = "//build/make.py"
+    args = shared_args + [
+             "defconfig",
+             "modules_prepare",
+           ]
+
+    # We never use the output but GN requires each target to have one, and for
+    # its timestamp to change after a recompile. Use the .config file.
+    outputs = [
+      "${target_out_dir}/.config",
+    ]
+    deps = [
+      ":${source_target}",
+    ]
+  }
+
+  action(target_name) {
+    script = "//build/make.py"
+    output_file = "${target_out_dir}/${target_name}.bin"
+    args = shared_args + [
+             "--copy_out_file",
+             rebase_path("${target_out_dir}/arch/arm64/boot/Image"),
+             rebase_path(output_file),
+           ]
+    outputs = [
+      output_file,
+    ]
+    deps = [
+      ":${defconfig_target}",
+      ":${source_target}",
+    ]
+  }
+
+  # Subtarget for a prebuilt image, if defined.
+  if (defined(invoker.prebuilt)) {
+    copy(prebuilt_target) {
+      sources = [
+        invoker.prebuilt,
+      ]
+      outputs = [
+        "${target_out_dir}/${prebuilt_target}.bin",
+      ]
+    }
+  }
+}
+
+template("linux_kernel_module") {
+  # Out-of-tree modules cannot be built outside of their directory.
+  # So as to avoid parallel builds under different toolchains clashing,
+  # work around by copying source files to `target_out_dir`.
+
+  source_target = "${target_name}__source"
+
+  module_dir = "./"
+
+  source_dir_copy(source_target) {
+    path = module_dir
+  }
+
+  action(target_name) {
+    forward_variables_from(invoker, [ "testonly" ])
+    script = "//build/make.py"
+    args = [
+      "--directory",
+      rebase_path(target_out_dir),
+      "HAFNIUM_PATH=" + rebase_path("//"),
+      "KERNEL_PATH=" + rebase_path(invoker.kernel_dir),
+      "O=" +
+          rebase_path(get_label_info(invoker.kernel_target, "target_out_dir")),
+      "CC=" + rebase_path("//prebuilts/linux-x64/clang/bin/clang"),
+      "LD=" +
+          rebase_path("//prebuilts/linux-x64/gcc/bin/aarch64-linux-android-ld"),
+      "AR=" +
+          rebase_path("//prebuilts/linux-x64/gcc/bin/aarch64-linux-android-ar"),
+      "NM=" + rebase_path("//prebuilts/linux-x64/clang/bin/llvm-nm"),
+      "OBJCOPY=" + rebase_path(
+              "//prebuilts/linux-x64/gcc/bin/aarch64-linux-android-objcopy"),
+      "OBJDUMP=" + rebase_path("//prebuilts/linux-x64/clang/bin/llvm-objdump"),
+      "STRIP=" + rebase_path("//prebuilts/linux-x64/clang/bin/llvm-strip"),
+      "ARCH=arm64",
+      "CROSS_COMPILE=aarch64-linux-gnu-",
+    ]
+    outputs = [
+      "${target_out_dir}/${invoker.module_name}.ko",
+    ]
+    deps = [
+      ":${source_target}",
+      "${invoker.kernel_target}__defconfig",
+    ]
+  }
+}
diff --git a/build/make.py b/build/make.py
new file mode 100644
index 0000000..15cce9b
--- /dev/null
+++ b/build/make.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Runs make to build a target."""
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+
+def Main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--directory", required=True)
+    parser.add_argument("--copy_out_file", nargs=2,
+                        help="Copy file after building. Takes two params: <src> <dest>")
+    args, make_args = parser.parse_known_args()
+
+    os.chdir(args.directory)
+    os.environ["PWD"] = args.directory
+    status = subprocess.call(["make"] + make_args)
+    if status != 0:
+        return status
+
+    if args.copy_out_file is not None:
+        shutil.copyfile(args.copy_out_file[0], args.copy_out_file[1])
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(Main())
diff --git a/build/parse_strace_open.py b/build/parse_strace_open.py
new file mode 100755
index 0000000..a971220
--- /dev/null
+++ b/build/parse_strace_open.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script which parses the output of `strace` and dumping a list of files
+that were touched by the traced processes outside of whitelisted folders.
+It assumes that strace was invoked with the following arguments:
+    -e trace=%file,chdir,%process   record required syscalls
+    -qq                             silence 'exit code' records
+    -o <file>                       output format is different when writing
+                                    to a file from printing to the console
+"""
+
+import argparse
+import os
+import sys
+
+FORK_SYSCALLS = [
+    "clone",
+    "fork",
+    "vfork",
+    ]
+OPEN_SYSCALLS = [
+    "access",
+    "creat",
+    "lstat",
+    "mkdir",
+    "open",
+    "openat",
+    "readlink",
+    "stat",
+    ]
+
+def get_unfinished(line):
+    pos = line.find("<unfinished ...>")
+    if pos < 0:
+        return None
+    else:
+        return line[:pos]
+
+def get_resumed(line):
+    pos = line.find(" resumed>")
+    if pos < 0:
+        return None
+    else:
+        return line[pos + len(" resumed>"):]
+
+def merge_unfinished_lines(lines):
+    """Process input lines and merge those split by an interrupting syscall."""
+    # Lines in the order they were started being written.
+    finished = []
+
+    # Pending unfinished lines. Map from PID to index in `finished`.
+    cursor = {}
+
+    for line in lines:
+        pid = int(line.split()[0])
+
+        resumed = get_resumed(line)
+        if resumed is not None:
+            assert(pid in cursor)
+            unfinished = get_unfinished(resumed)
+            if unfinished is not None:
+                finished[cursor[pid]] += unfinished
+            else:
+                finished[cursor[pid]] += resumed
+                del(cursor[pid])
+        else:
+            assert(pid not in cursor)
+            unfinished = get_unfinished(line)
+            if unfinished is not None:
+                # Line is unfinished. Store its location to `cursor`.
+                cursor[pid] = len(finished)
+                finished += [ unfinished ]
+            else:
+                finished += [ line ]
+    return finished
+
+def abs_path(cwd, path):
+    """If `path` is relative, resolve it against the current working directory.
+       Also normalize the resulting path."""
+    if path[0] != '/':
+        path = os.path.join(cwd, path)
+    path = os.path.abspath(path)
+    # while '//' in path:
+    #     path = path.replace('//', '/')
+    path = os.path.realpath(path)
+    return path
+
+def get_touched_files(lines, orig_cwd):
+    """Parse strace output and return all files that an open()-like syscall was
+       called on."""
+    files = set()
+
+    # Map from PID to the current working directory.
+    cwd = {}
+
+    # Map from PID to executable name
+    executable = {}
+
+    # Map from PID to the PID of the process which forked it.
+    fork_of = {}
+
+    first_pid = True
+    for line in lines:
+        # Split line: <pid>  <syscall info>
+        line = line.split()
+        pid = int(line[0])
+        syscall = " ".join(line[1:])
+
+        # If seeing a PID for the first time, derive its working directory
+        # from its parent.
+        if pid not in cwd:
+            if first_pid:
+                # Very first line of strace output. Set working directory from
+                # command line arguments (should match cwd of strace).
+                first_pid = False
+                cwd[pid] = orig_cwd
+            else:
+                # There should have been a fork/clone syscall which spawned this
+                # process. Inherit its working directory.
+                cwd[pid] = cwd[fork_of[pid]]
+
+        # We are looking for lines which match:
+        #   name(arg1, arg2, ..., argN) = result
+        left_bracket = syscall.find("(")
+        right_bracket = syscall.rfind(")")
+        assign_sign = syscall.rfind("=")
+        if left_bracket < 0 or right_bracket < 0 or assign_sign < right_bracket:
+            continue
+
+        syscall_name = syscall[:left_bracket]
+        syscall_result = syscall[assign_sign+2:]
+
+        syscall_args = syscall[left_bracket+1:right_bracket].split(",")
+        syscall_args = list(map(lambda x: x.strip(), syscall_args))
+
+        if syscall_name in FORK_SYSCALLS:
+            # If this is a fork, keep track of the parent-child relationship.
+            # The child's PID is the syscall's return code.
+            new_pid = int(syscall_result)
+            fork_of[new_pid] = pid
+            executable[new_pid] = executable[pid]
+        elif syscall_name == "chdir":
+            # If this is a change of working directory, keep track of it.
+            # It is in the first argument in quotes.
+            new_dir = syscall_args[0][1:-1]
+            cwd[pid] = abs_path(cwd[pid], new_dir)
+        elif syscall_name == "execve":
+            # If this is executing a new program, record its name.
+            # It is in the first argument in quotes.
+            binary_name = syscall_args[0][1:-1]
+            executable[pid] = binary_name
+        elif syscall_name in OPEN_SYSCALLS:
+            # If this is a syscall touching a file, record the path.
+            # We ignore the result code, i.e. record the path even if the
+            # syscall failed to open it.
+            arg_idx = 0
+            if syscall_name == "openat":
+                # openat() can open a file (second arg) relative to a given
+                # folder (first arg). We only support passing AT_FDCWD, ie.
+                # resolve against the current working directory.
+                arg_idx = 1
+                assert(syscall_args[0] == "AT_FDCWD")
+            fname = abs_path(cwd[pid], syscall_args[arg_idx][1:-1])
+            # Record the file and the name of the program which touched it.
+            files.add((fname, executable[pid]))
+    return files
+
+def filter_results(files, root_dir):
+    """Remove paths which are whitelisted from the results."""
+    # Anything in the Hafnium directory is allowed.
+    files = filter(lambda x: not x[0].startswith(root_dir + "/"), files)
+    # Clang puts intermediate files in /tmp.
+    files = filter(lambda x: not x[0].startswith("/tmp/"), files)
+    return list(files)
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("root_dir",
+                        help="Root directory of Hafnium, cwd of strace")
+    args, make_args = parser.parse_known_args()
+
+    stdin = map(lambda x: x.strip(), sys.stdin.readlines())
+    stdin = merge_unfinished_lines(stdin)
+    files = get_touched_files(stdin, args.root_dir)
+    files = filter_results(files, args.root_dir)
+    files = sorted(list(files))
+
+    print("\n".join(map(lambda x: "{} ({})".format(x[0], x[1]), files)))
+
+if __name__ == "__main__":
+    main(sys.argv)
diff --git a/build/run_in_container.sh b/build/run_in_container.sh
new file mode 100755
index 0000000..4090c38
--- /dev/null
+++ b/build/run_in_container.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(dirname ${SCRIPT_DIR})"
+
+source "${SCRIPT_DIR}/docker/common.inc"
+
+if [ "${HAFNIUM_HERMETIC_BUILD:-}" == "inside" ]
+then
+	echo "ERROR: Invoked $0 recursively" 1>&2
+	exit 1
+fi
+
+# Set up a temp directory and register a cleanup function on exit.
+TMP_DIR="$(mktemp -d)"
+function cleanup() {
+	rm -rf "${TMP_DIR}"
+}
+trap cleanup EXIT
+
+# Build local image and write its hash to a temporary file.
+IID_FILE="${TMP_DIR}/imgid.txt"
+"${DOCKER}" build \
+	--build-arg LOCAL_UID="$(id -u)" \
+	--build-arg LOCAL_GID="$(id -g)" \
+	--iidfile="${IID_FILE}" \
+	-f "${SCRIPT_DIR}/docker/Dockerfile.local" \
+	"${SCRIPT_DIR}/docker"
+IMAGE_ID="$(cat ${IID_FILE})"
+
+# Parse command line arguments
+INTERACTIVE=false
+ALLOW_PTRACE=false
+while true
+do
+	case "${1:-}" in
+	-i)
+		INTERACTIVE=true
+		shift
+		;;
+	-p)
+		ALLOW_PTRACE=true
+		shift
+		;;
+	-*)
+		echo "ERROR: Unknown command line flag: $1" 1>&2
+		echo "Usage: $0 [-i] [-p] <command>"
+		exit 1
+		;;
+	*)
+		break
+		;;
+	esac
+done
+
+ARGS=()
+# Run with a pseduo-TTY for nicer logging.
+ARGS+=(-t)
+# Run interactive if this script was invoked with '-i'.
+if [ "${INTERACTIVE}" == "true" ]
+then
+	ARGS+=(-i)
+fi
+# Allow ptrace() syscall if invoked with '-p'.
+if [ "${ALLOW_PTRACE}" == "true" ]
+then
+	echo "WARNING: Docker seccomp profile is disabled!" 1>&2
+	ARGS+=(--cap-add=SYS_PTRACE --security-opt seccomp=unconfined)
+fi
+# Propagate "HAFNIUM_*" environment variables.
+# Note: Cannot use `env | while` because the loop would run inside a child
+# process and would not have any effect on variables in the parent.
+while read -r ENV_LINE
+do
+	VAR_NAME="$(echo ${ENV_LINE} | cut -d= -f1)"
+	case "${VAR_NAME}" in
+	HAFNIUM_HERMETIC_BUILD)
+		# Skip this one. It will be overridden below.
+		;;
+	HAFNIUM_*)
+		ARGS+=(-e "${ENV_LINE}")
+		;;
+	esac
+done <<< "$(env)"
+# Set environment variable informing the build that we are running inside
+# a container.
+ARGS+=(-e HAFNIUM_HERMETIC_BUILD=inside)
+# Bind-mount the Hafnium root directory. We mount it at the same absolute
+# location so that all paths match across the host and guest.
+ARGS+=(-v "${ROOT_DIR}":"${ROOT_DIR}")
+# Make all files outside of the Hafnium directory read-only to ensure that all
+# generated files are written there.
+ARGS+=(--read-only)
+# Mount a writable /tmp folder. Required by LLVM/Clang for intermediate files.
+ARGS+=(--tmpfs /tmp)
+# Set working directory.
+ARGS+=(-w "${ROOT_DIR}")
+
+echo "Running in container: $*" 1>&2
+${DOCKER} run \
+	${ARGS[@]} \
+	"${IMAGE_ID}" \
+	/bin/bash -c "$*"
diff --git a/build/strace_open.sh b/build/strace_open.sh
new file mode 100755
index 0000000..9a9f431
--- /dev/null
+++ b/build/strace_open.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_PATH="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")"
+ROOT_DIR="$(realpath ${SCRIPT_DIR}/..)"
+
+if [ "${HAFNIUM_HERMETIC_BUILD:-}" == "true" ]
+then
+	exec "${ROOT_DIR}/build/run_in_container.sh" -p ${SCRIPT_PATH} $@
+fi
+
+if [ $# != 1 ]
+then
+	echo "Usage: $0 <output_file>" 1>&2
+	exit 1
+fi
+
+MAKE="$(which make)"
+STRACE="$(which strace)"
+
+# Set up a temp directory and register a cleanup function on exit.
+TMP_DIR="$(mktemp -d)"
+function cleanup() {
+	rm -rf "${TMP_DIR}"
+}
+trap cleanup EXIT
+
+STRACE_LOG="${TMP_DIR}/strace.log"
+
+echo "Building with strace"
+pushd ${ROOT_DIR}
+${MAKE} clobber
+${STRACE} \
+	-o "${STRACE_LOG}" \
+	-f \
+	-qq \
+	-e trace=%file,chdir,%process \
+	${MAKE}
+popd
+
+echo "Processing strace output"
+"${SCRIPT_DIR}/parse_strace_open.py" ${ROOT_DIR} < "${STRACE_LOG}" > "$1"
diff --git a/build/toolchain/BUILD.gn b/build/toolchain/BUILD.gn
new file mode 100644
index 0000000..07a35e2
--- /dev/null
+++ b/build/toolchain/BUILD.gn
@@ -0,0 +1,46 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/embedded.gni")
+import("//build/toolchain/host.gni")
+
+host_toolchain("host") {
+  use_platform = false
+}
+
+# Toolchain for building tests which run under Linux under Hafnium.
+embedded_clang_toolchain("aarch64_linux_clang") {
+  target = "aarch64-linux-musleabi"
+
+  # TODO: Remove //inc/system if we can stop using the version of stdatomic.h
+  # from the Android prebuilt Clang.
+  extra_cflags =
+      "-nostdinc -isystem" +
+      rebase_path("//prebuilts/linux-aarch64/musl/include") + " -isystem" +
+      rebase_path("//prebuilts/linux-x64/clang/lib64/clang/9.0.3/include") +
+      " -isystem" + rebase_path("//inc/system")
+  extra_defines = "-D_LIBCPP_HAS_MUSL_LIBC=1 -D_GNU_SOURCE=1"
+  extra_ldflags = "-no-pie -lc --library-path=" +
+                  rebase_path("//prebuilts/linux-aarch64/musl/lib/") + " " +
+                  rebase_path("//prebuilts/linux-aarch64/musl/lib/crt1.o") +
+                  " " + rebase_path(
+                      "//prebuilts/linux-x64/clang/lib64/clang/9.0.3/lib/linux/libclang_rt.builtins-aarch64-android.a")
+  toolchain_args = {
+    use_platform = true
+    plat_arch = "fake"
+    plat_boot_flow = "//src/arch/fake:boot_flow"
+    plat_console = "//src/arch/fake:console"
+    plat_iommu = "//src/iommu:absent"
+  }
+}
diff --git a/build/toolchain/embedded.gni b/build/toolchain/embedded.gni
new file mode 100644
index 0000000..fa4aeae
--- /dev/null
+++ b/build/toolchain/embedded.gni
@@ -0,0 +1,356 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare_args() {
+  # Set by arch toolchain. Prefix for binutils tools.
+  tool_prefix = ""
+
+  # Enable link time optimizations
+  use_lto = true
+}
+
+# Template for embedded toolchains; there is no support for C++ or libraries.
+# Instead, use source_set to group source together.
+template("embedded_cc_toolchain") {
+  toolchain(target_name) {
+    assert(defined(invoker.cc), "\"cc\" must be defined for ${target_name}.")
+    assert(defined(invoker.ld), "\"ld\" must be defined for ${target_name}.")
+
+    # Collect extra flags from the toolchain.
+    extra_defines = ""
+    extra_cflags = "-ffunction-sections -fdata-sections"
+    if (use_lto) {
+      extra_cflags += " -flto"
+    }
+    extra_ldflags = "-pie --gc-sections"
+
+    if (defined(invoker.extra_defines)) {
+      extra_defines += " ${invoker.extra_defines}"
+    }
+    if (defined(invoker.extra_cflags)) {
+      extra_cflags += " ${invoker.extra_cflags}"
+    }
+    if (defined(invoker.extra_ldflags)) {
+      extra_ldflags += " ${invoker.extra_ldflags}"
+    }
+
+    # Define the tools.
+    tool("cc") {
+      depfile = "{{output}}.d"
+      command = "${invoker.cc} -MMD -MF $depfile ${extra_defines} {{defines}} {{include_dirs}} ${extra_cflags} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
+      depsformat = "gcc"
+      description = "CC {{output}}"
+      outputs = [
+        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+      ]
+    }
+
+    tool("asm") {
+      depfile = "{{output}}.d"
+      command = "${invoker.cc} -MMD -MF $depfile ${extra_defines} {{defines}} {{include_dirs}} ${extra_cflags} {{asmflags}} -c {{source}} -o {{output}}"
+      depsformat = "gcc"
+      description = "ASM {{output}}"
+      outputs = [
+        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+      ]
+    }
+
+    tool("link") {
+      outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
+      rspfile = "$outfile.rsp"
+      command = "${invoker.ld} ${extra_ldflags} {{ldflags}} -o $outfile --start-group @$rspfile --end-group"
+      description = "LINK $outfile"
+      default_output_dir = "{{root_out_dir}}"
+      rspfile_content = "{{inputs}}"
+      outputs = [
+        outfile,
+      ]
+    }
+
+    tool("alink") {
+      outfile = "{{target_out_dir}}/{{target_output_name}}.a"
+      command = "rm -f $outfile && ${invoker.ar} -rc $outfile {{inputs}}"
+      description = "ALINK $outfile"
+      outputs = [
+        outfile,
+      ]
+    }
+
+    tool("stamp") {
+      command = "touch {{output}}"
+      description = "STAMP {{output}}"
+    }
+
+    tool("copy") {
+      command = "cp -af {{source}} {{output}}"
+      description = "COPY {{source}} {{output}}"
+    }
+
+    toolchain_args = {
+      forward_variables_from(invoker.toolchain_args, "*")
+    }
+  }
+}
+
+# Specialize for clang.
+template("embedded_clang_toolchain") {
+  assert(defined(invoker.target),
+         "\"target\" must be defined for ${target_name}.")
+  assert(defined(invoker.extra_defines),
+         "\"extra_defines\" must be defined for ${target_name}")
+  assert(defined(invoker.extra_cflags),
+         "\"extra_cflags\" must be defined for ${target_name}")
+  assert(defined(invoker.extra_ldflags),
+         "\"extra_ldflags\" must be defined for ${target_name}")
+
+  embedded_cc_toolchain(target_name) {
+    cc = "clang"
+    ld = "ld.lld"
+    ar = "llvm-ar"
+
+    forward_variables_from(invoker,
+                           [
+                             "extra_defines",
+                             "extra_cflags",
+                             "extra_ldflags",
+                           ])
+
+    # TODO: Remove //inc/system if we can stop using the version of stdatomic.h
+    # from the Android prebuilt Clang.
+    extra_cflags +=
+        " -target ${invoker.target} -fcolor-diagnostics -nostdinc -isystem" +
+        rebase_path("//prebuilts/linux-x64/clang/lib64/clang/9.0.3/include") +
+        " -isystem" + rebase_path("//inc/system")
+    extra_ldflags +=
+        " -O2 -lto-O2 --icf=all --fatal-warnings --color-diagnostics"
+
+    toolchain_args = {
+      if (defined(invoker.toolchain_args)) {
+        forward_variables_from(invoker.toolchain_args, "*")
+      }
+    }
+  }
+}
+
+# Specialize for mixed toolchain with clang and bfd linker.
+template("embedded_clang_bfd_toolchain") {
+  assert(defined(invoker.target),
+         "\"target\" must be defined for ${target_name}.")
+  assert(defined(invoker.tool_prefix),
+         "\"tool_prefix\" must be defined for ${target_name}.")
+  assert(defined(invoker.extra_defines),
+         "\"extra_defines\" must be defined for ${target_name}")
+  assert(defined(invoker.extra_cflags),
+         "\"extra_cflags\" must be defined for ${target_name}")
+  assert(defined(invoker.extra_ldflags),
+         "\"extra_ldflags\" must be defined for ${target_name}")
+
+  embedded_cc_toolchain(target_name) {
+    cc = "clang"
+    ld = "${invoker.tool_prefix}ld.bfd"
+    ar = "llvm-ar"
+
+    forward_variables_from(invoker,
+                           [
+                             "extra_defines",
+                             "extra_cflags",
+                             "extra_ldflags",
+                           ])
+    extra_cflags += " -target ${invoker.target} -fcolor-diagnostics"
+    extra_ldflags += " --fatal-warnings"
+    if (use_lto) {
+      extra_ldflags += " -O2 -lto-O2 --icf=all"
+    }
+
+    toolchain_args = {
+      if (defined(invoker.toolchain_args)) {
+        forward_variables_from(invoker.toolchain_args, "*")
+      }
+    }
+  }
+}
+
+# Expand to clang variants.
+template("embedded_platform_toolchain") {
+  assert(defined(invoker.arch), "\"arch\" must be defined for ${target_name}.")
+  assert(defined(invoker.target),
+         "\"target\" must be defined for ${target_name}.")
+  assert(defined(invoker.tool_prefix),
+         "\"tool_prefix\" must be defined for ${target_name}.")
+  assert(defined(invoker.origin_address),
+         "\"origin_address\" must be defined for ${target_name}.")
+  assert(defined(invoker.heap_pages),
+         "\"heap_pages\" must be defined for ${target_name}.")
+  assert(defined(invoker.max_cpus),
+         "\"max_cpus\" must be defined for ${target_name}.")
+  assert(defined(invoker.max_vms),
+         "\"max_vms\" must be defined for ${target_name}.")
+  assert(defined(invoker.platform_name),
+         "\"platform_name\" must be defined for ${target_name}.")
+
+  extra_defines = ""
+  extra_cflags = "-fno-builtin -ffreestanding -fpic"
+  extra_ldflags = "--defsym=ORIGIN_ADDRESS=${invoker.origin_address}"
+  if (defined(invoker.extra_defines)) {
+    extra_defines += " ${invoker.extra_defines}"
+  }
+  if (defined(invoker.extra_cflags)) {
+    extra_cflags += " ${invoker.extra_cflags}"
+  }
+  if (defined(invoker.extra_ldflags)) {
+    extra_ldflags += " ${invoker.extra_ldflags}"
+  }
+  toolchain_args = {
+    use_platform = true
+    plat_name = invoker.platform_name
+    plat_arch = invoker.arch
+    plat_heap_pages = invoker.heap_pages
+    plat_max_cpus = invoker.max_cpus
+    plat_max_vms = invoker.max_vms
+    if (defined(invoker.toolchain_args)) {
+      forward_variables_from(invoker.toolchain_args, "*")
+    }
+  }
+
+  embedded_clang_toolchain("${target_name}_clang") {
+    target = invoker.target
+  }
+
+  embedded_clang_bfd_toolchain("${target_name}_clang_bfd") {
+    target = invoker.target
+    tool_prefix = invoker.tool_prefix
+  }
+}
+
+# Specialize for different architectures.
+
+template("aarch64_common_toolchain") {
+  assert(defined(invoker.cpu), "\"cpu\" must be defined for ${target_name}.")
+  assert(defined(invoker.target),
+         "\"target\" must be defined for ${target_name}")
+  assert(defined(invoker.tool_prefix),
+         "\"tool_prefix\" must be defined for ${target_name}")
+  assert(defined(invoker.origin_address),
+         "\"origin_address\" must be defined for ${target_name}.")
+  assert(defined(invoker.console),
+         "\"console\" must be defined for ${target_name}.")
+  assert(defined(invoker.heap_pages),
+         "\"heap_pages\" must be defined for ${target_name}.")
+  assert(defined(invoker.max_cpus),
+         "\"max_cpus\" must be defined for ${target_name}.")
+  assert(defined(invoker.max_vms),
+         "\"max_vms\" must be defined for ${target_name}.")
+  if (invoker.gic_version == 3 || invoker.gic_version == 4) {
+    assert(defined(invoker.gicd_base_address),
+           "\"gicd_base_address\" must be defined for ${target_name}.")
+    assert(defined(invoker.gicr_base_address),
+           "\"gicr_base_address\" must be defined for ${target_name}.")
+  }
+  assert(defined(invoker.platform_name),
+         "\"platform_name\" must be defined for ${target_name}.")
+
+  embedded_platform_toolchain(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "origin_address",
+                             "heap_pages",
+                             "max_cpus",
+                             "max_vms",
+                             "platform_name",
+                             "extra_ldflags",
+                           ])
+    arch = "aarch64"
+    target = invoker.target
+    tool_prefix = invoker.tool_prefix
+    extra_cflags = "-mcpu=${invoker.cpu} -mstrict-align"
+    if (defined(invoker.extra_cflags)) {
+      extra_cflags += " ${invoker.extra_cflags}"
+    }
+
+    extra_defines = ""
+    if (defined(invoker.extra_defines)) {
+      extra_defines += " ${invoker.extra_defines}"
+    }
+
+    if (invoker.gic_version > 0) {
+      extra_defines += " -DGIC_VERSION=${invoker.gic_version}"
+    }
+    if (invoker.gic_version == 3 || invoker.gic_version == 4) {
+      extra_defines += " -DGICD_BASE=${invoker.gicd_base_address} -DGICR_BASE=${invoker.gicr_base_address}"
+    }
+
+    toolchain_args = {
+      plat_boot_flow = invoker.boot_flow
+      plat_console = invoker.console
+      plat_iommu = invoker.iommu
+      forward_variables_from(invoker.toolchain_args, "*")
+    }
+  }
+}
+
+template("aarch64_toolchain") {
+  aarch64_common_toolchain("${target_name}") {
+    forward_variables_from(invoker, "*")
+    target = "aarch64-none-eabi"
+    tool_prefix = "aarch64-linux-gnu-"  # TODO: this isn't right for bare metal but it works.
+    platform_name = target_name
+  }
+}
+
+template("aarch64_toolchains") {
+  aarch64_toolchain("${target_name}") {
+    forward_variables_from(invoker,
+                           [
+                             "origin_address",
+                             "boot_flow",
+                             "console",
+                             "iommu",
+                             "gic_version",
+                             "gicd_base_address",
+                             "gicr_base_address",
+                             "heap_pages",
+                             "max_cpus",
+                             "max_vms",
+                             "toolchain_args",
+                           ])
+    cpu = "${invoker.cpu}+nofp"
+
+    # Add a macro so files can tell whether they are not being built for a VM.
+    extra_defines = " -DVM_TOOLCHAIN=0"
+  }
+
+  # Toolchain for building test VMs which run under Hafnium.
+  aarch64_toolchain("${target_name}_vm") {
+    forward_variables_from(invoker,
+                           [
+                             "origin_address",
+                             "gic_version",
+                             "gicd_base_address",
+                             "gicr_base_address",
+                             "max_cpus",
+                             "toolchain_args",
+                           ])
+    cpu = "${invoker.cpu}+fp"
+    boot_flow = "//src/arch/fake:boot_flow"
+    console = "//src/arch/aarch64/hftest:console"
+    iommu = "//src/iommu:absent"
+
+    # Nonsense values because they are required but shouldn't be used.
+    heap_pages = 0
+    max_vms = 0
+
+    # Add a macro so files can tell whether they are being built for a VM.
+    extra_defines = " -DVM_TOOLCHAIN=1"
+  }
+}
diff --git a/build/toolchain/gen_offset_size_header.py b/build/toolchain/gen_offset_size_header.py
new file mode 100755
index 0000000..72c4c93
--- /dev/null
+++ b/build/toolchain/gen_offset_size_header.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generate a header file with definitions of constants parsed from a binary."""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+HF_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+BINUTILS_ROOT = os.path.join(HF_ROOT, "prebuilts", "linux-x64", "gcc", "bin")
+STRINGS = os.path.join(BINUTILS_ROOT, "aarch64-linux-android-strings")
+
+PROLOGUE = """
+/**
+ * This file was auto-generated by {}.
+ * Changes will be overwritten.
+ */
+
+#pragma once
+
+""".format(__file__)
+
+def main():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("bin_file",
+		help="binary file to be parsed for definitions of constants")
+	parser.add_argument("out_file", help="output file");
+	args = parser.parse_args()
+
+	# Regex for finding definitions: <HAFNIUM_DEFINE name #value />
+	regex = re.compile(r'<HAFNIUM_DEFINE\s([A-Za-z0-9_]+)\s#([0-9]+) />')
+
+	# Extract strings from the input binary file.
+	stdout = subprocess.check_output([ STRINGS, args.bin_file ])
+	stdout = str(stdout).split(os.linesep)
+
+	with open(args.out_file, "w") as f:
+		f.write(PROLOGUE)
+		for line in stdout:
+			for match in regex.findall(line):
+				f.write("#define {} ({})\n".format(
+					match[0], match[1]))
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/build/toolchain/host.gni b/build/toolchain/host.gni
new file mode 100644
index 0000000..feffa11
--- /dev/null
+++ b/build/toolchain/host.gni
@@ -0,0 +1,162 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Template for host toolchains.
+template("host_cc_toolchain") {
+  toolchain(target_name) {
+    assert(defined(invoker.ar), "\"ar\" must be defined for ${target_name}.")
+    assert(defined(invoker.cc), "\"cc\" must be defined for ${target_name}.")
+    assert(defined(invoker.cxx), "\"cxx\" must be defined for ${target_name}.")
+
+    # Collect extra flags from the toolchain.
+    extra_defines = ""
+    extra_cflags = ""
+    extra_ldflags = ""
+    if (defined(invoker.extra_defines)) {
+      extra_defines += " ${invoker.extra_defines}"
+    }
+    if (defined(invoker.extra_cflags)) {
+      extra_cflags += " ${invoker.extra_cflags}"
+    }
+    if (defined(invoker.extra_ldflags)) {
+      extra_ldflags += " ${invoker.extra_ldflags}"
+    }
+
+    tool("cc") {
+      depfile = "{{output}}.d"
+      command = "${invoker.cc} -MMD -MF $depfile ${extra_defines} {{defines}} {{include_dirs}} ${extra_cflags} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
+      depsformat = "gcc"
+      description = "CC {{output}}"
+      outputs = [
+        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+      ]
+    }
+
+    tool("cxx") {
+      depfile = "{{output}}.d"
+      command = "${invoker.cxx} -MMD -MF $depfile ${extra_defines} {{defines}} {{include_dirs}} ${extra_cflags} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}"
+      depsformat = "gcc"
+      description = "CXX {{output}}"
+      outputs = [
+        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+      ]
+    }
+
+    tool("alink") {
+      rspfile = "{{output}}.rsp"
+      command = "rm -f {{output}} && ${invoker.ar} rcs {{output}} @$rspfile"
+      description = "AR {{target_output_name}}{{output_extension}}"
+      rspfile_content = "{{inputs}}"
+      outputs = [
+        "{{target_out_dir}}/{{target_output_name}}{{output_extension}}",
+      ]
+      default_output_extension = ".a"
+      output_prefix = "lib"
+    }
+
+    tool("solink") {
+      soname = "{{target_output_name}}{{output_extension}}"  # e.g. "libfoo.so".
+      sofile = "{{output_dir}}/$soname"
+      rspfile = soname + ".rsp"
+
+      command = "${invoker.cxx} -shared ${extra_ldflags} {{ldflags}} -o $sofile -Wl,-soname=$soname @$rspfile"
+      rspfile_content = "-Wl,--whole-archive {{inputs}} {{solibs}} -Wl,--no-whole-archive {{libs}}"
+
+      description = "SOLINK $soname"
+
+      # Use this for {{output_extension}} expansions unless a target manually
+      # overrides it (in which case {{output_extension}} will be what the target
+      # specifies).
+      default_output_extension = ".so"
+
+      # Use this for {{output_dir}} expansions unless a target manually overrides
+      # it (in which case {{output_dir}} will be what the target specifies).
+      default_output_dir = "{{root_out_dir}}"
+
+      outputs = [
+        sofile,
+      ]
+      link_output = sofile
+      depend_output = sofile
+      output_prefix = "lib"
+    }
+
+    tool("link") {
+      outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
+      rspfile = "$outfile.rsp"
+      command = "${invoker.cxx} ${extra_ldflags} {{ldflags}} -o $outfile -Wl,--start-group @$rspfile {{solibs}} -Wl,--end-group {{libs}}"
+      description = "LINK $outfile"
+      default_output_dir = "{{root_out_dir}}"
+      rspfile_content = "{{inputs}}"
+      outputs = [
+        outfile,
+      ]
+    }
+
+    tool("stamp") {
+      command = "touch {{output}}"
+      description = "STAMP {{output}}"
+    }
+
+    tool("copy") {
+      command = "cp -af {{source}} {{output}}"
+      description = "COPY {{source}} {{output}}"
+    }
+
+    if (defined(invoker.toolchain_args)) {
+      toolchain_args = {
+        forward_variables_from(invoker.toolchain_args, "*")
+      }
+    }
+  }
+}
+
+template("host_toolchain") {
+  assert(defined(invoker.use_platform),
+         "\"use_platform\" must be defined for ${target_name}.")
+  if (invoker.use_platform) {
+    assert(defined(invoker.heap_pages),
+           "\"heap_pages\" must be defined for ${target_name}.")
+    assert(defined(invoker.max_cpus),
+           "\"max_cpus\" must be defined for ${target_name}.")
+    assert(defined(invoker.max_vms),
+           "\"max_vms\" must be defined for ${target_name}.")
+  }
+
+  # Specialize for clang.
+  host_cc_toolchain("${target_name}_clang") {
+    ar = "llvm-ar"
+    cc = "clang -fcolor-diagnostics"
+    cxx = "clang++ -fcolor-diagnostics -stdlib=libc++"
+
+    # TODO: remove the need for this
+    extra_defines = "-DPL011_BASE=0"
+
+    if (invoker.use_platform) {
+      toolchain_args = {
+        use_platform = true
+
+        # When building for the ${target_name}, use the fake architecture to make things
+        # testable.
+        plat_arch = "fake"
+        plat_boot_flow = "//src/arch/fake:boot_flow"
+        plat_console = "//src/arch/fake:console"
+        plat_iommu = "//src/iommu:absent"
+        plat_heap_pages = invoker.heap_pages
+        plat_max_cpus = invoker.max_cpus
+        plat_max_vms = invoker.max_vms
+      }
+    }
+  }
+}
diff --git a/build/toolchain/offset_size_header.gni b/build/toolchain/offset_size_header.gni
new file mode 100644
index 0000000..b409284
--- /dev/null
+++ b/build/toolchain/offset_size_header.gni
@@ -0,0 +1,84 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This template auto-generate a C header file with "#define" constants, e.g.
+# struct sizes and member offsets.
+#
+# It uses a trick similar to other projects, e.g. Linux, where the integer
+# constant is used as an immediate in an inline assembly block. The source file
+# is compiled and the constant extracted by a script which generates the header
+# file. For easy grep-ing, the constant is compiled into a '.ascii' string,
+# surrounded by magic strings, and extracted using the 'strings' binutils tool.
+#
+# To guarantee correctness, the same source file is compiled again as part
+# of the parent target but this time the declarations are converted to
+# static_asserts to check the values at its compile-time.
+template("offset_size_header") {
+  target_lib = "${target_name}__lib"
+  target_header = "${target_name}__header"
+
+  # Compile source files into binaries that contain strings with definitions
+  # of constants.
+  static_library(target_lib) {
+    forward_variables_from(invoker,
+                           [
+                             "sources",
+                             "deps",
+                             "test_only",
+                           ])
+    defines = [ "GENERATE_BINARY" ]
+
+    # Disable LTO to force emitting assembly.
+    cflags = [ "-fno-lto" ]
+  }
+
+  # Extract strings from the static library, parse definitions and generate
+  # a header file.
+  action(target_header) {
+    forward_variables_from(invoker, [ "test_only" ])
+    lib_file = "${target_out_dir}/${target_lib}.a"
+    out_file = "${root_gen_dir}/offset_size_header/${invoker.path}"
+
+    script = "//build/toolchain/gen_offset_size_header.py"
+    args = [
+      rebase_path(lib_file, root_build_dir),
+      rebase_path(out_file, root_build_dir),
+    ]
+    deps = [
+      ":$target_lib",
+    ]
+    outputs = [
+      out_file,
+    ]
+  }
+
+  # This source_set will be compiled into the target that depends on this one.
+  # This generates static_asserts which check the constants in the generated
+  # header against compile-time structs.
+  source_set(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "sources",
+                             "test_only",
+                           ])
+    cflags = [
+      "-include",
+      invoker.path,
+    ]
+    defines = [ "VERIFY_HEADER" ]
+    deps = [
+      ":$target_header",
+    ]
+  }
+}
diff --git a/build/toolchain/platform.gni b/build/toolchain/platform.gni
new file mode 100644
index 0000000..7ff4b5a
--- /dev/null
+++ b/build/toolchain/platform.gni
@@ -0,0 +1,40 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration of the build for the platform.
+declare_args() {
+  # The name of the platform.
+  plat_name = ""
+
+  # The architecture of the platform.
+  plat_arch = ""
+
+  # Boot flow driver to be used by the platform, specified as build target.
+  plat_boot_flow = ""
+
+  # Console driver to be used for the platform, specified as build target.
+  plat_console = ""
+
+  # IOMMU driver to be used for the platform, specified as build target.
+  plat_iommu = ""
+
+  # The number of pages to allocate for the hypervisor heap.
+  plat_heap_pages = 0
+
+  # The maximum number of CPUs available on the platform.
+  plat_max_cpus = 1
+
+  # The maximum number of VMs required for the platform.
+  plat_max_vms = 0
+}
diff --git a/docs/Architecture.md b/docs/Architecture.md
new file mode 100644
index 0000000..e0261d8
--- /dev/null
+++ b/docs/Architecture.md
@@ -0,0 +1,199 @@
+# Hafnium architecture
+
+The purpose of Hafnium is to provide memory isolation between a set of security
+domains, to better separate untrusted code from security-critical code. It is
+implemented as a type-1 hypervisor, where each security domain is a VM.
+
+On AArch64 (currently the only supported architecture) it runs at EL2, while the
+VMs it manages run at EL1 (and user space applications within those VMs at EL0).
+A Secure Monitor such as
+[Trusted Firmware-A](https://www.trustedfirmware.org/about/) runs underneath it
+at EL3.
+
+Hafnium provides memory isolation between these VMs by managing their stage 2
+page tables, and using IOMMUs to restrict how DMA devices can be used to access
+memory. It must also prevent them from accessing system resources in a way which
+would allow them to escape this containment. It also provides:
+
+*   Means for VMs to communicate with each other through message passing and
+    memory sharing, according to the Arm
+    [Secure Partition Communication Interface (SPCI)](https://developer.arm.com/docs/den0077/a).
+*   Emulation of some basic hardware features such as timers.
+*   A simple paravirtualised interrupt controller for secondary VMs, as they
+    don't have access to hardware interrupts.
+*   A simple logging API for bringup and low-level debugging of VMs.
+
+See the [VM interface](VmInterface.md) documentation for more details.
+
+Hafnium makes a distinction between a **primary VM**, which would typically run
+the main user-facing operating system such as Android, and a number of
+**secondary VMs** which are smaller and exist to provide various services to the
+primary VM. The primary VM typically owns the majority of the system resources,
+and is likely to be more latency-sensitive as it is running user-facing tasks.
+Some of the differences between primary and secondary VMs are explained below.
+
+[TOC]
+
+## Security model
+
+Hafnium runs a set of VMs without trusting any of them. Neither do the VMs trust
+each other. Hafnium aims to prevent malicious software running in one VM from
+compromising any of the other VMs. Specifically, we guarantee
+**confidentiality** and **memory integrity** of each VM: no other VM should be
+able to read or modify the memory that belongs to a VM without that VM's
+consent.
+
+We do not make any guarantees of **availability** of VMs, except for the primary
+VM. In other words, a compromised primary VM may prevent secondary VMs from
+running, but not gain unauthorised access to their memory. A compromised
+secondary VM should not be able to prevent the primary VM or other secondary VMs
+from running.
+
+## Design principles
+
+Hafnium is designed with the following principles in mind:
+
+*   Open design
+    *   Hafnium is developed as open source, available for all to use,
+        contribute and scrutinise.
+*   Economy of mechanism
+    *   Hafnium strives to be as small and simple of possible, to reduce the
+        attack surface.
+    *   This also makes Hafnium more amenable to formal verification.
+*   Least privilege
+    *   Each VM is a separate security domain and is given access only to what
+        it needs, to reduce the impact if it is compromised.
+    *   Everything that doesn't strictly need to be part of Hafnium itself (in
+        EL2) should be moved to a VM (in EL1).
+*   Defence in depth
+    *   Hafnium provides an extra layer of security isolation on top of those
+        provided by the OS kernel, to better isolate sensitive workloads from
+        untrusted code.
+
+## VM model
+
+A [VM](../inc/hf/vm.h) in Hafnium consists of:
+
+*   A set of memory pages owned by and/or available to the VM, stored in the
+    stage 2 page table managed by Hafnium.
+*   One or more vCPUs. (The primary VM always has the same number of vCPUs as
+    the system has physical CPUs; secondary VMs have a configurable number.)
+*   A one page TX buffer used for sending messages to other VMs.
+*   A one page RX buffer used for receiving messages from other VMs.
+*   Some configuration information (VM ID, whitelist of allowed SMCs).
+*   Some internal state maintained by Hafnium (locks, mailbox wait lists,
+    mailbox state, log buffer).
+
+Each [vCPU](../inc/hf/vcpu.h) also has:
+
+*   A set of saved registers, for when it isn't being run on a physical CPU.
+*   A current state (switched off, ready to run, running, waiting for a message
+    or interrupt, aborted).
+*   A set of virtual interrupts which may be enabled and/or pending.
+*   Some internal locking state.
+
+VMs and their vCPUs are configured statically from a [manifest](Manifest.md)
+read at boot time. There is no way to create or destroy VMs at run time.
+
+## System resources
+
+### CPU
+
+Unlike many other type-1 hypervisors, Hafnium does not include a scheduler.
+Instead, we rely on the primary VM to handle scheduling, calling Hafnium when it
+wants to run a secondary VM's vCPU. This is because:
+
+*   In line with our design principles of _economy of mechanism_ and _least
+    privilege_, we prefer to avoid complexity in Hafnium and instead rely on VMs
+    to handle complex tasks.
+*   According to our security model, we don't guarantee availability of
+    secondary VMs, so it is acceptable for a compromised primary VM to deny CPU
+    time to secondary VMs.
+*   A lot of effort has been put into making the Linux scheduler work well to
+    maintain a responsive user experience without jank, manage power
+    efficiently, and handle heterogeneous CPU architectures such as big.LITTLE.
+    We would rather avoid re-implementing this.
+
+Hafnium therefore maintains a 1:1 mapping of physical CPUs to vCPUs for the
+primary VM, and allows the primary VM to control the power state of physical
+CPUs directly through the standard Arm Power State Coordination Interface
+(PSCI). The primary VM should then create kernel threads for each secondary VM
+vCPU and schedule them to run the vCPUs according to the
+[interface expectations defined by Hafnium](SchedulerExpectations.md). PSCI
+calls made by secondary VMs are handled by Hafnium, to change the state of the
+VM's vCPUs. In the case of (Android) Linux running in the primary VM this is
+handled by the Hafnium kernel module.
+
+#### Example
+
+For example, considering a simple system with a single physical CPU, and a
+single secondary VM with one vCPU, where the primary VM kernel has created
+**thread 1** to run the secondary VM's vCPU while **thread 2** is some other
+normal thread:
+
+![scheduler example sequence diagram](scheduler.png)
+
+1.  Scheduler chooses thread 1 to run.
+2.  Scheduler runs thread 1, and configures a physical timer to expire once the
+    quantum runs out.
+3.  Thread 1 is responsible for running a vCPU, so it asks Hafnium to run it.
+4.  Hafnium switches to the secondary VM vCPU.
+5.  Eventually the quantum runs out and the physical timer interrupts the CPU.
+6.  Hafnium traps the interrupt. Physical interrupts are owned by the primary
+    VM, so it switches back to the primary VM.
+7.  The interrupt handler in the primary VM gets invoked, and calls the
+    scheduler.
+8.  Scheduler chooses a different thread to run (thread 2).
+9.  Scheduler runs thread 2.
+
+### Memory
+
+At boot time each VM owns a mutually exclusive subset of memory pages, as
+configured by the [manifest](Manifest.md). These pages are all identity mapped
+in the stage 2 page table which Hafnium manages for the VM, so that it has full
+access to use them however it wishes.
+
+Hafnium maintains state of which VM **owns** each page, and which VMs have
+**access** to it. It does this using the stage 2 page tables of the VMs, with
+some extra application-defined bits in the page table entries. A VM may share,
+lend or donate memory pages to another VM using the appropriate SPCI requests. A
+given page of memory may never be shared with more than two VMs, either in terms
+of ownership or access. Thus, the following states are possible for each page,
+for some values of X and Y:
+
+*   Owned by VM X, accessible only by VM X
+    *   This is the initial state for each page, and also the state of a page
+        that has been donated.
+*   Owned by VM X, accessible only by VM Y
+    *   This state is reached when a page is lent.
+*   Owned by VM X, accessible by VMs X and Y
+    *   This state is reached when a page is shared.
+
+For now, in the interests of simplicity, Hafnium always uses identity mapping in
+all page tables it manages (stage 2 page tables for VMs, and stage 1 for itself)
+– i.e. the IPA (intermediate physical address) is always equal to the PA
+(physical address) in the stage 2 page table, if it is mapped at all.
+
+### Devices
+
+From Hafnium's point of view a device consists of:
+
+*   An MMIO address range (i.e. a set of pages).
+*   A set of interrupts that the device may generate.
+*   Some IOMMU configuration associated with the device.
+
+For now, each device is associated with exactly one VM, which is statically
+assigned at boot time (through the manifest) and cannot be changed at runtime.
+
+Hafnium is responsible for mapping the device's MMIO pages into the owning VM's
+stage 2 page table with the appropriate attributes, and for configuring the
+IOMMU so that the device can only access the memory that is accessible by its
+owning VM. This needs to be kept in sync as the VM's memory access changes with
+memory sharing operations. Hafnium may also need to re-initialise the IOMMU if
+the device is powered off and powered on again.
+
+The primary VM is responsible for forwarding interrupts to the owning VM, in
+case the device is owned by a secondary VM. This does mean that a compromised
+primary VM may choose not to forward interrupts, or to inject spurious
+interrupts, but this is consistent with our security model that secondary VMs
+are not guaranteed any level of availability.
diff --git a/docs/CodeStructure.md b/docs/CodeStructure.md
new file mode 100644
index 0000000..b3fa8d6
--- /dev/null
+++ b/docs/CodeStructure.md
@@ -0,0 +1,75 @@
+# Code structure
+
+The Hafnium repository contains Hafnium itself, along with unit tests and
+integration tests, a small client library for VMs, a Linux kernel module for the
+primary VM, prebuilt binaries of tools needed for building it and running tests.
+Everything is built with [GN](https://gn.googlesource.com/gn/).
+
+Hafnium can be built for an **architecture**, currently including:
+
+*   `aarch64`: 64-bit Armv8
+*   `fake`: A dummy architecture used for running unit tests on the host system.
+
+And for a **platform**, such as:
+
+*   `aem_v8a_fvp`: The Arm [Fixed Virtual Platform](FVP.md) emulator.
+*   `qemu_aarch64`: QEMU emulating an AArch64 device.
+*   `rpi4`: A Raspberry Pi 4 board.
+
+Each platform has a single associated architecture.
+
+The source tree is organised as follows:
+
+*   [`build`](../build): Common GN configuration, build scripts, and linker
+    script.
+*   [`docs`](.): Documentation
+*   [`driver/linux`](../driver/linux): Linux kernel driver for Hafnium, for use
+    in the primary VM.
+*   [`inc`](../inc): Header files...
+    *   [`hf`](../inc/hf): ... internal to Hafnium
+        *   [`arch`](../inc/hf/arch): Architecture-dependent modules, which have
+            a common interface but separate implementations per architecture.
+            This includes details of CPU initialisation, exception handling,
+            timers, page table management, and other system registers.
+        *   [`plat`](../inc/hf/plat): Platform-dependent modules, which have a
+            common interface but separate implementations per platform. This
+            includes details of the boot flow, and a UART driver for the debug
+            log console.
+    *   [`system`](../inc/system): ... which are included by the `stdatomic.h`
+        which we use from Android Clang but not really needed, so we use dummy
+        empty versions.
+    *   [`vmapi/hf`](../inc/vmapi/hf): ... for the interface exposed to VMs.
+*   [`kokoro`](../kokoro): Scripts and configuration for continuous integration
+    and presubmit checks.
+*   [`prebuilts`](../prebuilts): Prebuilt binaries needed for building Hafnium
+    or running tests.
+*   [`project`](../project): Configuration and extra code for each **project**.
+    A project is a set of one or more _platforms_ (see above) that are built
+    together. Hafnium comes with the [`reference`](../project/reference) project
+    for running it on some common emulators and development boards. To port
+    Hafnium to a new board, you can create a new project under this directory
+    with the platform or platforms you want to add, without affecting the core
+    Hafnium code.
+*   [`src`](../src): Source code for Hafnium itself in C and assembly, and
+    [unit tests](Testing.md) in C++.
+    *   [`arch`](../src/arch): Implementation of architecture-dependent modules.
+*   [`test`](../test): [Integration tests](Testing.md)
+    *   [`arch`](../test/arch): Tests for components of Hafnium that need to be
+        run on a real architecture.
+    *   [`hftest`](../test/hftest): A simple test framework that supports
+        running tests standalone on bare metal, in VMs under Hafnium, or as
+        user-space binaries under Linux under Hafnium.
+    *   [`linux`](../test/linux): Tests which are run in a Linux VM under
+        Hafnium.
+    *   [`vmapi`](../test/vmapi): Tests which are run in minimal test VMs under
+        Hafnium.
+        *   [`arch`](../test/vmapi/arch): Tests which are rely on specific
+            architectural details such as the GIC version.
+        *   [`primary_only`](../test/vmapi/primary_only): Tests which run only a
+            single (primary) VM.
+        *   [`primary_with_secondaries`](../test/vmapi/primary_with_secondaries):
+            Test which run with a primary VM and one or more secondary VMs to
+            test how they interact.
+*   [`third_party`](../third_party): Third party code needed for building
+    Hafnium.
+*   [`vmlib`](../vmlib): A small client library for VMs running under Hafnium.
diff --git a/docs/FVP.md b/docs/FVP.md
new file mode 100644
index 0000000..0db0bef
--- /dev/null
+++ b/docs/FVP.md
@@ -0,0 +1,40 @@
+# Running Hafnium under Arm FVP
+
+Arm offers a series of emulators known as Fixed Virtual Platforms (FVPs), which
+simulate various processors. They are generally more accurate to the hardware
+than QEMU, at the cost of being considerably slower. We support running
+[tests](Testing.md) on the FVP as well as QEMU.
+
+## Set up
+
+1.  Download the
+    [Armv8-A Base Platform FVP](https://developer.arm.com/products/system-design/fixed-virtual-platforms)
+    from Arm.
+1.  Unzip it to a directory called `fvp` alongside the root directory of your
+    Hafnium checkout.
+
+## Running tests
+
+To run tests with the FVP instead of QEMU, from the root directory of your
+Hafnium checkout:
+
+```shell
+$ make && kokoro/ubuntu/test.sh --fvp
+```
+
+See the `fvp` function in
+[`hftest.py`](http://cs/hafnium/test/hftest/hftest.py?q=symbol:fvp) for details
+on how this works.
+
+## Other resources
+
+When running tests under the FVP we also use a prebuilt version of TF-A, which
+is checked in under
+[`prebuilts/linux-aarch64/arm-trusted-firmware/`](https://hafnium.googlesource.com/hafnium/prebuilts/+/refs/heads/master/linux-aarch64/arm-trusted-firmware/).
+The
+[README](https://hafnium.googlesource.com/hafnium/prebuilts/+/refs/heads/master/linux-aarch64/arm-trusted-firmware/README.md)
+there has details on how it was built. The source code is available from the
+[Arm Trusted Firmware site](https://developer.trustedfirmware.org/dashboard/view/6/).
+
+Documentation of the FVP (including memory maps) is
+[available from Arm](https://static.docs.arm.com/100966/1101/fast_models_fvp_rg_100966_1101_00_en.pdf).
diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md
new file mode 100644
index 0000000..43ef6c5
--- /dev/null
+++ b/docs/GettingStarted.md
@@ -0,0 +1,95 @@
+# Getting started
+
+[TOC]
+
+## Getting the source code
+
+```shell
+git clone --recurse-submodules https://hafnium.googlesource.com/hafnium && (cd hafnium && f=`git rev-parse --git-dir`/hooks/commit-msg ; curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f)
+```
+
+To upload a commit for review:
+
+```shell
+git push origin HEAD:refs/for/master
+```
+
+Browse source at https://hafnium.googlesource.com/hafnium. Review CLs at
+https://hafnium-review.googlesource.com/.
+
+See details of [how to contribute](../CONTRIBUTING.md).
+
+## Compiling the hypervisor
+
+Install prerequisites:
+
+```shell
+sudo apt install make libssl-dev flex bison
+```
+
+By default, the hypervisor is built with clang for a few target platforms along
+with tests. Each project in the `project` directory specifies a root
+configurations of the build. Adding a project is the preferred way to extend
+support to new platforms. The target project that is built is selected by the
+`PROJECT` make variable, the default project is 'reference'.
+
+```shell
+make PROJECT=<project_name>
+```
+
+The compiled image can be found under `out/<project>`, for example the QEMU
+image is at `out/reference/qemu_aarch64_clang/hafnium.bin`.
+
+## Running on QEMU
+
+You will need at least version 2.9 for QEMU. The following command line can be
+used to run Hafnium on it:
+
+```shell
+qemu-system-aarch64 -M virt,gic_version=3 -cpu cortex-a57 -nographic -machine virtualization=true -kernel out/reference/qemu_aarch64_clang/hafnium.bin
+```
+
+Though it is admittedly not very useful because it doesn't have any virtual
+machines to run. Follow the [Hafnium RAM disk](HafniumRamDisk.md) instructions
+to create an initial RAM disk for Hafnium with Linux as the primary VM.
+
+Next, you need to create a manifest which will describe the VM to Hafnium.
+Follow the [Manifest](Manifest.md) instructions and build a DTBO with:
+```
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+		vm1 {
+			debug_name = "Linux VM";
+			kernel_filename = "vmlinuz";
+			ramdisk_filename = "initrd.img";
+		};
+	};
+};
+```
+
+Dump the DTB used by QEMU:
+```shell
+qemu-system-aarch64 -M virt,gic_version=3 -cpu cortex-a57 -nographic -machine virtualization=true -kernel out/reference/qemu_aarch64_clang/hafnium.bin -initrd initrd.img -append "rdinit=/sbin/init" -machine dumpdtb=qemu.dtb
+```
+and follow instructions in [Manifest](Manifest.md) to overlay it with the manifest.
+
+The following command line will run Hafnium, with the RAM disk just created,
+which will then boot into the primary Linux VM:
+
+```shell
+qemu-system-aarch64 -M virt,gic_version=3 -cpu cortex-a57 -nographic -machine virtualization=true -kernel out/reference/qemu_aarch64_clang/hafnium.bin -initrd initrd.img -append "rdinit=/sbin/init" -dtb qemu_with_manifest.dtb
+```
+
+## Running tests
+
+After building, presubmit tests can be run with the following command line:
+
+```shell
+./kokoro/ubuntu/test.sh
+```
+
+Read about [testing](Testing.md) for more details about the tests.
diff --git a/docs/HafniumRamDisk.md b/docs/HafniumRamDisk.md
new file mode 100644
index 0000000..c09568d
--- /dev/null
+++ b/docs/HafniumRamDisk.md
@@ -0,0 +1,20 @@
+# Hafnium RAM disk
+
+Hafnium expects to find the following files in the root directory of its RAM
+disk:
+
+*   `vmlinuz` -- the kernel of the primary VM.
+*   `initrd.img` -- the initial ramdisk of the primary VM.
+*   kernels for the secondary VMs, whose names are described in the manifest.
+
+Follow the [preparing Linux](PreparingLinux.md) instructions to produce
+`vmlinuz` and `initrd.img` for a basic Linux primary VM.
+
+## Create a RAM disk for Hafnium
+
+Assuming that a subdirectory called `initrd` contains the files listed in the
+previous section, we can build `initrd.img` with the following command:
+
+```shell
+cd initrd; find . | cpio -o > ../initrd.img; cd -
+```
diff --git a/docs/HermeticBuild.md b/docs/HermeticBuild.md
new file mode 100644
index 0000000..1d3f66a
--- /dev/null
+++ b/docs/HermeticBuild.md
@@ -0,0 +1,95 @@
+# Hermetic build
+
+Hafnium build is not hermetic as it uses some system tools and libraries, e.g.
+`bison` and `libssl`. To ensure consistency and repeatability, the team
+maintains and periodically publishes a container image as the reference build
+environment. The image is hosted on Google Cloud Platform as
+`eu.gcr.io/hafnium-build/hafnium_ci`.
+
+Building inside a container is always enabled only for Kokoro pre-submit tests
+but can be enabled for local builds too. It is disabled by default as it
+requires the use of Docker which currently supports rootless containers only in
+nightly builds. As rootless container tools mature, Hafnium may change the
+default settings. For now, running the hermetic build locally is intended
+primarily to reproduce issues in pre-submit tests.
+
+[TOC]
+
+## Installing Docker
+
+### Stable
+
+If you don't mind running a Docker daemon with root privileges on your system,
+you can follow the [official guide](https://docs.docker.com/install/) to install
+Docker, or [go/installdocker](https://goto.google.com/installdocker) if you are
+a Googler.
+
+Because the daemon runs as root, files generated by the container are owned by
+root as well. To work around this, the build will automatically derive a local
+container image from the base container, adding user `hafnium` with the same
+UID/GID as the local user.
+
+### Nightly with rootless
+
+The latest nightly version of Docker has support for running containers with
+user namespaces, thus eliminating the need for a daemon with root privileges. It
+can be installed into the local user's `bin` directory with a script:
+
+```shell
+curl -fsSL https://get.docker.com/rootless -o get-docker.sh
+sh get-docker.sh
+```
+
+The script will also walk you through the installation of dependencies, changes
+to system configuration files and environment variable values needed by the
+client to discover the rootless daemon.
+
+## Enabling for local builds
+
+Hermetic builds are controlled by the `HAFNIUM_HERMETIC_BUILD` environment
+variable. Setting it to `true` instructs the build to run commands inside the
+container. Any other value disables the feature.
+
+To always enable hermetic builds, put this line in your `~/.bashrc`:
+
+```shell
+export HAFNIUM_HERMETIC_BUILD=true
+```
+
+When you now run `make`, you should see the following line:
+
+```shell
+$ make
+Running in container: make all
+...
+```
+
+## Running commands inside the container
+
+An arbitrary command can be executed inside the container with
+`build/run_in_container.sh [-i] <command> ...`. This is done automatically
+inside `Makefile` and `kokoro/ubuntu/build.sh` which detect whether they are
+already running inside the container and respawn themselves using
+`run_in_container.sh` if not.
+
+For example, you can spawn a shell with:
+
+```shell
+./build/run_in_container.sh -i bash
+```
+
+## Building container image
+
+The container image is defined in `build/docker/Dockerfile` and can be built
+locally:
+
+```shell
+./build/docker/build.sh
+```
+
+Owners of the `hafnium-build` GCP repository can publish the new image (requires
+[go/cloud-sdk](https://goto.google.com/cloud-sdk) installed and authenticated):
+
+```shell
+./build/docker/publish.sh
+```
diff --git a/docs/Manifest.md b/docs/Manifest.md
new file mode 100644
index 0000000..8552fb1
--- /dev/null
+++ b/docs/Manifest.md
@@ -0,0 +1,98 @@
+# Hafnium Manifest
+
+[TOC]
+
+## Format
+
+The format of the manifest is a simple DeviceTree overlay:
+
+```
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+
+		vm1 {
+			debug_name = "name";
+			kernel_filename = "vmlinuz";
+			ramdisk_filename = "initrd.img";
+		};
+
+		vm2 {
+			debug_name = "name";
+			kernel_filename = "filename";
+			vcpu_count = <N>;
+			mem_size = <M>;
+		};
+		...
+	};
+};
+```
+
+Note: `&{/}` is a syntactic sugar expanded by the DTC compiler. Make sure to
+use the DTC in `prebuilts/` as the version packaged with your OS may not support
+it yet.
+
+## Example
+
+The following manifest defines a primary VM with two secondary VMs. The first
+secondary VM has 1MB of memory, 2 CPUs and kernel image called `kernel0`
+(matches filename in Hafnium's [ramdisk](HafniumRamDisk.md)). The second has 2MB
+of memory, 4 CPUs and, by omitting the `kernel_filename` property, a kernel
+preloaded into memory. The primary VM is given all remaining memory, the same
+number of CPUs as the hardware, a kernel image called `vmlinuz` and a ramdisk
+`initrd.img`. Secondaries cannot have a ramdisk.
+
+```
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+
+		vm1 {
+			debug_name = "primary VM";
+			kernel_filename = "vmlinuz";
+			ramdisk_filename = "initrd.img";
+
+			smc_whitelist = <
+				0x04000000
+				0x3200ffff
+				>;
+		};
+
+		vm2 {
+			debug_name = "secondary VM 1";
+			kernel_filename = "kernel0";
+			vcpu_count = <2>;
+			mem_size = <0x100000>;
+
+			smc_whitelist_permissive;
+		};
+
+		vm3 {
+			debug_name = "secondary VM 2";
+			vcpu_count = <4>;
+			mem_size = <0x200000>;
+		};
+	};
+};
+```
+
+## Compiling
+
+Hafnium expects the manifest as part of the board FDT, i.e. DeviceTree in binary
+format (DTB).
+
+First, compile the manifest into a DTBO (binary overlay):
+```shell
+prebuilts/linux-x64/dtc/dtc -I dts -O dtb --out-version 17 -o manifest.dtbo <manifest_source_file>
+```
+
+Then overlay it with the DTB of your board:
+```shell
+prebuilts/linux-x64/dtc/fdtoverlay -i <board DTB> -o <output DTB> manifest.dtbo
+```
diff --git a/docs/PreparingLinux.md b/docs/PreparingLinux.md
new file mode 100644
index 0000000..3332096
--- /dev/null
+++ b/docs/PreparingLinux.md
@@ -0,0 +1,88 @@
+# Preparing Linux
+
+To boot Linux, a kernel image (`vmlinuz`) and a suitable initial RAM disk
+(`initrd.img`) need to be created.
+
+[TOC]
+
+## Build the kernel
+
+The Linux kernel for the primary VM can be built using the following
+command-line:
+
+```shell
+git clone https://github.com/torvalds/linux.git
+cd linux
+ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make defconfig
+ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make -j24
+```
+
+The compiled image is stored in `arch/arm64/boot/Image`. This will later be
+copied to the Hafnium RAM disk's root as `vmlinuz`.
+
+## Build the kernel Module
+
+From the Hafnium root directory, the following commands can be used to compile
+the kernel module, replacing `<kernel-path>` with the path to the kernel checked
+out in the previous section:
+
+```shell
+cd hafnium/driver/linux/
+ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- KERNEL_PATH=<kernel-path> make
+```
+
+The compiled module is called `hafnium.ko`, and will later be copied into the
+RAM disk for Linux.
+
+## Build Busybox
+
+To make Linux useful, it needs a shell. These following instructions will
+construct a file system for the Linux RAM disk with the BusyBox shell as the
+init process.
+
+```shell
+git clone git://busybox.net/busybox.git
+cd busybox
+ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make defconfig
+ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make menuconfig
+```
+
+At this point you should ensure that the option `Settings > Build static binary
+(no shared libs)` is selected. Then you can proceed with the following commands:
+
+```shell
+ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make -j24
+ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make install
+cd _install
+mkdir proc
+mkdir sys
+mkdir -p etc/init.d
+cat <<EOF > etc/init.d/rcS
+#!bin/sh
+mount -t proc none /proc
+mount -t sysfs none /sys
+EOF
+chmod u+x etc/init.d/rcS
+grep -v tty ../examples/inittab > ./etc/inittab
+```
+
+## Create a RAM disk for Linux
+
+At this point you can copy into the current directory additional files you may
+want in the RAM disk, for example, the kernel module built in the previous
+section. Assuming the BusyBox root directory is in the same parent directory as
+the Hafnium root directory:
+
+```shell
+cp ../../hafnium/driver/linux/hafnium.ko .
+```
+
+Then run the following commands:
+
+```shell
+find . | cpio -o -H newc | gzip > ../initrd.img
+cd ..
+```
+
+The resulting file is `initrd.img`. It should be copied to the Hafnium RAM
+disk's root.
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000..635da70
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,20 @@
+# Documentation index
+
+*   [README](../README.md)
+*   [Contributing](../CONTRIBUTING.md)
+*   [Style guide](StyleGuide.md)
+
+## Building and running Hafnium
+*   [Getting started](GettingStarted.md)
+*   [Building a RAM disk with VMs for Hafnium to run](HafniumRamDisk.md)
+*   [Manifest format](Manifest.md)
+*   [Building a Linux image for Hafnium](PreparingLinux.md)
+*   [Building Hafnium hermetically with Docker](HermeticBuild.md)
+*   [Running Hafnium under Arm FVP](FVP.md)
+
+## Design details
+*   [Architecture](Architecture.md)
+*   [Code structure](CodeStructure.md)
+*   [Test infrastructure](Testing.md)
+*   [The interface Hafnium provides to VMs](VmInterface.md)
+*   [Scheduler VM expectations](SchedulerExpectations.md)
diff --git a/docs/SchedulerExpectations.md b/docs/SchedulerExpectations.md
new file mode 100644
index 0000000..7dfe193
--- /dev/null
+++ b/docs/SchedulerExpectations.md
@@ -0,0 +1,107 @@
+# Scheduler VM expectations
+
+Hafnium requires there to be a special 'primary' or 'scheduler' VM which is
+responsible for scheduling the other VMs. There are some particular expectations
+on this VM that are required for the rest of the system to function normally.
+
+[TOC]
+
+## Scheduling
+
+The scheduler VM is responsible for scheduling the vCPUs of all the other VMs.
+It should request information about the VMs in the system using the
+`SPCI_PARTITION_INFO_GET` function, and then schedule their vCPUs as it wishes.
+The recommended way of doing this is to create a kernel thread for each vCPU,
+which will repeatedly run that vCPU by calling `SPCI_RUN`.
+
+`SPCI_RUN` will return one of several possible functions, which must be handled
+as follows:
+
+### `SPCI_INTERRUPT`
+
+The vCPU has been preempted but still has work to do. If the scheduling quantum
+has not expired, the scheduler MUST call `hf_vcpu_run` on the vCPU to allow it
+to continue.
+
+### `SPCI_YIELD`
+
+The vCPU has voluntarily yielded the CPU. The scheduler SHOULD take a scheduling
+decision to give cycles to those that need them but MUST call `hf_vcpu_run` on
+the vCPU at a later point.
+
+### `SPCI_MSG_WAIT`
+
+The vCPU is blocked waiting for a message. The scheduler MUST take it off the
+run queue and not call `SPCI_RUN` on the vCPU until it has either:
+
+*   injected an interrupt
+*   sent it a message
+*   received `HF_SPCI_RUN_WAKE_UP` for it from another vCPU
+*   the timeout provided in `w2` is not `SPCI_SLEEP_INDEFINITE` and the
+    specified duration has expired.
+
+### `SPCI_MSG_SEND`
+
+A message has been sent by the vCPU. If the recipient is the scheduler VM itself
+then it can handle it as it pleases. Otherwise the scheduler MUST run a vCPU
+from the recipient VM and priority SHOULD be given to those vCPUs that are
+waiting for a message. The scheduler should call SPCI_RUN again on the sending
+VM as usual.
+
+### `SPCI_RX_RELEASE`
+
+The vCPU has made the mailbox writable and there are pending waiters. The
+scheduler MUST call `hf_mailbox_waiter_get()` repeatedly and notify all waiters
+by injecting an `HF_MAILBOX_WRITABLE_INTID` interrupt. The scheduler should call
+SPCI_RUN again on the sending VM as usual.
+
+### `HF_SPCI_RUN_WAIT_FOR_INTERRUPT`
+
+_This is a Hafnium-specific function not part of the SPCI standard._
+
+The vCPU is blocked waiting for an interrupt. The scheduler MUST take it off the
+run queue and not call `SPCI_RUN` on the vCPU until it has either:
+
+*   injected an interrupt
+*   received `HF_SPCI_RUN_WAKE_UP` for it from another vCPU
+*   the timeout provided in `w2` is not `SPCI_SLEEP_INDEFINITE` and the
+    specified duration has expired.
+
+### `HF_SPCI_RUN_WAKE_UP`
+
+_This is a Hafnium-specific function not part of the SPCI standard._
+
+Hafnium would like `hf_vcpu_run` to be called on another vCPU, specified by
+`hf_vcpu_run_return.wake_up`. The scheduler MUST either wake the vCPU in
+question up if it is blocked, or preempt and re-run it if it is already running
+somewhere. This gives Hafnium a chance to update any CPU state which might have
+changed. The scheduler should call SPCI_RUN again on the sending VM as usual.
+
+### `SPCI_ERROR`
+
+#### `SPCI_ABORTED`
+
+The vCPU has aborted triggering the whole VM to abort. The scheduler MUST treat
+this the same as `HF_SPCI_RUN_WAKE_UP` for all the other vCPUs of the VM. For
+this vCPU the scheduler SHOULD either never call SPCI_RUN on the vCPU again, or
+treat it the same as `HF_SPCI_RUN_WAIT_FOR_INTERRUPT`.
+
+#### Any other error code
+
+This should not happen if the scheduler VM has called `SPCI_RUN` correctly, but
+in case there is some other error it should be logged. The scheduler SHOULD
+either try again or suspend the vCPU indefinitely.
+
+## Interrupt handling
+
+The scheduler VM is responsible for handling all hardware interrupts. Many of
+these will be intended for the scheduler VM itself and it can handle them as
+usual. However, it must also:
+
+*   Enable, handle and ignore interrupts for the non-secure hypervisor physical
+    timer (PPI 10, IRQ 26).
+*   Forward interrupts intended for secondary VMs to an appropriate vCPU of the
+    VM by calling `hf_interrupt_inject` and then running the vCPU as usual with
+    `SPCI_RUN`. (If the vCPU is already running at the time that
+    `hf_interrupt_inject` is called then it must be preempted and run again so
+    that Hafnium can inject the interrupt.)
diff --git a/docs/StyleGuide.md b/docs/StyleGuide.md
new file mode 100644
index 0000000..3cdd290
--- /dev/null
+++ b/docs/StyleGuide.md
@@ -0,0 +1,83 @@
+# Style guide
+
+Hafnium's coding style has been based on the
+[Linux style](https://www.kernel.org/doc/html/v4.17/process/coding-style.html)
+with explicit modifications:
+
+*   Always use braces for conditionals and loops. (No SSL `goto fail;`, thanks.)
+
+Following this, we generally fall back to the subset of the
+[Google C++ style guide](https://google.github.io/styleguide/cppguide.html) that
+is applicable to C.
+
+We try to automate this where possible with clang-format and clang-tidy but that
+doesn't capture everything we'd like today. Where the style enforced by this
+tooling conflicts with what is in this document we accept what the tooling
+requires, and try to improve it if possible.
+
+[TOC]
+
+## Clarifications
+
+*   Yes, it does mean all variables are declared, C90-style, at the top of
+    scope, even those loop induction variables.
+*   Linux encourages no braces around single-statement branches. We follow
+    Google and require braces around all scope blocks.
+
+## Naming symbols
+
+*   Arch-specific functions should start with `arch_`.
+*   Platform-specific functions should start with `plat_`.
+*   Non-static functions should generally start with the name of the file they
+    are declared in (after the `arch_` or `plat_` prefix if appropriate), though
+    there are quite a few exceptions to this rule.
+*   Prefer `x_count` over `num_x`.
+
+## Prose
+
+These rules apply to comments and other natural language text.
+
+*   Capitalize acronyms.
+    *   CPU, vCPU, VM, EL2, SPCI, QEMU
+*   Spell out Hafnium in full, not Hf.
+*   Use single spaces.
+*   Sentences end with full stops.
+*   If the comment fits on one line use `/* */`, otherwise space it out:
+
+    ```
+    /*
+     * Informative long comment
+     * with extra information.
+     */
+    ```
+
+*   Doc-ish comments start with `/**`.
+
+    *   Use for:
+        *   Function definitions (not declarations)
+        *   Struct declarations
+        *   Enum values
+    *   Do not use for:
+        *   Macros
+        *   Definitions of globals
+
+*   References to code symbols use backticks, e.g. `` `my_symbol` ``.
+
+## Coding practices
+
+*   Function macros should be functions instead, that way you get types.
+*   Lock ordering is described at the top of *api.c*.
+*   Use opaque types to avoid implicit casts when it will help avoid mistakes.
+    e.g. *addr.h*
+*   Avoid inline casting. C doesn't give much protection so be formal about the
+    transformations. e.g. *addr.h*
+*   If a function acquires a resource, there must be a single exit path to free
+    the resource. Tracking down multiple exit points is hard and requires
+    duplicated code which is harder. This may require splitting functions into
+    subfunctions. Early exit is okay if there aren't any clean up tasks.
+*   Don't use function pointers. It makes analysis hard and is often a target of
+    attacks.
+*   Be liberal with CHECK. Use it to assert pre-/post- conditions.
+*   No self-modifying code.
+*   Build targets should include all the direct dependencies for their sources,
+    where possible, rather than relying on transitive dependencies.
diff --git a/docs/Testing.md b/docs/Testing.md
new file mode 100644
index 0000000..d79aafc
--- /dev/null
+++ b/docs/Testing.md
@@ -0,0 +1,82 @@
+# Testing
+
+[TOC]
+
+## Overview
+
+Hafnium has 4 main kinds of tests:
+
+1.  Host tests
+    *   Unit tests of core functionality, e.g. page table manipulation.
+    *   Source in `src/*_test.cc`.
+    *   Using the [Google Test](https://github.com/google/googletest) framework,
+        built against 'fake' architecture (`src/arch/fake`).
+1.  Arch tests
+    *   Architecture-specific unit tests, e.g. MMU setup.
+    *   Source under `test/arch`.
+    *   Using our own _hftest_ framework, with `standalone_main.c`.
+    *   Build own hypervisor image, run in EL2.
+1.  VM API tests
+    *   Exercise hypervisor API from both primary and secondary VMs.
+    *   Source under `test/vmapi`.
+    *   Tests are run from the primary VM under a normal build of the Hafnium
+        hypervisor, possibly communicating with a helper service in one or more
+        secondary VMs.
+    *   Using our own _hftest_ framework, with `standalone_main.c` for the
+        primary VM and `hftest_service.c` for secondary VMs.
+    *   Build own primary and secondary VMs, run in EL1 under actual Hafnium
+        image.
+1.  Linux tests
+    *   Exercise the Hafnium Linux kernel module.
+    *   Source under `test/linux`.
+    *   Tests are run from userspace (PID 1) under Linux in the primary VM under
+        Hafnium, possibly with other secondary VMs.
+    *   Using our own _hftest_ framework, with `linux_main.c`.
+
+Host tests run directly on the host machine where they are built, whereas the
+other 3 types can run under an emulator such as QEMU, or on real hardware.
+
+## Presubmit
+
+Presubmit builds everything, runs all tests and checks the source for formatting
+and lint errors. This can be run locally with:
+
+```shell
+./kokoro/ubuntu/build.sh
+```
+
+Or to just run the tests after having built everything manually run:
+
+```shell
+./kokoro/ubuntu/test.sh
+```
+
+## QEMU tests
+
+These tests boot Hafnium on QEMU and the VMs make calls to Hafnium to test its
+behaviour. They can also be run on the Arm [FVP](FVP.md) and in some cases on
+real hardware.
+
+### hftest
+
+Having a framework for tests makes them easier to read and write. _hftest_ is a
+framework to meet the needs of VM based tests for Hafnium. It consists of:
+
+*   assertions
+*   test declarations
+*   base VM image
+*   driver script
+
+Assertions should be familiar from other testing libraries. They make use of
+C11's `_Generic` expressions for type genericity.
+
+Test declarations name the test and the suite that the test is part of.
+Declarations are converted into descriptors stored in the `.hftest` section of
+the VM image which allows the image to inspect the structure of the tests it
+contains. The linker sorts the descriptors by their symbol name which is how
+descriptors from the same suite are grouped together for easier parsing.
+
+The base VM image offers a command line interface, via the bootargs, to query
+the tests in the image and to run specific tests. The driver script uses this
+interface to execute tests, each with a fresh QEMU boot to give a fresh
+environment.
diff --git a/docs/VmInterface.md b/docs/VmInterface.md
new file mode 100644
index 0000000..47de251
--- /dev/null
+++ b/docs/VmInterface.md
@@ -0,0 +1,177 @@
+# VM interface
+
+This page provides an overview of the interface Hafnium provides to VMs. Hafnium
+makes a distinction between the 'primary VM', which controls scheduling and has
+more direct access to some hardware, and 'secondary VMs' which exist mostly to
+provide services to the primary VM, and have a more paravirtualised interface.
+The intention is that the primary VM can run a mostly unmodified operating
+system (such as Linux) with the addition of a Hafnium driver which
+[fulfils certain expectations](SchedulerExpectations.md), while secondary VMs
+will run more specialised trusted OSes or bare-metal code which is designed with
+Hafnium in mind.
+
+The interface documented here is what is planned for the first release of
+Hafnium, not necessarily what is currently implemented.
+
+[TOC]
+
+## CPU scheduling
+
+The primary VM will have one vCPU for each physical CPU, and control the
+scheduling.
+
+Secondary VMs will have a configurable number of vCPUs, scheduled on arbitrary
+physical CPUs at the whims of the primary VM scheduler.
+
+All VMs will start with a single active vCPU. Subsequent vCPUs can be started
+through PSCI.
+
+## PSCI
+
+The primary VM will be able to control the physical CPUs through the following
+PSCI 1.1 calls, which will be forwarded to the underlying implementation in EL3:
+
+*   PSCI_VERSION
+*   PSCI_FEATURES
+*   PSCI_SYSTEM_OFF
+*   PSCI_SYSTEM_RESET
+*   PSCI_AFFINITY_INFO
+*   PSCI_CPU_SUSPEND
+*   PSCI_CPU_OFF
+*   PSCI_CPU_ON
+
+All other PSCI calls are unsupported.
+
+Secondary VMs will be able to control their vCPUs through the following PSCI 1.1
+calls, which will be implemented by Hafnium:
+
+*   PSCI_VERSION
+*   PSCI_FEATURES
+*   PSCI_AFFINITY_INFO
+*   PSCI_CPU_SUSPEND
+*   PSCI_CPU_OFF
+*   PSCI_CPU_ON
+
+All other PSCI calls are unsupported.
+
+## Hardware timers
+
+The primary VM will have access to both the physical and virtual EL1 timers
+through the usual control registers (`CNT[PV]_TVAL_EL0` and `CNT[PV]_CTL_EL0`).
+
+Secondary VMs will have access to the virtual timer only, which will be emulated
+with help from the kernel driver in the primary VM.
+
+## Interrupts
+
+The primary VM will have direct access to control the physical GIC, and receive
+all interrupts (other than anything already trapped by TrustZone). It will be
+responsible for forwarding any necessary interrupts to secondary VMs. The
+Interrupt Translation Service (ITS) will be disabled by Hafnium so that it
+cannot be used to circumvent access controls.
+
+Secondary VMs will have access to a simple paravirtualized interrupt controller
+through two hypercalls: one to enable or disable a given virtual interrupt ID,
+and one to get and acknowledge the next pending interrupt. There is no concept
+of interrupt priorities or a distinction between edge and level triggered
+interrupts. Secondary VMs may also inject interrupts into their own vCPUs.
+
+## Performance counters
+
+VMs will be blocked from accessing performance counter registers (for the
+performance monitor extensions described in chapter D5 of the Armv8-A reference
+manual) in production, to prevent them from being used as a side channel to leak
+data between VMs.
+
+Hafnium may allow VMs to use them in debug builds.
+
+## Debug registers
+
+VMs will be blocked from accessing debug registers in production builds, to
+prevent them from being used to circumvent access controls.
+
+Hafnium may allow VMs to use these registers in debug builds.
+
+## RAS Extension registers
+
+Secondary VMs will be blocked from using registers associated with the RAS
+Extension.
+
+## Asynchronous message passing
+
+VMs will be able to send messages of up to 4 KiB to each other asynchronously,
+with no queueing, as specified by SPCI.
+
+## Memory
+
+VMs will statically be given access to mutually-exclusive regions of the
+physical address space at boot. This includes MMIO space for controlling
+devices, plus a fixed amount of RAM for secondaries, and all remaining address
+space to the primary. Note that this means that only one VM can control any
+given page of MMIO registers for a device.
+
+VMs may choose to donate or share their memory with other VMs at runtime. Any
+given page may be shared with at most 2 VMs at once (including the original
+owning VM). Memory which has been donated or shared may not be forcefully
+reclaimed, but the VM with which it was shared may choose to return it.
+
+## Cache
+
+VMs will be blocked from using cache maintenance instructions that operate by
+set/way. These operations are difficult to virtualize, and could expose the
+system to side-channel attacks.
+
+## Logging
+
+VMs may send a character to a shared log by means of a hypercall or SMC call.
+These log messages will be buffered per VM to make complete lines, then output
+to a Hafnium-owned UART and saved in a shared ring buffer which may be extracted
+from RAM dumps. VM IDs will be prepended to these logs.
+
+This log API is intended for use in early bringup and low-level debugging. No
+sensitive data should be logged through it. Higher level logs can be sent to the
+primary VM through the asynchronous message passing mechanism described above,
+or through shared memory.
+
+## Configuration
+
+Hafnium will read configuration from a flattened device tree blob (FDT). This
+may either be the same device tree used for the other details of the system or a
+separate minimal one just for Hafnium. This will include at least:
+
+*   The available RAM.
+*   The number of secondary VMs, how many vCPUs each should have, how much
+    memory to assign to each of them, and where to load their initial images.
+    (Most likely the initial image will be a minimal loader supplied with
+    Hafnium which will validate and load the rest of the image from the primary
+    later on.)
+*   Which devices exist on the system, their details (MMIO regions, interrupts
+    and SYSMMU details), and which VM each is assigned to.
+    *   A single physical device may be split into multiple logical ‘devices’
+        from Hafnium’s point of view if necessary to have different VMs own
+        different parts of it.
+*   A whitelist of which SMC calls each VM is allowed to make.
+
+## Failure handling
+
+If a secondary VM tries to do something it shouldn't, Hafnium will either inject
+a fault or kill it and inform the primary VM. The primary VM may choose to
+restart the system or to continue without the secondary VM.
+
+If the primary VM tries to do something it shouldn't, Hafnium will either inject
+a fault or restart the system.
+
+## TrustZone communication
+
+The primary VM will be able to communicate with a TEE running in TrustZone
+either through SPCI messages or through whitelisted SMC calls, and through
+shared memory.
+
+## Other SMC calls
+
+Other than the PSCI calls described above and those used to communicate with
+Hafnium, all other SMC calls will be blocked by default. Hafnium will allow SMC
+calls to be whitelisted on a per-VM, per-function ID basis, as part of the
+static configuration described above. These whitelisted SMC calls will be
+forwarded to the EL3 handler with the client ID (as described by the SMCCC) set
+to the calling VM's ID.
diff --git a/docs/scheduler.png b/docs/scheduler.png
new file mode 100644
index 0000000..afce871
--- /dev/null
+++ b/docs/scheduler.png
Binary files differ
diff --git a/driver/linux/.gitignore b/driver/linux/.gitignore
new file mode 100644
index 0000000..8fd2089
--- /dev/null
+++ b/driver/linux/.gitignore
@@ -0,0 +1,7 @@
+*.ko
+*.mod.c
+*.o
+.*o.cmd
+.tmp_versions/
+Module.symvers
+modules.order
diff --git a/driver/linux/BUILD.gn b/driver/linux/BUILD.gn
new file mode 100644
index 0000000..6c7d688
--- /dev/null
+++ b/driver/linux/BUILD.gn
@@ -0,0 +1,21 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/linux/linux.gni")
+
+linux_kernel_module("linux") {
+  module_name = "hafnium"
+  kernel_target = "//third_party/linux:linux"
+  kernel_dir = "//third_party/linux"
+}
diff --git a/driver/linux/LICENSE b/driver/linux/LICENSE
new file mode 100644
index 0000000..d159169
--- /dev/null
+++ b/driver/linux/LICENSE
@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/driver/linux/Makefile b/driver/linux/Makefile
new file mode 100644
index 0000000..0add537
--- /dev/null
+++ b/driver/linux/Makefile
@@ -0,0 +1,45 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# version 2 as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# By default, assume this was checked out as a submodule of the Hafnium repo
+# and that Linux was checked out along side that checkout. These paths can be
+# overridden if that assumption is incorrect.
+HAFNIUM_PATH ?= $(CURDIR)/../..
+
+ifneq ($(KERNELRELEASE),)
+
+obj-m += hafnium.o
+
+hafnium-y += main.o
+hafnium-y += vmlib/aarch64/call.o
+hafnium-y += vmlib/spci.o
+
+ccflags-y = -I$(HAFNIUM_PATH)/inc/vmapi -I$(M)/inc
+
+else
+
+KERNEL_PATH ?= $(HAFNIUM_PATH)/third_party/linux
+ARCH ?= arm64
+CROSS_COMPILE ?= aarch64-linux-gnu-
+CHECKPATCH ?= $(KERNEL_PATH)/scripts/checkpatch.pl -q
+
+all:
+	cp -r $(HAFNIUM_PATH)/vmlib/ $(CURDIR)
+	make -C $(KERNEL_PATH) HAFNIUM_PATH=$(HAFNIUM_PATH) M=$(CURDIR) O=$(O) ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) modules
+
+clean:
+	make -C $(KERNEL_PATH) HAFNIUM_PATH=$(HAFNIUM_PATH) M=$(CURDIR) O=$(O) clean
+	rm -rf vmlib
+
+checkpatch:
+	$(CHECKPATCH) -f main.c
+
+endif
diff --git a/inc/uapi/hf/socket.h b/driver/linux/inc/uapi/hf/socket.h
similarity index 100%
rename from inc/uapi/hf/socket.h
rename to driver/linux/inc/uapi/hf/socket.h
diff --git a/main.c b/driver/linux/main.c
similarity index 100%
rename from main.c
rename to driver/linux/main.c
diff --git a/inc/hf/abort.h b/inc/hf/abort.h
new file mode 100644
index 0000000..f5548e0
--- /dev/null
+++ b/inc/hf/abort.h
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdnoreturn.h>
+
+noreturn void abort(void);
diff --git a/inc/hf/addr.h b/inc/hf/addr.h
new file mode 100644
index 0000000..7845aa9
--- /dev/null
+++ b/inc/hf/addr.h
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hf/arch/types.h"
+
+/** An opaque type for a physical address. */
+typedef struct {
+	uintpaddr_t pa;
+} paddr_t;
+
+/** An opaque type for an intermediate physical address. */
+typedef struct {
+	uintpaddr_t ipa;
+} ipaddr_t;
+
+/** An opaque type for a virtual address. */
+typedef struct {
+	uintvaddr_t va;
+} vaddr_t;
+
+/**
+ * Initializes a physical address.
+ */
+static inline paddr_t pa_init(uintpaddr_t p)
+{
+	return (paddr_t){.pa = p};
+}
+
+/**
+ * Extracts the absolute physical address.
+ */
+static inline uintpaddr_t pa_addr(paddr_t pa)
+{
+	return pa.pa;
+}
+
+/**
+ * Advances a physical address.
+ */
+static inline paddr_t pa_add(paddr_t pa, size_t n)
+{
+	return pa_init(pa_addr(pa) + n);
+}
+
+/**
+ * Returns the difference between two physical addresses.
+ */
+static inline size_t pa_difference(paddr_t start, paddr_t end)
+{
+	return pa_addr(end) - pa_addr(start);
+}
+
+/**
+ * Initializes an intermeditate physical address.
+ */
+static inline ipaddr_t ipa_init(uintpaddr_t ipa)
+{
+	return (ipaddr_t){.ipa = ipa};
+}
+
+/**
+ * Extracts the absolute intermediate physical address.
+ */
+static inline uintpaddr_t ipa_addr(ipaddr_t ipa)
+{
+	return ipa.ipa;
+}
+
+/**
+ * Advances an intermediate physical address.
+ */
+static inline ipaddr_t ipa_add(ipaddr_t ipa, size_t n)
+{
+	return ipa_init(ipa_addr(ipa) + n);
+}
+
+/**
+ * Initializes a virtual address.
+ */
+static inline vaddr_t va_init(uintvaddr_t v)
+{
+	return (vaddr_t){.va = v};
+}
+
+/**
+ * Extracts the absolute virtual address.
+ */
+static inline uintvaddr_t va_addr(vaddr_t va)
+{
+	return va.va;
+}
+
+/**
+ * Casts a physical address to a virtual address.
+ */
+static inline vaddr_t va_from_pa(paddr_t pa)
+{
+	return va_init(pa_addr(pa));
+}
+
+/**
+ * Casts a physical address to an intermediate physical address.
+ */
+static inline ipaddr_t ipa_from_pa(paddr_t pa)
+{
+	return ipa_init(pa_addr(pa));
+}
+
+/**
+ * Casts a virtual address to a physical address.
+ */
+static inline paddr_t pa_from_va(vaddr_t va)
+{
+	return pa_init(va_addr(va));
+}
+
+/**
+ * Casts an intermediate physical address to a physical address.
+ */
+static inline paddr_t pa_from_ipa(ipaddr_t ipa)
+{
+	return pa_init(ipa_addr(ipa));
+}
+
+/**
+ * Casts a pointer to a virtual address.
+ */
+static inline vaddr_t va_from_ptr(const void *p)
+{
+	return (vaddr_t){.va = (uintvaddr_t)p};
+}
+
+/**
+ * Casts a virtual address to a pointer. Only use when the virtual address is
+ * mapped for the calling context.
+ * TODO: check the mapping for a range and return a memiter?
+ */
+static inline void *ptr_from_va(vaddr_t va)
+{
+	return (void *)va_addr(va);
+}
diff --git a/inc/hf/api.h b/inc/hf/api.h
new file mode 100644
index 0000000..cfbff86
--- /dev/null
+++ b/inc/hf/api.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/cpu.h"
+#include "hf/mpool.h"
+#include "hf/vm.h"
+
+#include "vmapi/hf/call.h"
+#include "vmapi/hf/spci.h"
+
+void api_init(struct mpool *ppool);
+spci_vm_count_t api_vm_get_count(void);
+spci_vcpu_count_t api_vcpu_get_count(spci_vm_id_t vm_id,
+				     const struct vcpu *current);
+void api_regs_state_saved(struct vcpu *vcpu);
+int64_t api_mailbox_writable_get(const struct vcpu *current);
+int64_t api_mailbox_waiter_get(spci_vm_id_t vm_id, const struct vcpu *current);
+int64_t api_debug_log(char c, struct vcpu *current);
+
+struct vcpu *api_preempt(struct vcpu *current);
+struct vcpu *api_wait_for_interrupt(struct vcpu *current);
+struct vcpu *api_vcpu_off(struct vcpu *current);
+struct vcpu *api_abort(struct vcpu *current);
+struct vcpu *api_wake_up(struct vcpu *current, struct vcpu *target_vcpu);
+
+int64_t api_interrupt_enable(uint32_t intid, bool enable, struct vcpu *current);
+uint32_t api_interrupt_get(struct vcpu *current);
+int64_t api_interrupt_inject(spci_vm_id_t target_vm_id,
+			     spci_vcpu_index_t target_vcpu_idx, uint32_t intid,
+			     struct vcpu *current, struct vcpu **next);
+
+struct spci_value api_spci_msg_send(spci_vm_id_t sender_vm_id,
+				    spci_vm_id_t receiver_vm_id, uint32_t size,
+				    uint32_t attributes, struct vcpu *current,
+				    struct vcpu **next);
+struct spci_value api_spci_msg_recv(bool block, struct vcpu *current,
+				    struct vcpu **next);
+struct spci_value api_spci_rx_release(struct vcpu *current, struct vcpu **next);
+struct spci_value api_spci_rxtx_map(ipaddr_t send, ipaddr_t recv,
+				    uint32_t page_count, struct vcpu *current,
+				    struct vcpu **next);
+void api_yield(struct vcpu *current, struct vcpu **next);
+struct spci_value api_spci_version(void);
+struct spci_value api_spci_id_get(const struct vcpu *current);
+struct spci_value api_spci_features(uint32_t function_id);
+struct spci_value api_spci_run(spci_vm_id_t vm_id, spci_vcpu_index_t vcpu_idx,
+			       const struct vcpu *current, struct vcpu **next);
diff --git a/inc/hf/arch/cpu.h b/inc/hf/arch/cpu.h
new file mode 100644
index 0000000..a7fe5a8
--- /dev/null
+++ b/inc/hf/arch/cpu.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hf/arch/types.h"
+
+#include "hf/addr.h"
+#include "hf/vcpu.h"
+
+#include "vmapi/hf/spci.h"
+
+/**
+ * Reset the register values other than the PC and argument which are set with
+ * `arch_regs_set_pc_arg()`.
+ */
+void arch_regs_reset(struct vcpu *vcpu);
+
+/**
+ * Updates the given registers so that when a vCPU runs, it starts off at the
+ * given address (pc) with the given argument.
+ *
+ * This function must only be called on an arch_regs that is known not be in use
+ * by any other physical CPU.
+ */
+void arch_regs_set_pc_arg(struct arch_regs *r, ipaddr_t pc, uintreg_t arg);
+
+/**
+ * Updates the register holding the return value of a function.
+ *
+ * This function must only be called on an arch_regs that is known not be in use
+ * by any other physical CPU.
+ */
+void arch_regs_set_retval(struct arch_regs *r, struct spci_value v);
+
+/**
+ * Initialize and reset CPU-wide register values.
+ */
+void arch_cpu_init(void);
diff --git a/inc/hf/arch/init.h b/inc/hf/arch/init.h
new file mode 100644
index 0000000..0ec80a5
--- /dev/null
+++ b/inc/hf/arch/init.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/**
+ * Performs arch specific boot time initialization.
+ *
+ * It must only be called once, on first boot and must be called as early as
+ * possible.
+ */
+void arch_one_time_init(void);
diff --git a/inc/hf/arch/irq.h b/inc/hf/arch/irq.h
new file mode 100644
index 0000000..c1ee943
--- /dev/null
+++ b/inc/hf/arch/irq.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/**
+ * Disables interrupts.
+ */
+void arch_irq_disable(void);
+
+/**
+ * Enables interrupts.
+ */
+void arch_irq_enable(void);
diff --git a/inc/hf/arch/mm.h b/inc/hf/arch/mm.h
new file mode 100644
index 0000000..e8cfa36
--- /dev/null
+++ b/inc/hf/arch/mm.h
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "hf/addr.h"
+
+/*
+ * A page table entry (PTE) will take one of the following forms:
+ *
+ *  1. absent        : There is no mapping.
+ *  2. invalid block : Represents a block that is not in the address space.
+ *  3. valid block   : Represents a block that is in the address space.
+ *  4. table         : Represents a reference to a table of PTEs.
+ */
+
+/**
+ * Creates an absent PTE.
+ */
+pte_t arch_mm_absent_pte(uint8_t level);
+
+/**
+ * Creates a table PTE.
+ */
+pte_t arch_mm_table_pte(uint8_t level, paddr_t pa);
+
+/**
+ * Creates a block PTE.
+ */
+pte_t arch_mm_block_pte(uint8_t level, paddr_t pa, uint64_t attrs);
+
+/**
+ * Checks whether a block is allowed at the given level of the page table.
+ */
+bool arch_mm_is_block_allowed(uint8_t level);
+
+/**
+ * Determines if a PTE is present i.e. it contains information and therefore
+ * needs to exist in the page table. Any non-absent PTE is present.
+ */
+bool arch_mm_pte_is_present(pte_t pte, uint8_t level);
+
+/**
+ * Determines if a PTE is valid i.e. it can affect the address space. Tables and
+ * valid blocks fall into this category. Invalid blocks do not as they hold
+ * information about blocks that are not in the address space.
+ */
+bool arch_mm_pte_is_valid(pte_t pte, uint8_t level);
+
+/**
+ * Determines if a PTE is a block and represents an address range, valid or
+ * invalid.
+ */
+bool arch_mm_pte_is_block(pte_t pte, uint8_t level);
+
+/**
+ * Determines if a PTE represents a reference to a table of PTEs.
+ */
+bool arch_mm_pte_is_table(pte_t pte, uint8_t level);
+
+/**
+ * Clears the bits of an address that are ignored by the page table. In effect,
+ * the address is rounded down to the start of the corresponding PTE range.
+ */
+paddr_t arch_mm_clear_pa(paddr_t pa);
+
+/**
+ * Extracts the start address of the PTE range.
+ */
+paddr_t arch_mm_block_from_pte(pte_t pte, uint8_t level);
+
+/**
+ * Extracts the address of the table referenced by the PTE.
+ */
+paddr_t arch_mm_table_from_pte(pte_t pte, uint8_t level);
+
+/**
+ * Extracts the attributes of the PTE.
+ */
+uint64_t arch_mm_pte_attrs(pte_t pte, uint8_t level);
+
+/**
+ * Merges the attributes of a block into those of its containing table.
+ */
+uint64_t arch_mm_combine_table_entry_attrs(uint64_t table_attrs,
+					   uint64_t block_attrs);
+
+/**
+ * Invalidates the given range of stage-1 TLB.
+ */
+void arch_mm_invalidate_stage1_range(vaddr_t va_begin, vaddr_t va_end);
+
+/**
+ * Invalidates the given range of stage-2 TLB.
+ */
+void arch_mm_invalidate_stage2_range(ipaddr_t va_begin, ipaddr_t va_end);
+
+/**
+ * Writes back the given range of virtual memory to such a point that all cores
+ * and devices will see the updated values. The corresponding cache lines are
+ * also invalidated.
+ */
+void arch_mm_flush_dcache(void *base, size_t size);
+
+/**
+ * Gets the maximum level allowed in the page table for stage-1.
+ */
+uint8_t arch_mm_stage1_max_level(void);
+
+/**
+ * Gets the maximum level allowed in the page table for stage-2.
+ */
+uint8_t arch_mm_stage2_max_level(void);
+
+/**
+ * Gets the number of concatenated page tables used at the root for stage-1.
+ *
+ * Tables are concatenated at the root to avoid introducing another level in the
+ * page table meaning the table is shallow and wide. Each level is an extra
+ * memory access when walking the table so keeping it shallow reduces the memory
+ * accesses to aid performance.
+ */
+uint8_t arch_mm_stage1_root_table_count(void);
+
+/**
+ * Gets the number of concatenated page tables used at the root for stage-2.
+ */
+uint8_t arch_mm_stage2_root_table_count(void);
+
+/**
+ * Converts the mode into stage-1 attributes for a block PTE.
+ */
+uint64_t arch_mm_mode_to_stage1_attrs(uint32_t mode);
+
+/**
+ * Converts the mode into stage-2 attributes for a block PTE.
+ */
+uint64_t arch_mm_mode_to_stage2_attrs(uint32_t mode);
+
+/**
+ * Converts the stage-2 block attributes back to the corresponding mode.
+ */
+uint32_t arch_mm_stage2_attrs_to_mode(uint64_t attrs);
+
+/**
+ * Initializes the arch specific memory management.
+ */
+bool arch_mm_init(paddr_t table);
diff --git a/inc/hf/arch/plat/smc.h b/inc/hf/arch/plat/smc.h
new file mode 100644
index 0000000..29493fc
--- /dev/null
+++ b/inc/hf/arch/plat/smc.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "vmapi/hf/spci.h"
+
+/**
+ * Called after an SMC has been forwarded. `args` contains the arguments passed
+ * to the SMC and `ret` contains the return values that will be set in the vCPU
+ * registers after this call returns.
+ */
+void plat_smc_post_forward(struct spci_value args, struct spci_value *ret);
diff --git a/inc/hf/arch/std.h b/inc/hf/arch/std.h
new file mode 100644
index 0000000..9fd3ec7
--- /dev/null
+++ b/inc/hf/arch/std.h
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
+
+int memcmp(const void *a, const void *b, size_t n);
+
+int strcmp(const char *a, const char *b);
+
+#define ctz(x) __builtin_ctz(x)
+
+/* Compatibility with old compilers */
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+/**
+ * Check whether the value `v` is aligned to the boundary `a`,
+ * with `a` power of 2.
+ */
+#if __has_builtin(__builtin_is_aligned)
+#define is_aligned(v, a) __builtin_is_aligned((v), (a))
+#else
+#define is_aligned(v, a) (((uintptr_t)(v) & (a - 1)) == 0)
+#endif
+
+/**
+ * Align up the value `v` to the boundary `a`, with `a` power of 2.
+ */
+#if __has_builtin(__builtin_align_up)
+#define align_up(v, a) __builtin_align_up((v), (a))
+#else
+#define align_up(v, a) (((uintptr_t)(v) + (a - 1)) & ~(a - 1))
+#endif
+
+/**
+ * Align down the value `v` to the boundary `a`, with `a` power of 2.
+ */
+#if __has_builtin(__builtin_align_down)
+#define align_down(v, a) __builtin_align_down((v), (a))
+#else
+#define align_down(v, a) ((uintptr_t)(v) & ~(a - 1))
+#endif
+
+#ifndef be16toh
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+
+#define be16toh(v) __builtin_bswap16(v)
+#define be32toh(v) __builtin_bswap32(v)
+#define be64toh(v) __builtin_bswap64(v)
+
+#define htobe16(v) __builtin_bswap16(v)
+#define htobe32(v) __builtin_bswap32(v)
+#define htobe64(v) __builtin_bswap64(v)
+
+#define le16toh(v) (v)
+#define le32toh(v) (v)
+#define le64toh(v) (v)
+
+#define htole16(v) (v)
+#define htole32(v) (v)
+#define htole64(v) (v)
+
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+
+#define be16toh(v) (v)
+#define be32toh(v) (v)
+#define be64toh(v) (v)
+
+#define htobe16(v) (v)
+#define htobe32(v) (v)
+#define htobe64(v) (v)
+
+#define le16toh(v) __builtin_bswap16(v)
+#define le32toh(v) __builtin_bswap32(v)
+#define le64toh(v) __builtin_bswap64(v)
+
+#define htole16(v) __builtin_bswap16(v)
+#define htole32(v) __builtin_bswap32(v)
+#define htole64(v) __builtin_bswap64(v)
+
+#else
+
+/*
+ * __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ &&
+ * __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
+ */
+
+#error "Unsupported byte order"
+
+#endif
+#endif
diff --git a/inc/hf/arch/timer.h b/inc/hf/arch/timer.h
new file mode 100644
index 0000000..cd2a433
--- /dev/null
+++ b/inc/hf/arch/timer.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "hf/arch/types.h"
+
+/**
+ * Sets the bit to mask virtual timer interrupts.
+ */
+void arch_timer_mask(struct arch_regs *regs);
+
+/**
+ * Checks whether the virtual timer is enabled and its interrupt not masked.
+ */
+bool arch_timer_enabled(struct arch_regs *regs);
+
+/**
+ * Returns the number of nanoseconds remaining on the virtual timer as stored in
+ * the given `arch_regs`, or 0 if it has already expired. This is undefined if
+ * the timer is not enabled.
+ */
+uint64_t arch_timer_remaining_ns(struct arch_regs *regs);
+
+/**
+ * Returns whether the timer is ready to fire: i.e. it is enabled, not masked,
+ * and the condition is met.
+ */
+bool arch_timer_pending(struct arch_regs *regs);
+
+/**
+ * Checks whether the virtual timer is enabled and its interrupt not masked, for
+ * the currently active vCPU.
+ */
+bool arch_timer_enabled_current(void);
+
+/**
+ * Disable the virtual timer for the currently active vCPU.
+ */
+void arch_timer_disable_current(void);
+
+/**
+ * Returns the number of nanoseconds remaining on the virtual timer of the
+ * currently active vCPU, or 0 if it has already expired. This is undefined if
+ * the timer is not enabled.
+ */
+uint64_t arch_timer_remaining_ns_current(void);
diff --git a/inc/hf/arch/vm.h b/inc/hf/arch/vm.h
new file mode 100644
index 0000000..5612388
--- /dev/null
+++ b/inc/hf/arch/vm.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/vm.h"
+
+/**
+ * Set architecture-specific features for the specified VM.
+ */
+void arch_vm_features_set(struct vm *vm);
diff --git a/inc/hf/boot_flow.h b/inc/hf/boot_flow.h
new file mode 100644
index 0000000..39d7060
--- /dev/null
+++ b/inc/hf/boot_flow.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/boot_params.h"
+#include "hf/manifest.h"
+#include "hf/memiter.h"
+#include "hf/mm.h"
+
+bool boot_flow_init(const struct fdt_node *fdt_root, struct manifest *manifest,
+		    struct boot_params *boot_params);
+
+bool boot_flow_update(struct mm_stage1_locked stage1_locked,
+		      const struct manifest *manifest,
+		      struct boot_params_update *p, struct memiter *cpio,
+		      struct mpool *ppool);
diff --git a/inc/hf/boot_params.h b/inc/hf/boot_params.h
new file mode 100644
index 0000000..a55ee73
--- /dev/null
+++ b/inc/hf/boot_params.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+
+#include "hf/arch/cpu.h"
+
+#include "hf/fdt.h"
+#include "hf/mm.h"
+#include "hf/mpool.h"
+
+#define MAX_MEM_RANGES 20
+
+struct mem_range {
+	paddr_t begin;
+	paddr_t end;
+};
+
+struct boot_params {
+	cpu_id_t cpu_ids[MAX_CPUS];
+	size_t cpu_count;
+	struct mem_range mem_ranges[MAX_MEM_RANGES];
+	size_t mem_ranges_count;
+	paddr_t initrd_begin;
+	paddr_t initrd_end;
+	uintreg_t kernel_arg;
+};
+
+struct boot_params_update {
+	struct mem_range reserved_ranges[MAX_MEM_RANGES];
+	size_t reserved_ranges_count;
+	paddr_t initrd_begin;
+	paddr_t initrd_end;
+};
diff --git a/inc/hf/check.h b/inc/hf/check.h
new file mode 100644
index 0000000..41e005f
--- /dev/null
+++ b/inc/hf/check.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/panic.h"
+
+/**
+ * Only use to check assumptions which, if false, mean the system is in a bad
+ * state and it is unsafe to continue.
+ *
+ * Do not use if the condition could ever be legitimately false e.g. when
+ * processing external inputs.
+ */
+#define CHECK(x)                                                              \
+	do {                                                                  \
+		if (!(x)) {                                                   \
+			panic("assertion failed (%s) at %s:%d", #x, __FILE__, \
+			      __LINE__);                                      \
+		}                                                             \
+	} while (0)
diff --git a/inc/hf/cpio.h b/inc/hf/cpio.h
new file mode 100644
index 0000000..aebe1d5
--- /dev/null
+++ b/inc/hf/cpio.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+
+#include "hf/memiter.h"
+#include "hf/string.h"
+
+bool cpio_get_file(const struct memiter *cpio, const struct string *name,
+		   struct memiter *it);
diff --git a/inc/hf/cpu.h b/inc/hf/cpu.h
new file mode 100644
index 0000000..af7391f
--- /dev/null
+++ b/inc/hf/cpu.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/arch/cpu.h"
+
+/* TODO: Fix alignment such that `cpu` structs are in different cache lines. */
+struct cpu {
+	/** CPU identifier. Doesn't have to be contiguous. */
+	cpu_id_t id;
+
+	/** Pointer to bottom of the stack. */
+	void *stack_bottom;
+
+	/** See api.c for the partial ordering on locks. */
+	struct spinlock lock;
+
+	/** Determines whether the CPU is currently on. */
+	bool is_on;
+};
+
+void cpu_module_init(const cpu_id_t *cpu_ids, size_t count);
+
+size_t cpu_index(struct cpu *c);
+bool cpu_on(struct cpu *c, ipaddr_t entry, uintreg_t arg);
+void cpu_off(struct cpu *c);
+struct cpu *cpu_find(cpu_id_t id);
+uint8_t *cpu_get_buffer(cpu_id_t cpu_id);
+uint32_t cpu_get_buffer_size(cpu_id_t cpu_id);
diff --git a/inc/hf/dlog.h b/inc/hf/dlog.h
new file mode 100644
index 0000000..f8c0080
--- /dev/null
+++ b/inc/hf/dlog.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdarg.h>
+#include <stddef.h>
+
+#include "hf/spci.h"
+
+#define DLOG_BUFFER_SIZE 8192
+
+extern size_t dlog_buffer_offset;
+extern char dlog_buffer[];
+
+#if DEBUG
+void dlog_enable_lock(void);
+void dlog(const char *fmt, ...);
+void vdlog(const char *fmt, va_list args);
+#else
+#define dlog_enable_lock()
+#define dlog(...)
+#define vdlog(fmt, args)
+#endif
+
+void dlog_flush_vm_buffer(spci_vm_id_t id, char buffer[], size_t length);
diff --git a/inc/hf/fdt.h b/inc/hf/fdt.h
new file mode 100644
index 0000000..8f72856
--- /dev/null
+++ b/inc/hf/fdt.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct fdt_node {
+	const struct fdt_header *hdr;
+	const char *begin;
+	const char *end;
+	const char *strs;
+};
+
+size_t fdt_header_size(void);
+uint32_t fdt_total_size(struct fdt_header *hdr);
+void fdt_dump(const struct fdt_header *hdr);
+bool fdt_root_node(struct fdt_node *node, const struct fdt_header *hdr);
+bool fdt_find_child(struct fdt_node *node, const char *child);
+bool fdt_first_child(struct fdt_node *node, const char **child_name);
+bool fdt_next_sibling(struct fdt_node *node, const char **sibling_name);
+bool fdt_read_property(const struct fdt_node *node, const char *name,
+		       const char **buf, uint32_t *size);
+bool fdt_parse_number(const char *data, uint32_t size, uint64_t *value);
+
+void fdt_add_mem_reservation(struct fdt_header *hdr, uint64_t addr,
+			     uint64_t len);
diff --git a/inc/hf/fdt_handler.h b/inc/hf/fdt_handler.h
new file mode 100644
index 0000000..70521b5
--- /dev/null
+++ b/inc/hf/fdt_handler.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/boot_params.h"
+#include "hf/fdt.h"
+#include "hf/mm.h"
+#include "hf/mpool.h"
+
+struct fdt_header *fdt_map(struct mm_stage1_locked stage1_locked,
+			   paddr_t fdt_addr, struct fdt_node *n,
+			   struct mpool *ppool);
+bool fdt_unmap(struct mm_stage1_locked stage1_locked, struct fdt_header *fdt,
+	       struct mpool *ppool);
+bool fdt_find_cpus(const struct fdt_node *root, cpu_id_t *cpu_ids,
+		   size_t *cpu_count);
+bool fdt_find_memory_ranges(const struct fdt_node *root, struct boot_params *p);
+bool fdt_find_initrd(const struct fdt_node *root, paddr_t *begin, paddr_t *end);
+
+/** Apply an update to the FDT. */
+bool fdt_patch(struct mm_stage1_locked stage1_locked, paddr_t fdt_addr,
+	       struct boot_params_update *p, struct mpool *ppool);
diff --git a/inc/hf/io.h b/inc/hf/io.h
new file mode 100644
index 0000000..f4a80a4
--- /dev/null
+++ b/inc/hf/io.h
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hf/arch/barriers.h"
+
+#include "hf/check.h"
+
+/* Opaque types for different sized fields of memory mapped IO. */
+
+typedef struct {
+	volatile uint8_t *ptr;
+} io8_t;
+
+typedef struct {
+	volatile uint16_t *ptr;
+} io16_t;
+
+typedef struct {
+	volatile uint32_t *ptr;
+} io32_t;
+
+typedef struct {
+	volatile uint64_t *ptr;
+} io64_t;
+
+typedef struct {
+	volatile uint8_t *base;
+	size_t count;
+} io8_array_t;
+
+typedef struct {
+	volatile uint16_t *base;
+	size_t count;
+} io16_array_t;
+
+typedef struct {
+	volatile uint32_t *base;
+	size_t count;
+} io32_array_t;
+
+typedef struct {
+	volatile uint64_t *base;
+	size_t count;
+} io64_array_t;
+
+/* Contructors for literals. */
+
+#define IO8_C(addr) ((io8_t){.ptr = (volatile uint8_t *)(addr)})
+#define IO16_C(addr) ((io16_t){.ptr = (volatile uint16_t *)(addr)})
+#define IO32_C(addr) ((io32_t){.ptr = (volatile uint32_t *)(addr)})
+#define IO64_C(addr) ((io64_t){.ptr = (volatile uint64_t *)(addr)})
+
+#define IO8_ARRAY_C(addr, cnt) \
+	((io8_array_t){.base = (volatile uint8_t *)(addr), .count = (cnt)})
+#define IO16_ARRAY_C(addr, cnt) \
+	((io16_array_t){.base = (volatile uint16_t *)(addr), .count = (cnt)})
+#define IO32_ARRAY_C(addr, cnt) \
+	((io32_array_t){.base = (volatile uint32_t *)(addr), .count = (cnt)})
+#define IO64_ARRAY_C(addr, cnt) \
+	((io64_array_t){.base = (volatile uint64_t *)(addr), .count = (cnt)})
+
+/** Read from memory-mapped IO. */
+
+static inline uint8_t io_read8(io8_t io)
+{
+	return *io.ptr;
+}
+
+static inline uint16_t io_read16(io16_t io)
+{
+	return *io.ptr;
+}
+
+static inline uint32_t io_read32(io32_t io)
+{
+	return *io.ptr;
+}
+
+static inline uint64_t io_read64(io64_t io)
+{
+	return *io.ptr;
+}
+
+static inline uint8_t io_read8_array(io8_array_t io, size_t n)
+{
+	CHECK(n < io.count);
+	return io.base[n];
+}
+
+static inline uint16_t io_read16_array(io16_array_t io, size_t n)
+{
+	CHECK(n < io.count);
+	return io.base[n];
+}
+
+static inline uint32_t io_read32_array(io32_array_t io, size_t n)
+{
+	CHECK(n < io.count);
+	return io.base[n];
+}
+
+static inline uint64_t io_read64_array(io64_array_t io, size_t n)
+{
+	CHECK(n < io.count);
+	return io.base[n];
+}
+
+/**
+ * Read from memory-mapped IO with memory barrier.
+ *
+ * The read is ordered before subsequent memory accesses.
+ */
+
+static inline uint8_t io_read8_mb(io8_t io)
+{
+	uint8_t v = io_read8(io);
+
+	data_sync_barrier();
+	return v;
+}
+
+static inline uint16_t io_read16_mb(io16_t io)
+{
+	uint16_t v = io_read16(io);
+
+	data_sync_barrier();
+	return v;
+}
+
+static inline uint32_t io_read32_mb(io32_t io)
+{
+	uint32_t v = io_read32(io);
+
+	data_sync_barrier();
+	return v;
+}
+
+static inline uint64_t io_read64_mb(io64_t io)
+{
+	uint64_t v = io_read64(io);
+
+	data_sync_barrier();
+	return v;
+}
+
+static inline uint8_t io_read8_array_mb(io8_array_t io, size_t n)
+{
+	uint8_t v = io_read8_array(io, n);
+
+	data_sync_barrier();
+	return v;
+}
+
+static inline uint16_t io_read16_array_mb(io16_array_t io, size_t n)
+{
+	uint16_t v = io_read16_array(io, n);
+
+	data_sync_barrier();
+	return v;
+}
+
+static inline uint32_t io_read32_array_mb(io32_array_t io, size_t n)
+{
+	uint32_t v = io_read32_array(io, n);
+
+	data_sync_barrier();
+	return v;
+}
+
+static inline uint64_t io_read64_array_mb(io64_array_t io, size_t n)
+{
+	uint64_t v = io_read64_array(io, n);
+
+	data_sync_barrier();
+	return v;
+}
+
+/* Write to memory-mapped IO. */
+
+static inline void io_write8(io8_t io, uint8_t v)
+{
+	*io.ptr = v;
+}
+
+static inline void io_write16(io16_t io, uint16_t v)
+{
+	*io.ptr = v;
+}
+
+static inline void io_write32(io32_t io, uint32_t v)
+{
+	*io.ptr = v;
+}
+
+static inline void io_write64(io64_t io, uint64_t v)
+{
+	*io.ptr = v;
+}
+
+static inline void io_write8_array(io8_array_t io, size_t n, uint8_t v)
+{
+	CHECK(n < io.count);
+	io.base[n] = v;
+}
+
+static inline void io_write16_array(io16_array_t io, size_t n, uint16_t v)
+{
+	CHECK(n < io.count);
+	io.base[n] = v;
+}
+
+static inline void io_write32_array(io32_array_t io, size_t n, uint32_t v)
+{
+	CHECK(n < io.count);
+	io.base[n] = v;
+}
+
+static inline void io_write64_array(io64_array_t io, size_t n, uint64_t v)
+{
+	CHECK(n < io.count);
+	io.base[n] = v;
+}
+
+/*
+ * Write to memory-mapped IO with memory barrier.
+ *
+ * The write is ordered after previous memory accesses.
+ */
+
+static inline void io_write8_mb(io8_t io, uint8_t v)
+{
+	data_sync_barrier();
+	io_write8(io, v);
+}
+
+static inline void io_write16_mb(io16_t io, uint16_t v)
+{
+	data_sync_barrier();
+	io_write16(io, v);
+}
+
+static inline void io_write32_mb(io32_t io, uint32_t v)
+{
+	data_sync_barrier();
+	io_write32(io, v);
+}
+
+static inline void io_write64_mb(io64_t io, uint64_t v)
+{
+	data_sync_barrier();
+	io_write64(io, v);
+}
+
+static inline void io_write8_array_mb(io8_array_t io, size_t n, uint8_t v)
+{
+	data_sync_barrier();
+	io_write8_array(io, n, v);
+}
+
+static inline void io_write16_array_mb(io16_array_t io, size_t n, uint16_t v)
+{
+	data_sync_barrier();
+	io_write16_array(io, n, v);
+}
+
+static inline void io_write32_array_mb(io32_array_t io, size_t n, uint32_t v)
+{
+	data_sync_barrier();
+	io_write32_array(io, n, v);
+}
+
+static inline void io_write64_array_mb(io64_array_t io, size_t n, uint64_t v)
+{
+	data_sync_barrier();
+	io_write64_array(io, n, v);
+}
diff --git a/inc/hf/layout.h b/inc/hf/layout.h
new file mode 100644
index 0000000..4193abf
--- /dev/null
+++ b/inc/hf/layout.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/addr.h"
+
+paddr_t layout_text_begin(void);
+paddr_t layout_text_end(void);
+
+paddr_t layout_rodata_begin(void);
+paddr_t layout_rodata_end(void);
+
+paddr_t layout_data_begin(void);
+paddr_t layout_data_end(void);
+
+paddr_t layout_initrd_begin(void);
+paddr_t layout_initrd_end(void);
+
+paddr_t layout_fdt_begin(void);
+paddr_t layout_fdt_end(void);
+
+paddr_t layout_image_end(void);
+
+paddr_t layout_primary_begin(void);
diff --git a/inc/hf/list.h b/inc/hf/list.h
new file mode 100644
index 0000000..30fefa9
--- /dev/null
+++ b/inc/hf/list.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+
+struct list_entry {
+	struct list_entry *next;
+	struct list_entry *prev;
+};
+
+#define LIST_INIT(l)                   \
+	{                              \
+		.next = &l, .prev = &l \
+	}
+#define CONTAINER_OF(ptr, type, field) \
+	((type *)((char *)ptr - offsetof(type, field)))
+
+static inline void list_init(struct list_entry *e)
+{
+	e->next = e;
+	e->prev = e;
+}
+
+static inline void list_append(struct list_entry *l, struct list_entry *e)
+{
+	e->next = l;
+	e->prev = l->prev;
+
+	e->next->prev = e;
+	e->prev->next = e;
+}
+
+static inline void list_prepend(struct list_entry *l, struct list_entry *e)
+{
+	e->next = l->next;
+	e->prev = l;
+
+	e->next->prev = e;
+	e->prev->next = e;
+}
+
+static inline bool list_empty(struct list_entry *l)
+{
+	return l->next == l;
+}
+
+static inline void list_remove(struct list_entry *e)
+{
+	e->prev->next = e->next;
+	e->next->prev = e->prev;
+	list_init(e);
+}
diff --git a/inc/hf/load.h b/inc/hf/load.h
new file mode 100644
index 0000000..b76dc1d
--- /dev/null
+++ b/inc/hf/load.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hf/boot_params.h"
+#include "hf/cpio.h"
+#include "hf/manifest.h"
+#include "hf/memiter.h"
+#include "hf/mm.h"
+#include "hf/mpool.h"
+
+bool load_vms(struct mm_stage1_locked stage1_locked,
+	      const struct manifest *manifest, const struct memiter *cpio,
+	      const struct boot_params *params,
+	      struct boot_params_update *update, struct mpool *ppool);
diff --git a/inc/hf/manifest.h b/inc/hf/manifest.h
new file mode 100644
index 0000000..0e628e5
--- /dev/null
+++ b/inc/hf/manifest.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/fdt.h"
+#include "hf/memiter.h"
+#include "hf/spci.h"
+#include "hf/string.h"
+#include "hf/vm.h"
+
+/**
+ * Holds information about one of the VMs described in the manifest.
+ */
+struct manifest_vm {
+	/* Properties defined for both primary and secondary VMs. */
+	struct string debug_name;
+	struct string kernel_filename;
+	struct smc_whitelist smc_whitelist;
+
+	union {
+		/* Properties specific to the primary VM. */
+		struct {
+			struct string ramdisk_filename;
+		} primary;
+		/* Properties specific to secondary VMs. */
+		struct {
+			uint64_t mem_size;
+			spci_vcpu_count_t vcpu_count;
+		} secondary;
+	};
+};
+
+/**
+ * Hafnium manifest parsed from FDT.
+ */
+struct manifest {
+	spci_vm_count_t vm_count;
+	struct manifest_vm vm[MAX_VMS];
+};
+
+enum manifest_return_code {
+	MANIFEST_SUCCESS = 0,
+	MANIFEST_ERROR_NO_HYPERVISOR_FDT_NODE,
+	MANIFEST_ERROR_NOT_COMPATIBLE,
+	MANIFEST_ERROR_RESERVED_VM_ID,
+	MANIFEST_ERROR_NO_PRIMARY_VM,
+	MANIFEST_ERROR_TOO_MANY_VMS,
+	MANIFEST_ERROR_PROPERTY_NOT_FOUND,
+	MANIFEST_ERROR_MALFORMED_STRING,
+	MANIFEST_ERROR_STRING_TOO_LONG,
+	MANIFEST_ERROR_MALFORMED_STRING_LIST,
+	MANIFEST_ERROR_MALFORMED_INTEGER,
+	MANIFEST_ERROR_INTEGER_OVERFLOW,
+	MANIFEST_ERROR_MALFORMED_INTEGER_LIST,
+	MANIFEST_ERROR_MALFORMED_BOOLEAN,
+};
+
+enum manifest_return_code manifest_init(struct manifest *manifest,
+					const struct fdt_node *fdt_root);
+
+const char *manifest_strerror(enum manifest_return_code ret_code);
diff --git a/inc/hf/memiter.h b/inc/hf/memiter.h
new file mode 100644
index 0000000..e3f0db8
--- /dev/null
+++ b/inc/hf/memiter.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct memiter {
+	const char *next;
+	const char *limit;
+};
+
+void memiter_init(struct memiter *it, const void *data, size_t size);
+bool memiter_parse_uint(struct memiter *it, uint64_t *value);
+bool memiter_parse_str(struct memiter *it, struct memiter *str);
+bool memiter_iseq(const struct memiter *it, const char *str);
+bool memiter_advance(struct memiter *it, size_t v);
+
+const void *memiter_base(const struct memiter *it);
+size_t memiter_size(const struct memiter *it);
diff --git a/inc/hf/mm.h b/inc/hf/mm.h
new file mode 100644
index 0000000..43d4c79
--- /dev/null
+++ b/inc/hf/mm.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdalign.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "hf/arch/mm.h"
+
+#include "hf/addr.h"
+#include "hf/mpool.h"
+#include "hf/static_assert.h"
+
+/* Keep macro alignment */
+/* clang-format off */
+
+#define PAGE_SIZE (1 << PAGE_BITS)
+#define MM_PTE_PER_PAGE (PAGE_SIZE / sizeof(pte_t))
+
+/* The following are arch-independent page mapping modes. */
+#define MM_MODE_R UINT32_C(0x0001) /* read */
+#define MM_MODE_W UINT32_C(0x0002) /* write */
+#define MM_MODE_X UINT32_C(0x0004) /* execute */
+#define MM_MODE_D UINT32_C(0x0008) /* device */
+
+/*
+ * Memory in stage-1 is either valid (present) or invalid (absent).
+ *
+ * Memory in stage-2 has more states to track sharing, borrowing and giving of
+ * memory. The states are made up of three parts:
+ *
+ *  1. V = valid/invalid    : Whether the memory is part of the VM's address
+ *                            space. A fault will be generated if accessed when
+ *                            invalid.
+ *  2. O = owned/unowned    : Whether the memory is owned by the VM.
+ *  3. X = exclusive/shared : Whether access is exclusive to the VM or shared
+ *                            with at most one other.
+ *
+ * These parts compose to form the following state:
+ *
+ *  -  V  O  X : Owner of memory with exclusive access.
+ *  -  V  O !X : Owner of memory with access shared with at most one other VM.
+ *  -  V !O  X : Borrower of memory with exclusive access.
+ *  -  V !O !X : Borrower of memory where access is shared with the owner.
+ *  - !V  O  X : Owner of memory lent to a VM that has exclusive access.
+ *
+ *  - !V  O !X : Unused. Owner of shared memory always has access.
+ *  - !V !O  X : Unused. Next entry is used for invalid memory.
+ *
+ *  - !V !O !X : Invalid memory. Memory is unrelated to the VM.
+ *
+ *  Modes are selected so that owner of exclusive memory is the default.
+ */
+#define MM_MODE_INVALID UINT32_C(0x0010)
+#define MM_MODE_UNOWNED UINT32_C(0x0020)
+#define MM_MODE_SHARED  UINT32_C(0x0040)
+
+/* The mask for a mode that is considered unmapped. */
+#define MM_MODE_UNMAPPED_MASK (MM_MODE_INVALID | MM_MODE_UNOWNED)
+
+#define MM_FLAG_COMMIT  0x01
+#define MM_FLAG_UNMAP   0x02
+#define MM_FLAG_STAGE1  0x04
+
+/* clang-format on */
+
+struct mm_page_table {
+	alignas(PAGE_SIZE) pte_t entries[MM_PTE_PER_PAGE];
+};
+static_assert(sizeof(struct mm_page_table) == PAGE_SIZE,
+	      "A page table must take exactly one page.");
+static_assert(alignof(struct mm_page_table) == PAGE_SIZE,
+	      "A page table must be page aligned.");
+
+struct mm_ptable {
+	/** Address of the root of the page table. */
+	paddr_t root;
+};
+
+/** The type of addresses stored in the page table. */
+typedef uintvaddr_t ptable_addr_t;
+
+/** Represents the currently locked stage-1 page table of the hypervisor. */
+struct mm_stage1_locked {
+	struct mm_ptable *ptable;
+};
+
+void mm_vm_enable_invalidation(void);
+
+bool mm_ptable_init(struct mm_ptable *t, int flags, struct mpool *ppool);
+ptable_addr_t mm_ptable_addr_space_end(int flags);
+
+bool mm_vm_init(struct mm_ptable *t, struct mpool *ppool);
+void mm_vm_fini(struct mm_ptable *t, struct mpool *ppool);
+bool mm_vm_identity_map(struct mm_ptable *t, paddr_t begin, paddr_t end,
+			uint32_t mode, struct mpool *ppool, ipaddr_t *ipa);
+bool mm_vm_identity_prepare(struct mm_ptable *t, paddr_t begin, paddr_t end,
+			    uint32_t mode, struct mpool *ppool);
+void mm_vm_identity_commit(struct mm_ptable *t, paddr_t begin, paddr_t end,
+			   uint32_t mode, struct mpool *ppool, ipaddr_t *ipa);
+bool mm_vm_unmap(struct mm_ptable *t, paddr_t begin, paddr_t end,
+		 struct mpool *ppool);
+void mm_vm_defrag(struct mm_ptable *t, struct mpool *ppool);
+void mm_vm_dump(struct mm_ptable *t);
+bool mm_vm_get_mode(struct mm_ptable *t, ipaddr_t begin, ipaddr_t end,
+		    uint32_t *mode);
+
+struct mm_stage1_locked mm_lock_stage1(void);
+void mm_unlock_stage1(struct mm_stage1_locked *lock);
+void *mm_identity_map(struct mm_stage1_locked stage1_locked, paddr_t begin,
+		      paddr_t end, uint32_t mode, struct mpool *ppool);
+bool mm_unmap(struct mm_stage1_locked stage1_locked, paddr_t begin, paddr_t end,
+	      struct mpool *ppool);
+void mm_defrag(struct mm_stage1_locked stage1_locked, struct mpool *ppool);
+
+bool mm_init(struct mpool *ppool);
diff --git a/inc/hf/mpool.h b/inc/hf/mpool.h
new file mode 100644
index 0000000..c6d6150
--- /dev/null
+++ b/inc/hf/mpool.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "hf/spinlock.h"
+
+struct mpool {
+	struct spinlock lock;
+	size_t entry_size;
+	struct mpool_chunk *chunk_list;
+	struct mpool_entry *entry_list;
+	struct mpool *fallback;
+};
+
+void mpool_enable_locks(void);
+void mpool_init(struct mpool *p, size_t entry_size);
+void mpool_init_from(struct mpool *p, struct mpool *from);
+void mpool_init_with_fallback(struct mpool *p, struct mpool *fallback);
+void mpool_fini(struct mpool *p);
+bool mpool_add_chunk(struct mpool *p, void *begin, size_t size);
+void *mpool_alloc(struct mpool *p);
+void *mpool_alloc_contiguous(struct mpool *p, size_t count, size_t align);
+void mpool_free(struct mpool *p, void *ptr);
diff --git a/inc/hf/offset_size_header.h b/inc/hf/offset_size_header.h
new file mode 100644
index 0000000..7d96950
--- /dev/null
+++ b/inc/hf/offset_size_header.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This header file is intended for use by files compiled with the
+ * 'offset_size_header' build rule. See overview in 'offset_size_header.gni'.
+ */
+
+#pragma once
+
+#if (defined GENERATE_BINARY) && (defined VERIFY_HEADER)
+#error Only one action can be specified
+
+#elif defined GENERATE_BINARY
+
+/**
+ * File being compiled to generate binaries with definitions of constants.
+ */
+
+/**
+ * Emit a function with an embedded string in the format:
+ *     <HAFNIUM_DEFINE name #value />
+ * These will be recognized by a script that generates the header file.
+ */
+#define DEFINE(sym, val)                                    \
+	void gen_header__##sym(void)                        \
+	{                                                   \
+		__asm__ volatile(                           \
+			"\n"                                \
+			".ascii \"\\n<HAFNIUM_DEFINE " #sym \
+			" %0 />\\n\"\n"                     \
+			".align 8\n" /* Align epilogue */   \
+			:                                   \
+			: "i"(val));                        \
+	}
+
+#define DEFINE_SIZEOF(sym, type) DEFINE(sym, sizeof(type))
+#define DEFINE_OFFSETOF(sym, type, field) DEFINE(sym, offsetof(type, field))
+
+#elif defined VERIFY_HEADER
+
+/**
+ * File being compiled as part of the main build to check the values in
+ * the auto-generated header file (included using a command-line flag).
+ */
+
+#include "hf/static_assert.h"
+
+#define DEFINE_SIZEOF(sym, type)                                 \
+	void gen_header__##sym(void)                             \
+	{                                                        \
+		static_assert(sizeof(type) == sym,               \
+			      "Generated struct size mismatch"); \
+	}
+
+#define DEFINE_OFFSETOF(sym, type, field)                          \
+	void gen_header__##sym(void)                               \
+	{                                                          \
+		static_assert(offsetof(type, field) == sym,        \
+			      "Generated struct offset mismatch"); \
+	}
+
+#else
+#error No action specified
+#endif
diff --git a/inc/hf/panic.h b/inc/hf/panic.h
new file mode 100644
index 0000000..89893bc
--- /dev/null
+++ b/inc/hf/panic.h
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdnoreturn.h>
+
+noreturn void panic(const char *fmt, ...);
diff --git a/inc/hf/plat/boot_flow.h b/inc/hf/plat/boot_flow.h
new file mode 100644
index 0000000..b5d8145
--- /dev/null
+++ b/inc/hf/plat/boot_flow.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/addr.h"
+#include "hf/boot_params.h"
+#include "hf/fdt.h"
+#include "hf/manifest.h"
+#include "hf/memiter.h"
+#include "hf/mm.h"
+
+paddr_t plat_boot_flow_get_fdt_addr(void);
+uintreg_t plat_boot_flow_get_kernel_arg(void);
+bool plat_boot_flow_get_initrd_range(const struct fdt_node *fdt_root,
+				     paddr_t *begin, paddr_t *end);
+bool plat_boot_flow_update(struct mm_stage1_locked stage1_locked,
+			   const struct manifest *manifest,
+			   struct boot_params_update *p, struct memiter *cpio,
+			   struct mpool *ppool);
diff --git a/inc/hf/plat/console.h b/inc/hf/plat/console.h
new file mode 100644
index 0000000..8fb4969
--- /dev/null
+++ b/inc/hf/plat/console.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/mm.h"
+#include "hf/mpool.h"
+
+/** Initialises the console hardware. */
+void plat_console_init(void);
+
+/** Initialises any memory mappings that the console driver needs. */
+void plat_console_mm_init(struct mm_stage1_locked stage1_locked,
+			  struct mpool *ppool);
+
+/** Puts a single character on the console. */
+void plat_console_putchar(char c);
diff --git a/inc/hf/plat/iommu.h b/inc/hf/plat/iommu.h
new file mode 100644
index 0000000..9ef791d
--- /dev/null
+++ b/inc/hf/plat/iommu.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/addr.h"
+#include "hf/fdt.h"
+#include "hf/vm.h"
+
+/**
+ * Initializes the platform IOMMU driver. The root node of the FDT is provided
+ * so that the driver can read from it. This can be used to map IOMMU devices
+ * into the hypervisor's address space so they are accessible by the driver.
+ */
+bool plat_iommu_init(const struct fdt_node *fdt_root,
+		     struct mm_stage1_locked stage1_locked,
+		     struct mpool *ppool);
+
+/**
+ * Maps the address range with the given mode for the given VM in the IOMMU.
+ *
+ * Assumes the identity map cannot fail. This may not always be true and if it
+ * isn't it will require careful thought on how to safely handle error cases
+ * when intermingled with MMU updates but it gives a starting point for drivers
+ * until those problems are understood.
+ *
+ * The modes are the same as the memory management modes but it is only required
+ * that read and write modes are enforced by the IOMMU driver.
+ */
+void plat_iommu_identity_map(struct vm_locked vm_locked, paddr_t begin,
+			     paddr_t end, uint32_t mode);
diff --git a/inc/hf/spci_internal.h b/inc/hf/spci_internal.h
new file mode 100644
index 0000000..1fc226f
--- /dev/null
+++ b/inc/hf/spci_internal.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/addr.h"
+#include "hf/vm.h"
+
+#include "vmapi/hf/spci.h"
+
+#define SPCI_VERSION_MAJOR 0x0
+#define SPCI_VERSION_MINOR 0x9
+
+#define SPCI_VERSION_MAJOR_OFFSET 16
+
+struct spci_mem_transitions {
+	uint32_t orig_from_mode;
+	uint32_t orig_to_mode;
+	uint32_t from_mode;
+	uint32_t to_mode;
+};
+
+/* TODO: Add device attributes: GRE, cacheability, shareability. */
+static inline uint32_t spci_memory_attrs_to_mode(uint16_t memory_attributes)
+{
+	uint32_t mode = 0;
+
+	switch (spci_get_memory_access_attr(memory_attributes)) {
+	case SPCI_MEMORY_RO_NX:
+		mode = MM_MODE_R;
+		break;
+	case SPCI_MEMORY_RO_X:
+		mode = MM_MODE_R | MM_MODE_X;
+		break;
+	case SPCI_MEMORY_RW_NX:
+		mode = MM_MODE_R | MM_MODE_W;
+		break;
+	case SPCI_MEMORY_RW_X:
+		mode = MM_MODE_R | MM_MODE_W | MM_MODE_X;
+		break;
+	}
+
+	return mode;
+}
+
+static inline struct spci_value spci_error(uint64_t error_code)
+{
+	return (struct spci_value){.func = SPCI_ERROR_32, .arg2 = error_code};
+}
+
+struct spci_value spci_msg_handle_architected_message(
+	struct vm_locked to_locked, struct vm_locked from_locked,
+	struct spci_memory_region *memory_region, uint32_t size,
+	uint32_t attributes, struct mpool *api_page_pool);
diff --git a/inc/hf/spinlock.h b/inc/hf/spinlock.h
new file mode 100644
index 0000000..2689a5c
--- /dev/null
+++ b/inc/hf/spinlock.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/*
+ * Includes the arch-specific definition of 'struct spinlock' and
+ * implementations of:
+ *  - SPINLOCK_INIT
+ *  - sl_lock()
+ *  - sl_unlock()
+ */
+#include "hf/arch/spinlock.h"
+
+static inline void sl_init(struct spinlock *l)
+{
+	*l = SPINLOCK_INIT;
+}
+
+/**
+ * Locks both locks, enforcing the lowest address first ordering for locks of
+ * the same kind.
+ */
+static inline void sl_lock_both(struct spinlock *a, struct spinlock *b)
+{
+	if (a < b) {
+		sl_lock(a);
+		sl_lock(b);
+	} else {
+		sl_lock(b);
+		sl_lock(a);
+	}
+}
diff --git a/inc/hf/static_assert.h b/inc/hf/static_assert.h
new file mode 100644
index 0000000..da71ac6
--- /dev/null
+++ b/inc/hf/static_assert.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#if !defined(__cplusplus)
+
+#define static_assert _Static_assert
+
+#endif
diff --git a/inc/hf/std.h b/inc/hf/std.h
new file mode 100644
index 0000000..9c6b494
--- /dev/null
+++ b/inc/hf/std.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/arch/std.h"
+
+typedef size_t rsize_t;
+
+/**
+ * Restrict the maximum range for range checked functions so as to be more
+ * likely to catch errors. This may need to be relaxed if it proves to be overly
+ * restrictive.
+ */
+#define RSIZE_MAX (128 * 1024 * 1024)
+
+/*
+ * Only the safer versions of these functions are exposed to reduce the chance
+ * of misusing the versions without bounds checking or null pointer checks.
+ *
+ * These functions don't return errno_t as per the specification and implicitly
+ * have a constraint handler that panics.
+ */
+void memset_s(void *dest, rsize_t destsz, int ch, rsize_t count);
+void memcpy_s(void *dest, rsize_t destsz, const void *src, rsize_t count);
+void memmove_s(void *dest, rsize_t destsz, const void *src, rsize_t count);
+
+void *memchr(const void *ptr, int ch, size_t count);
+
+size_t strnlen_s(const char *str, size_t strsz);
diff --git a/inc/hf/string.h b/inc/hf/string.h
new file mode 100644
index 0000000..9c64632
--- /dev/null
+++ b/inc/hf/string.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+
+/**
+ * Maximum length of a string including the NULL terminator.
+ * This is an arbitrary number and can be adjusted to fit use cases.
+ */
+#define STRING_MAX_SIZE 32
+
+enum string_return_code {
+	STRING_SUCCESS,
+	STRING_ERROR_INVALID_INPUT,
+	STRING_ERROR_TOO_LONG,
+};
+
+/**
+ * Statically-allocated string data structure with input validation to ensure
+ * strings are properly NULL-terminated.
+ *
+ * This is intentionally kept as simple as possible and should not be extended
+ * to perform complex string operations without a good use case.
+ */
+struct string {
+	char data[STRING_MAX_SIZE];
+};
+
+/**
+ * Macro to initialize `struct string` from a string constant.
+ * Triggers a compilation error if the string does not fit into the buffer.
+ */
+#define STRING_INIT(str) ((struct string){.data = str})
+
+enum string_return_code string_init(struct string *str, const char *data,
+				    size_t size);
+void string_init_empty(struct string *str);
+bool string_is_empty(const struct string *str);
+const char *string_data(const struct string *str);
diff --git a/inc/hf/vcpu.h b/inc/hf/vcpu.h
new file mode 100644
index 0000000..87d5e8c
--- /dev/null
+++ b/inc/hf/vcpu.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/addr.h"
+#include "hf/spinlock.h"
+
+#include "vmapi/hf/spci.h"
+
+/** The number of bits in each element of the interrupt bitfields. */
+#define INTERRUPT_REGISTER_BITS 32
+
+enum vcpu_state {
+	/** The vCPU is switched off. */
+	VCPU_STATE_OFF,
+
+	/** The vCPU is ready to be run. */
+	VCPU_STATE_READY,
+
+	/** The vCPU is currently running. */
+	VCPU_STATE_RUNNING,
+
+	/** The vCPU is waiting for a message. */
+	VCPU_STATE_BLOCKED_MAILBOX,
+
+	/** The vCPU is waiting for an interrupt. */
+	VCPU_STATE_BLOCKED_INTERRUPT,
+
+	/** The vCPU has aborted. */
+	VCPU_STATE_ABORTED,
+};
+
+struct interrupts {
+	/** Bitfield keeping track of which interrupts are enabled. */
+	uint32_t interrupt_enabled[HF_NUM_INTIDS / INTERRUPT_REGISTER_BITS];
+	/** Bitfield keeping track of which interrupts are pending. */
+	uint32_t interrupt_pending[HF_NUM_INTIDS / INTERRUPT_REGISTER_BITS];
+	/**
+	 * The number of interrupts which are currently both enabled and
+	 * pending. i.e. the number of bits set in interrupt_enable &
+	 * interrupt_pending.
+	 */
+	uint32_t enabled_and_pending_count;
+};
+
+struct vcpu_fault_info {
+	ipaddr_t ipaddr;
+	vaddr_t vaddr;
+	vaddr_t pc;
+	uint32_t mode;
+};
+
+struct vcpu {
+	struct spinlock lock;
+
+	/*
+	 * The state is only changed in the context of the vCPU being run. This
+	 * ensures the scheduler can easily keep track of the vCPU state as
+	 * transitions are indicated by the return code from the run call.
+	 */
+	enum vcpu_state state;
+
+	struct cpu *cpu;
+	struct vm *vm;
+	struct arch_regs regs;
+	struct interrupts interrupts;
+
+	/*
+	 * Determine whether the 'regs' field is available for use. This is set
+	 * to false when a vCPU is about to run on a physical CPU, and is set
+	 * back to true when it is descheduled.
+	 */
+	bool regs_available;
+};
+
+/** Encapsulates a vCPU whose lock is held. */
+struct vcpu_locked {
+	struct vcpu *vcpu;
+};
+
+struct vcpu_locked vcpu_lock(struct vcpu *vcpu);
+void vcpu_unlock(struct vcpu_locked *locked);
+void vcpu_init(struct vcpu *vcpu, struct vm *vm);
+void vcpu_on(struct vcpu_locked vcpu, ipaddr_t entry, uintreg_t arg);
+spci_vcpu_index_t vcpu_index(const struct vcpu *vcpu);
+bool vcpu_is_off(struct vcpu_locked vcpu);
+bool vcpu_secondary_reset_and_start(struct vcpu *vcpu, ipaddr_t entry,
+				    uintreg_t arg);
+
+bool vcpu_handle_page_fault(const struct vcpu *current,
+			    struct vcpu_fault_info *f);
diff --git a/inc/hf/vm.h b/inc/hf/vm.h
new file mode 100644
index 0000000..9fff714
--- /dev/null
+++ b/inc/hf/vm.h
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdatomic.h>
+
+#include "hf/arch/types.h"
+
+#include "hf/cpu.h"
+#include "hf/list.h"
+#include "hf/mm.h"
+#include "hf/mpool.h"
+
+#include "vmapi/hf/spci.h"
+
+#define MAX_SMCS 32
+#define LOG_BUFFER_SIZE 256
+
+/**
+ * The state of an RX buffer.
+ *
+ * EMPTY is the initial state. The follow state transitions are possible:
+ * * EMPTY → RECEIVED: message sent to the VM.
+ * * RECEIVED → READ: secondary VM returns from SPCI_MSG_WAIT or
+ *   SPCI_MSG_POLL, or primary VM returns from SPCI_RUN with an SPCI_MSG_SEND
+ *   where the receiver is itself.
+ * * READ → EMPTY: VM called SPCI_RX_RELEASE.
+ */
+enum mailbox_state {
+	/** There is no message in the mailbox. */
+	MAILBOX_STATE_EMPTY,
+
+	/** There is a message in the mailbox that is waiting for a reader. */
+	MAILBOX_STATE_RECEIVED,
+
+	/** There is a message in the mailbox that has been read. */
+	MAILBOX_STATE_READ,
+};
+
+struct wait_entry {
+	/** The VM that is waiting for a mailbox to become writable. */
+	struct vm *waiting_vm;
+
+	/**
+	 * Links used to add entry to a VM's waiter_list. This is protected by
+	 * the notifying VM's lock.
+	 */
+	struct list_entry wait_links;
+
+	/**
+	 * Links used to add entry to a VM's ready_list. This is protected by
+	 * the waiting VM's lock.
+	 */
+	struct list_entry ready_links;
+};
+
+struct mailbox {
+	enum mailbox_state state;
+	void *recv;
+	const void *send;
+
+	/** The ID of the VM which sent the message currently in `recv`. */
+	spci_vm_id_t recv_sender;
+
+	/** The size of the message currently in `recv`. */
+	uint32_t recv_size;
+
+	/** The attributes of the message currently in `recv`. */
+	uint32_t recv_attributes;
+
+	/**
+	 * List of wait_entry structs representing VMs that want to be notified
+	 * when the mailbox becomes writable. Once the mailbox does become
+	 * writable, the entry is removed from this list and added to the
+	 * waiting VM's ready_list.
+	 */
+	struct list_entry waiter_list;
+
+	/**
+	 * List of wait_entry structs representing VMs whose mailboxes became
+	 * writable since the owner of the mailbox registers for notification.
+	 */
+	struct list_entry ready_list;
+};
+
+struct smc_whitelist {
+	uint32_t smcs[MAX_SMCS];
+	uint16_t smc_count;
+	bool permissive;
+};
+
+struct vm {
+	spci_vm_id_t id;
+	struct smc_whitelist smc_whitelist;
+
+	/** See api.c for the partial ordering on locks. */
+	struct spinlock lock;
+	spci_vcpu_count_t vcpu_count;
+	struct vcpu vcpus[MAX_CPUS];
+	struct mm_ptable ptable;
+	struct mailbox mailbox;
+	char log_buffer[LOG_BUFFER_SIZE];
+	uint16_t log_buffer_length;
+
+	/**
+	 * Wait entries to be used when waiting on other VM mailboxes. See
+	 * comments on `struct wait_entry` for the lock discipline of these.
+	 */
+	struct wait_entry wait_entries[MAX_VMS];
+
+	atomic_bool aborting;
+
+	/** Arch-specific VM information. */
+	struct arch_vm arch;
+};
+
+/** Encapsulates a VM whose lock is held. */
+struct vm_locked {
+	struct vm *vm;
+};
+
+/** Container for two vm_locked structures */
+struct two_vm_locked {
+	struct vm_locked vm1;
+	struct vm_locked vm2;
+};
+
+bool vm_init(spci_vcpu_count_t vcpu_count, struct mpool *ppool,
+	     struct vm **new_vm);
+spci_vm_count_t vm_get_count(void);
+struct vm *vm_find(spci_vm_id_t id);
+struct vm_locked vm_lock(struct vm *vm);
+struct two_vm_locked vm_lock_both(struct vm *vm1, struct vm *vm2);
+void vm_unlock(struct vm_locked *locked);
+struct vcpu *vm_get_vcpu(struct vm *vm, spci_vcpu_index_t vcpu_index);
+struct wait_entry *vm_get_wait_entry(struct vm *vm, spci_vm_id_t for_vm);
+spci_vm_id_t vm_id_for_wait_entry(struct vm *vm, struct wait_entry *entry);
+
+bool vm_identity_map(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+		     uint32_t mode, struct mpool *ppool, ipaddr_t *ipa);
+bool vm_identity_prepare(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+			 uint32_t mode, struct mpool *ppool);
+void vm_identity_commit(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+			uint32_t mode, struct mpool *ppool, ipaddr_t *ipa);
+bool vm_unmap(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+	      struct mpool *ppool);
+bool vm_unmap_hypervisor(struct vm_locked vm_locked, struct mpool *ppool);
diff --git a/inc/system/sys/cdefs.h b/inc/system/sys/cdefs.h
new file mode 100644
index 0000000..ef2bc8f
--- /dev/null
+++ b/inc/system/sys/cdefs.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/*
+ * Empty file to make Android Clang stdatomic.h happy. It includes this internal
+ * glibc header which we don't have, but doesn't actually need it.
+ * TODO: Investigate why Android have replaced the upstream Clang version of
+ * stdatomic.h with one that appears to be from FreeBSD, possibly via Bionic, in
+ * their prebuilt version of Clang. If we can just use the upstream Clang we can
+ * probably remove this workaround.
+ */
diff --git a/inc/system/sys/types.h b/inc/system/sys/types.h
new file mode 100644
index 0000000..ef2bc8f
--- /dev/null
+++ b/inc/system/sys/types.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/*
+ * Empty file to make Android Clang stdatomic.h happy. It includes this internal
+ * glibc header which we don't have, but doesn't actually need it.
+ * TODO: Investigate why Android have replaced the upstream Clang version of
+ * stdatomic.h with one that appears to be from FreeBSD, possibly via Bionic, in
+ * their prebuilt version of Clang. If we can just use the upstream Clang we can
+ * probably remove this workaround.
+ */
diff --git a/inc/vmapi/hf/abi.h b/inc/vmapi/hf/abi.h
new file mode 100644
index 0000000..ed004b7
--- /dev/null
+++ b/inc/vmapi/hf/abi.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/spci.h"
+#include "hf/types.h"
+
+/* Keep macro alignment */
+/* clang-format off */
+
+/* TODO: Define constants below according to spec. */
+#define HF_VM_GET_COUNT                0xff01
+#define HF_VCPU_GET_COUNT              0xff02
+#define HF_MAILBOX_WRITABLE_GET        0xff03
+#define HF_MAILBOX_WAITER_GET          0xff04
+#define HF_INTERRUPT_ENABLE            0xff05
+#define HF_INTERRUPT_GET               0xff06
+#define HF_INTERRUPT_INJECT            0xff07
+
+/* Custom SPCI-like calls returned from SPCI_RUN. */
+#define HF_SPCI_RUN_WAIT_FOR_INTERRUPT 0xff09
+#define HF_SPCI_RUN_WAKE_UP            0xff0a
+
+/* This matches what Trusty and its ATF module currently use. */
+#define HF_DEBUG_LOG            0xbd000000
+
+/* clang-format on */
diff --git a/inc/vmapi/hf/call.h b/inc/vmapi/hf/call.h
new file mode 100644
index 0000000..f2455e2
--- /dev/null
+++ b/inc/vmapi/hf/call.h
@@ -0,0 +1,289 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/abi.h"
+#include "hf/spci.h"
+#include "hf/types.h"
+
+/**
+ * This function must be implemented to trigger the architecture-specific
+ * mechanism to call to the hypervisor.
+ */
+int64_t hf_call(uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3);
+struct spci_value spci_call(struct spci_value args);
+
+/**
+ * Returns the VM's own ID.
+ */
+static inline struct spci_value spci_id_get(void)
+{
+	return spci_call((struct spci_value){.func = SPCI_ID_GET_32});
+}
+
+/**
+ * Returns the VM's own ID.
+ */
+static inline spci_vm_id_t hf_vm_get_id(void)
+{
+	return spci_id_get().arg2;
+}
+
+/**
+ * Returns the number of secondary VMs.
+ */
+static inline spci_vm_count_t hf_vm_get_count(void)
+{
+	return hf_call(HF_VM_GET_COUNT, 0, 0, 0);
+}
+
+/**
+ * Returns the number of vCPUs configured in the given secondary VM.
+ */
+static inline spci_vcpu_count_t hf_vcpu_get_count(spci_vm_id_t vm_id)
+{
+	return hf_call(HF_VCPU_GET_COUNT, vm_id, 0, 0);
+}
+
+/**
+ * Runs the given vCPU of the given VM.
+ */
+static inline struct spci_value spci_run(spci_vm_id_t vm_id,
+					 spci_vcpu_index_t vcpu_idx)
+{
+	return spci_call((struct spci_value){.func = SPCI_RUN_32,
+					     spci_vm_vcpu(vm_id, vcpu_idx)});
+}
+
+/**
+ * Hints that the vCPU is willing to yield its current use of the physical CPU.
+ * This call always returns SPCI_SUCCESS.
+ */
+static inline struct spci_value spci_yield(void)
+{
+	return spci_call((struct spci_value){.func = SPCI_YIELD_32});
+}
+
+/**
+ * Configures the pages to send/receive data through. The pages must not be
+ * shared.
+ *
+ * Returns:
+ *  - SPCI_ERROR SPCI_INVALID_PARAMETERS if the given addresses are not properly
+ *    aligned or are the same.
+ *  - SPCI_ERROR SPCI_NO_MEMORY if the hypervisor was unable to map the buffers
+ *    due to insuffient page table memory.
+ *  - SPCI_ERROR SPCI_DENIED if the pages are already mapped or are not owned by
+ *    the caller.
+ *  - SPCI_SUCCESS on success if no further action is needed.
+ *  - SPCI_RX_RELEASE if it was called by the primary VM and the primary VM now
+ *    needs to wake up or kick waiters.
+ */
+static inline struct spci_value spci_rxtx_map(hf_ipaddr_t send,
+					      hf_ipaddr_t recv)
+{
+	return spci_call(
+		(struct spci_value){.func = SPCI_RXTX_MAP_32,
+				    .arg1 = send,
+				    .arg2 = recv,
+				    .arg3 = HF_MAILBOX_SIZE / SPCI_PAGE_SIZE});
+}
+
+/**
+ * Copies data from the sender's send buffer to the recipient's receive buffer.
+ *
+ * If the recipient's receive buffer is busy, it can optionally register the
+ * caller to be notified when the recipient's receive buffer becomes available.
+ *
+ * Attributes may include:
+ *  - SPCI_MSG_SEND_NOTIFY, to notify the caller when it should try again.
+ *  - SPCI_MSG_SEND_LEGACY_MEMORY_*, to send a legacy architected memory sharing
+ *    message.
+ *
+ * Returns SPCI_SUCCESS if the message is sent, or an error code otherwise:
+ *  - INVALID_PARAMETERS: one or more of the parameters do not conform.
+ *  - BUSY: the message could not be delivered either because the mailbox
+ *    was full or the target VM is not yet set up.
+ */
+static inline struct spci_value spci_msg_send(spci_vm_id_t sender_vm_id,
+					      spci_vm_id_t target_vm_id,
+					      uint32_t size,
+					      uint32_t attributes)
+{
+	return spci_call((struct spci_value){
+		.func = SPCI_MSG_SEND_32,
+		.arg1 = ((uint64_t)sender_vm_id << 16) | target_vm_id,
+		.arg3 = size,
+		.arg4 = attributes});
+}
+
+/**
+ * Called by secondary VMs to receive a message. This will block until a message
+ * is received.
+ *
+ * The mailbox must be cleared before a new message can be received.
+ *
+ * If no message is immediately available and there are no enabled and pending
+ * interrupts (irrespective of whether interrupts are enabled globally), then
+ * this will block until a message is available or an enabled interrupt becomes
+ * pending. This matches the behaviour of the WFI instruction on AArch64, except
+ * that a message becoming available is also treated like a wake-up event.
+ *
+ * Returns:
+ *  - SPCI_MSG_SEND if a message is successfully received.
+ *  - SPCI_ERROR SPCI_NOT_SUPPORTED if called from the primary VM.
+ *  - SPCI_ERROR SPCI_INTERRUPTED if an interrupt happened during the call.
+ */
+static inline struct spci_value spci_msg_wait(void)
+{
+	return spci_call((struct spci_value){.func = SPCI_MSG_WAIT_32});
+}
+
+/**
+ * Called by secondary VMs to receive a message. The call will return whether or
+ * not a message is available.
+ *
+ * The mailbox must be cleared before a new message can be received.
+ *
+ * Returns:
+ *  - SPCI_MSG_SEND if a message is successfully received.
+ *  - SPCI_ERROR SPCI_NOT_SUPPORTED if called from the primary VM.
+ *  - SPCI_ERROR SPCI_INTERRUPTED if an interrupt happened during the call.
+ *  - SPCI_ERROR SPCI_RETRY if there was no pending message.
+ */
+static inline struct spci_value spci_msg_poll(void)
+{
+	return spci_call((struct spci_value){.func = SPCI_MSG_POLL_32});
+}
+
+/**
+ * Releases the caller's mailbox so that a new message can be received. The
+ * caller must have copied out all data they wish to preserve as new messages
+ * will overwrite the old and will arrive asynchronously.
+ *
+ * Returns:
+ *  - SPCI_ERROR SPCI_DENIED on failure, if the mailbox hasn't been read.
+ *  - SPCI_SUCCESS on success if no further action is needed.
+ *  - SPCI_RX_RELEASE if it was called by the primary VM and the primary VM now
+ *    needs to wake up or kick waiters. Waiters should be retrieved by calling
+ *    hf_mailbox_waiter_get.
+ */
+static inline struct spci_value spci_rx_release(void)
+{
+	return spci_call((struct spci_value){.func = SPCI_RX_RELEASE_32});
+}
+
+/**
+ * Retrieves the next VM whose mailbox became writable. For a VM to be notified
+ * by this function, the caller must have called api_mailbox_send before with
+ * the notify argument set to true, and this call must have failed because the
+ * mailbox was not available.
+ *
+ * It should be called repeatedly to retrieve a list of VMs.
+ *
+ * Returns -1 if no VM became writable, or the id of the VM whose mailbox
+ * became writable.
+ */
+static inline int64_t hf_mailbox_writable_get(void)
+{
+	return hf_call(HF_MAILBOX_WRITABLE_GET, 0, 0, 0);
+}
+
+/**
+ * Retrieves the next VM waiting to be notified that the mailbox of the
+ * specified VM became writable. Only primary VMs are allowed to call this.
+ *
+ * Returns -1 on failure or if there are no waiters; the VM id of the next
+ * waiter otherwise.
+ */
+static inline int64_t hf_mailbox_waiter_get(spci_vm_id_t vm_id)
+{
+	return hf_call(HF_MAILBOX_WAITER_GET, vm_id, 0, 0);
+}
+
+/**
+ * Enables or disables a given interrupt ID.
+ *
+ * Returns 0 on success, or -1 if the intid is invalid.
+ */
+static inline int64_t hf_interrupt_enable(uint32_t intid, bool enable)
+{
+	return hf_call(HF_INTERRUPT_ENABLE, intid, enable, 0);
+}
+
+/**
+ * Gets the ID of the pending interrupt (if any) and acknowledge it.
+ *
+ * Returns HF_INVALID_INTID if there are no pending interrupts.
+ */
+static inline uint32_t hf_interrupt_get(void)
+{
+	return hf_call(HF_INTERRUPT_GET, 0, 0, 0);
+}
+
+/**
+ * Injects a virtual interrupt of the given ID into the given target vCPU.
+ * This doesn't cause the vCPU to actually be run immediately; it will be taken
+ * when the vCPU is next run, which is up to the scheduler.
+ *
+ * Returns:
+ *  - -1 on failure because the target VM or vCPU doesn't exist, the interrupt
+ *    ID is invalid, or the current VM is not allowed to inject interrupts to
+ *    the target VM.
+ *  - 0 on success if no further action is needed.
+ *  - 1 if it was called by the primary VM and the primary VM now needs to wake
+ *    up or kick the target vCPU.
+ */
+static inline int64_t hf_interrupt_inject(spci_vm_id_t target_vm_id,
+					  spci_vcpu_index_t target_vcpu_idx,
+					  uint32_t intid)
+{
+	return hf_call(HF_INTERRUPT_INJECT, target_vm_id, target_vcpu_idx,
+		       intid);
+}
+
+/**
+ * Sends a character to the debug log for the VM.
+ *
+ * Returns 0 on success, or -1 if it failed for some reason.
+ */
+static inline int64_t hf_debug_log(char c)
+{
+	return hf_call(HF_DEBUG_LOG, c, 0, 0);
+}
+
+/** Obtains the Hafnium's version of the implemented SPCI specification. */
+static inline struct spci_value spci_version(void)
+{
+	return spci_call((struct spci_value){.func = SPCI_VERSION_32});
+}
+
+/**
+ * Discovery function returning information about the implementation of optional
+ * SPCI interfaces.
+ *
+ * Returns:
+ *  - SPCI_SUCCESS in .func if the optional interface with function_id is
+ * implemented.
+ *  - SPCI_ERROR in .func if the optional interface with function_id is not
+ * implemented.
+ */
+static inline struct spci_value spci_features(uint32_t function_id)
+{
+	return spci_call((struct spci_value){.func = SPCI_FEATURES_32,
+					     .arg1 = function_id});
+}
diff --git a/inc/vmapi/hf/spci.h b/inc/vmapi/hf/spci.h
new file mode 100644
index 0000000..12600c1
--- /dev/null
+++ b/inc/vmapi/hf/spci.h
@@ -0,0 +1,319 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/types.h"
+
+/* clang-format off */
+
+#define SPCI_LOW_32_ID  0x84000060
+#define SPCI_HIGH_32_ID 0x8400007F
+#define SPCI_LOW_64_ID  0xC4000060
+#define SPCI_HIGH_32_ID 0x8400007F
+
+/* SPCI function identifiers. */
+#define SPCI_ERROR_32                 0x84000060
+#define SPCI_SUCCESS_32               0x84000061
+#define SPCI_INTERRUPT_32             0x84000062
+#define SPCI_VERSION_32               0x84000063
+#define SPCI_FEATURES_32              0x84000064
+#define SPCI_RX_RELEASE_32            0x84000065
+#define SPCI_RXTX_MAP_32              0x84000066
+#define SPCI_RXTX_UNMAP_32            0x84000067
+#define SPCI_PARTITION_INFO_GET_32    0x84000068
+#define SPCI_ID_GET_32                0x84000069
+#define SPCI_MSG_POLL_32              0x8400006A
+#define SPCI_MSG_WAIT_32              0x8400006B
+#define SPCI_YIELD_32                 0x8400006C
+#define SPCI_RUN_32                   0x8400006D
+#define SPCI_MSG_SEND_32              0x8400006E
+#define SPCI_MSG_SEND_DIRECT_REQ_32   0x8400006F
+#define SPCI_MSG_SEND_DIRECT_RESP_32  0x84000070
+
+/* SPCI error codes. */
+#define SPCI_NOT_SUPPORTED      INT32_C(-1)
+#define SPCI_INVALID_PARAMETERS INT32_C(-2)
+#define SPCI_NO_MEMORY          INT32_C(-3)
+#define SPCI_BUSY               INT32_C(-4)
+#define SPCI_INTERRUPTED        INT32_C(-5)
+#define SPCI_DENIED             INT32_C(-6)
+#define SPCI_RETRY              INT32_C(-7)
+#define SPCI_ABORTED            INT32_C(-8)
+
+/* SPCI function specific constants. */
+#define SPCI_MSG_RECV_BLOCK  0x1
+#define SPCI_MSG_RECV_BLOCK_MASK  0x1
+
+#define SPCI_MSG_SEND_NOTIFY 0x1
+#define SPCI_MSG_SEND_NOTIFY_MASK 0x1
+#define SPCI_MSG_SEND_LEGACY_MEMORY_DONATE 0x10
+#define SPCI_MSG_SEND_LEGACY_MEMORY_LEND 0x20
+#define SPCI_MSG_SEND_LEGACY_MEMORY_SHARE 0x30
+#define SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH 0x40
+#define SPCI_MSG_SEND_LEGACY_MEMORY_MASK 0x70
+
+#define SPCI_SLEEP_INDEFINITE 0
+
+/**
+ * For use where the SPCI specification refers explicitly to '4K pages'. Not to
+ * be confused with PAGE_SIZE, which is the translation granule Hafnium is
+ * configured to use.
+ */
+#define SPCI_PAGE_SIZE 4096
+
+/* The maximum length possible for a single message. */
+#define SPCI_MSG_PAYLOAD_MAX HF_MAILBOX_SIZE
+
+enum spci_memory_access {
+	SPCI_MEMORY_RO_NX,
+	SPCI_MEMORY_RO_X,
+	SPCI_MEMORY_RW_NX,
+	SPCI_MEMORY_RW_X,
+};
+
+enum spci_memory_type {
+	SPCI_MEMORY_DEVICE_MEM,
+	SPCI_MEMORY_NORMAL_MEM,
+};
+
+enum spci_memory_cacheability {
+	SPCI_MEMORY_CACHE_RESERVED = 0x0,
+	SPCI_MEMORY_CACHE_NON_CACHEABLE = 0x1,
+	SPCI_MEMORY_CACHE_WRITE_THROUGH = 0x2,
+	SPCI_MEMORY_CACHE_WRITE_BACK = 0x4,
+	SPCI_MEMORY_DEV_NGNRNE = 0x0,
+	SPCI_MEMORY_DEV_NGNRE = 0x1,
+	SPCI_MEMORY_DEV_NGRE = 0x2,
+	SPCI_MEMORY_DEV_GRE = 0x3,
+};
+
+enum spci_memory_shareability {
+	SPCI_MEMORY_SHARE_NON_SHAREABLE,
+	SPCI_MEMORY_RESERVED,
+	SPCI_MEMORY_OUTER_SHAREABLE,
+	SPCI_MEMORY_INNER_SHAREABLE,
+};
+
+#define SPCI_MEMORY_ACCESS_OFFSET (0x5U)
+#define SPCI_MEMORY_ACCESS_MASK ((0x3U) << SPCI_MEMORY_ACCESS_OFFSET)
+
+#define SPCI_MEMORY_TYPE_OFFSET (0x4U)
+#define SPCI_MEMORY_TYPE_MASK ((0x1U) << SPCI_MEMORY_TYPE_OFFSET)
+
+#define SPCI_MEMORY_CACHEABILITY_OFFSET (0x2U)
+#define SPCI_MEMORY_CACHEABILITY_MASK ((0x3U) <<\
+	SPCI_MEMORY_CACHEABILITY_OFFSET)
+
+#define SPCI_MEMORY_SHAREABILITY_OFFSET (0x0U)
+#define SPCI_MEMORY_SHAREABILITY_MASK ((0x3U) <<\
+	SPCI_MEMORY_SHAREABILITY_OFFSET)
+
+#define LEND_ATTR_FUNCTION_SET(name, offset, mask) \
+static inline void spci_set_memory_##name##_attr(uint16_t *attr,\
+	const enum spci_memory_##name perm)\
+{\
+	*attr = (*attr & ~(mask)) | ((perm  << offset) & mask);\
+}
+
+#define LEND_ATTR_FUNCTION_GET(name, offset, mask) \
+static inline enum spci_memory_##name spci_get_memory_##name##_attr(\
+	uint16_t attr)\
+{\
+	return (enum spci_memory_##name)((attr & mask) >> offset);\
+}
+
+LEND_ATTR_FUNCTION_SET(access, SPCI_MEMORY_ACCESS_OFFSET,
+	SPCI_MEMORY_ACCESS_MASK)
+LEND_ATTR_FUNCTION_GET(access, SPCI_MEMORY_ACCESS_OFFSET,
+	SPCI_MEMORY_ACCESS_MASK)
+
+LEND_ATTR_FUNCTION_SET(type, SPCI_MEMORY_TYPE_OFFSET, SPCI_MEMORY_TYPE_MASK)
+LEND_ATTR_FUNCTION_GET(type, SPCI_MEMORY_TYPE_OFFSET, SPCI_MEMORY_TYPE_MASK)
+
+LEND_ATTR_FUNCTION_SET(cacheability, SPCI_MEMORY_CACHEABILITY_OFFSET,
+	SPCI_MEMORY_CACHEABILITY_MASK)
+
+LEND_ATTR_FUNCTION_GET(cacheability, SPCI_MEMORY_CACHEABILITY_OFFSET,
+	SPCI_MEMORY_CACHEABILITY_MASK)
+
+LEND_ATTR_FUNCTION_SET(shareability, SPCI_MEMORY_SHAREABILITY_OFFSET,
+	SPCI_MEMORY_SHAREABILITY_MASK)
+
+LEND_ATTR_FUNCTION_GET(shareability, SPCI_MEMORY_SHAREABILITY_OFFSET,
+	SPCI_MEMORY_SHAREABILITY_MASK)
+
+/* clang-format on */
+
+/** The ID of a VM. These are assigned sequentially starting with an offset. */
+typedef uint16_t spci_vm_id_t;
+typedef uint32_t spci_memory_handle_t;
+
+/**
+ * A count of VMs. This has the same range as the VM IDs but we give it a
+ * different name to make the different semantics clear.
+ */
+typedef spci_vm_id_t spci_vm_count_t;
+
+/** The index of a vCPU within a particular VM. */
+typedef uint16_t spci_vcpu_index_t;
+
+/**
+ * A count of vCPUs. This has the same range as the vCPU indices but we give it
+ * a different name to make the different semantics clear.
+ */
+typedef spci_vcpu_index_t spci_vcpu_count_t;
+
+/** Parameter and return type of SPCI functions. */
+struct spci_value {
+	uint64_t func;
+	uint64_t arg1;
+	uint64_t arg2;
+	uint64_t arg3;
+	uint64_t arg4;
+	uint64_t arg5;
+	uint64_t arg6;
+	uint64_t arg7;
+};
+
+static inline spci_vm_id_t spci_msg_send_sender(struct spci_value args)
+{
+	return (args.arg1 >> 16) & 0xffff;
+}
+
+static inline spci_vm_id_t spci_msg_send_receiver(struct spci_value args)
+{
+	return args.arg1 & 0xffff;
+}
+
+static inline uint32_t spci_msg_send_size(struct spci_value args)
+{
+	return args.arg3;
+}
+
+static inline uint32_t spci_msg_send_attributes(struct spci_value args)
+{
+	return args.arg4;
+}
+
+static inline spci_vm_id_t spci_vm_id(struct spci_value args)
+{
+	return (args.arg1 >> 16) & 0xffff;
+}
+
+static inline spci_vcpu_index_t spci_vcpu_index(struct spci_value args)
+{
+	return args.arg1 & 0xffff;
+}
+
+static inline uint64_t spci_vm_vcpu(spci_vm_id_t vm_id,
+				    spci_vcpu_index_t vcpu_index)
+{
+	return ((uint32_t)vm_id << 16) | vcpu_index;
+}
+
+struct spci_memory_region_constituent {
+	/**
+	 * The base IPA of the constituent memory region, aligned to 4 kiB page
+	 * size granularity.
+	 */
+	uint64_t address;
+	/** The number of 4 kiB pages in the constituent memory region. */
+	uint32_t page_count;
+
+	uint32_t reserved;
+};
+
+struct spci_memory_region_attributes {
+	/** The ID of the VM to which the memory is being given or shared. */
+	spci_vm_id_t receiver;
+	/**
+	 * The attributes with which the memory region should be mapped in the
+	 * receiver's page table.
+	 */
+	uint16_t memory_attributes;
+};
+
+/** Flags to control the behaviour of a memory sharing transaction. */
+typedef uint32_t spci_memory_region_flags_t;
+
+/**
+ * Clear memory region contents after unmapping it from the sender and before
+ * mapping it for any receiver.
+ */
+#define SPCI_MEMORY_REGION_FLAG_CLEAR 0x1
+
+struct spci_memory_region {
+	/**
+	 * An implementation defined value associated with the receiver and the
+	 * memory region.
+	 */
+	uint32_t tag;
+	/** Flags to control behaviour of the transaction. */
+	spci_memory_region_flags_t flags;
+	/** Sender VM ID. */
+	spci_vm_id_t sender;
+	/** Reserved field, must be 0. */
+	uint16_t reserved;
+	/**
+	 * The total number of 4 kiB pages included in this memory region. This
+	 * must be equal to the sum of page counts specified in each
+	 * `spci_memory_region_constituent`.
+	 */
+	uint32_t page_count;
+	/**
+	 * The number of constituents (`spci_memory_region_constituent`)
+	 * included in this memory region.
+	 */
+	uint32_t constituent_count;
+	/**
+	 * The offset in bytes from the base address of this
+	 * `spci_memory_region` to the start of the first
+	 * `spci_memory_region_constituent`.
+	 */
+	uint32_t constituent_offset;
+	/**
+	 * The number of `spci_memory_region_attributes` entries included in
+	 * this memory region.
+	 */
+	uint32_t attribute_count;
+	/**
+	 * An array of `attribute_count` memory region attribute descriptors.
+	 * Each one specifies an endpoint and the attributes with which this
+	 * memory region should be mapped in that endpoint's page table.
+	 */
+	struct spci_memory_region_attributes attributes[];
+};
+
+/**
+ * Gets the constituent array for an `spci_memory_region`.
+ */
+static inline struct spci_memory_region_constituent *
+spci_memory_region_get_constituents(struct spci_memory_region *memory_region)
+{
+	return (struct spci_memory_region_constituent
+			*)((uint8_t *)memory_region +
+			   memory_region->constituent_offset);
+}
+
+uint32_t spci_memory_region_init(
+	struct spci_memory_region *memory_region, spci_vm_id_t sender,
+	spci_vm_id_t receiver,
+	const struct spci_memory_region_constituent constituents[],
+	uint32_t constituent_count, uint32_t tag,
+	spci_memory_region_flags_t flags, enum spci_memory_access access,
+	enum spci_memory_type type, enum spci_memory_cacheability cacheability,
+	enum spci_memory_shareability shareability);
diff --git a/inc/vmapi/hf/transport.h b/inc/vmapi/hf/transport.h
new file mode 100644
index 0000000..686143c
--- /dev/null
+++ b/inc/vmapi/hf/transport.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/**
+ * Header for Hafnium messages
+ *
+ * NOTE: This is a work in progress.  The final form of a Hafnium message header
+ * is likely to change.
+ */
+struct hf_msg_hdr {
+	uint64_t src_port;
+	uint64_t dst_port;
+};
diff --git a/inc/vmapi/hf/types.h b/inc/vmapi/hf/types.h
new file mode 100644
index 0000000..1f0d5d4
--- /dev/null
+++ b/inc/vmapi/hf/types.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/* Define the standard types for the platform. */
+#if defined(__linux__) && defined(__KERNEL__)
+
+#include <linux/types.h>
+
+#define INT32_C(c) c
+
+typedef phys_addr_t hf_ipaddr_t;
+
+#else
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+typedef uintptr_t hf_ipaddr_t;
+
+#endif
+
+/**
+ * An offset to use when assigning VM IDs.
+ * The offset is needed because VM ID 0 is reserved.
+ */
+#define HF_VM_ID_OFFSET 1
+
+/**
+ * The index and ID of the primary VM, which is responsible for scheduling.
+ *
+ * These are not equal because ID 0 is reserved for the hypervisor itself.
+ * Primary VM therefore gets ID 1 and all other VMs come after that.
+ */
+#define HF_PRIMARY_VM_INDEX 0
+#define HF_PRIMARY_VM_ID (HF_VM_ID_OFFSET + HF_PRIMARY_VM_INDEX)
+
+/** Sleep value for an indefinite period of time. */
+#define HF_SLEEP_INDEFINITE 0xffffffffffffffff
+
+/** The amount of data that can be sent to a mailbox. */
+#define HF_MAILBOX_SIZE 4096
+
+/** The number of virtual interrupt IDs which are supported. */
+#define HF_NUM_INTIDS 64
+
+/** Interrupt ID returned when there is no interrupt pending. */
+#define HF_INVALID_INTID 0xffffffff
+
+/** Interrupt ID indicating the mailbox is readable. */
+#define HF_MAILBOX_READABLE_INTID 1
+
+/** Interrupt ID indicating a mailbox is writable. */
+#define HF_MAILBOX_WRITABLE_INTID 2
+
+/** The virtual interrupt ID used for the virtual timer. */
+#define HF_VIRTUAL_TIMER_INTID 3
diff --git a/kokoro/ubuntu/build.sh b/kokoro/ubuntu/build.sh
new file mode 100755
index 0000000..ecedad6
--- /dev/null
+++ b/kokoro/ubuntu/build.sh
@@ -0,0 +1,188 @@
+#!/bin/bash
+#
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_PATH="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")"
+ROOT_DIR="$(dirname $(dirname "${SCRIPT_DIR}"))"
+
+REPO="${ROOT_DIR}/prebuilts/generic/repo/repo"
+
+# Fail on any error.
+set -e
+# Fail on any part of a pipeline failing.
+set -o pipefail
+# Treat unset variables as an error.
+set -u
+# Display commands being run.
+set -x
+
+# Returns true if `git status` reports uncommitted changes in the source tree.
+function is_repo_dirty() {
+	local cmd=(git status --porcelain=v1)
+	if [ -d ".repo" ]
+	then
+		# This source tree was checked out using `repo`. Check the
+		# status of all projects.
+		cmd=(${REPO} forall -c "${cmd[@]}")
+	fi
+	! (u="$(${cmd[@]})" && test -z "$u")
+	return $?
+}
+
+# Assigns value (second arg) of a variable (first arg) if it is not set already.
+function default_value {
+	local var_name=$1
+	local value=$2
+	export ${var_name}=${!var_name:-${value}}
+}
+
+# Assign default values to variables.
+if [ -v KOKORO_JOB_NAME ]
+then
+	# Default config for Kokoro builds.
+	default_value HAFNIUM_HERMETIC_BUILD true
+	default_value HAFNIUM_SKIP_LONG_RUNNING_TESTS false
+	default_value HAFNIUM_RUN_ALL_QEMU_CPUS true
+else
+	# Default config for local builds.
+	default_value HAFNIUM_HERMETIC_BUILD false
+	default_value HAFNIUM_SKIP_LONG_RUNNING_TESTS true
+	default_value HAFNIUM_RUN_ALL_QEMU_CPUS false
+fi
+
+# If HAFNIUM_HERMETIC_BUILD is "true", relaunch this script inside a container.
+# The 'run_in_container.sh' script will set the variable value to 'inside' to
+# avoid recursion.
+if [ "${HAFNIUM_HERMETIC_BUILD}" == "true" ]
+then
+	exec "${ROOT_DIR}/build/run_in_container.sh" ${SCRIPT_PATH} $@
+fi
+
+USE_FVP=false
+
+while test $# -gt 0
+do
+	case "$1" in
+	--fvp)
+		USE_FVP=true
+		;;
+	--skip-long-running-tests)
+		HAFNIUM_SKIP_LONG_RUNNING_TESTS=true
+		;;
+	--run-all-qemu-cpus)
+		HAFNIUM_RUN_ALL_QEMU_CPUS=true
+		;;
+	*)
+		echo "Unexpected argument $1"
+		exit 1
+		;;
+	esac
+	shift
+done
+
+CLANG=${PWD}/prebuilts/linux-x64/clang/bin/clang
+
+# Kokoro does something weird that makes all files look dirty to git diff-index;
+# this fixes it so that the Linux build doesn't think it has a dirty tree for
+# building the Hafnium kernel module (and so end up with a version magic string
+# that doesn't match the prebuilt kernel).
+(
+	cd third_party/linux &&
+	git status
+)
+
+#
+# Step 1: make sure it builds.
+#
+
+for proj in $(cd project/ && ls)
+do
+  make PROJECT=${proj}
+done
+
+#
+# Step 2: make sure it works.
+#
+
+TEST_ARGS=()
+if [ $USE_FVP == true ]
+then
+	TEST_ARGS+=(--fvp)
+fi
+if [ "${HAFNIUM_SKIP_LONG_RUNNING_TESTS}" == "true" ]
+then
+	TEST_ARGS+=(--skip-long-running-tests)
+fi
+if [ "${HAFNIUM_RUN_ALL_QEMU_CPUS}" == "true" ]
+then
+	TEST_ARGS+=(--run-all-qemu-cpus)
+fi
+./kokoro/ubuntu/test.sh ${TEST_ARGS[@]}
+
+#
+# Step 3: static analysis.
+#
+
+make check
+if is_repo_dirty
+then
+	echo "Run \`make check\' locally to fix this."
+	exit 1
+fi
+
+#
+# Step 4: make sure the code looks good.
+#
+
+make format
+if is_repo_dirty
+then
+	echo "Run \`make format\' locally to fix this."
+	exit 1
+fi
+
+make checkpatch
+
+#
+# Step 5: make sure there's not lint.
+#
+
+make tidy
+if is_repo_dirty
+then
+	echo "Run \`make tidy\' locally to fix this."
+	exit 1
+fi
+
+#
+# Step 6: make sure all the files have a license.
+#
+
+make license
+if is_repo_dirty
+then
+	echo "Run \`make license\' locally to fix this."
+	exit 1
+fi
+
+# Step 7: make sure the Linux driver maintains style. It's already built as part
+# of the tests.
+(
+export ARCH=arm64 &&
+export CROSS_COMPILE=aarch64-linux-gnu- &&
+cd driver/linux &&
+make checkpatch
+)
diff --git a/kokoro/ubuntu/continuous.cfg b/kokoro/ubuntu/continuous.cfg
new file mode 100644
index 0000000..cacd607
--- /dev/null
+++ b/kokoro/ubuntu/continuous.cfg
@@ -0,0 +1,14 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Location of the continuous bash script in Git.
+build_file: "hafnium/kokoro/ubuntu/build.sh"
+
+action {
+  define_artifacts {
+    regex: "git/hafnium/out/**/kokoro_log/**/*sponge_log.log"
+    regex: "git/hafnium/out/**/kokoro_log/**/*sponge_log.xml"
+    strip_prefix: "git/hafnium"
+  }
+}
+
+timeout_mins: 10
diff --git a/kokoro/ubuntu/presubmit.cfg b/kokoro/ubuntu/presubmit.cfg
new file mode 100644
index 0000000..7e56cce
--- /dev/null
+++ b/kokoro/ubuntu/presubmit.cfg
@@ -0,0 +1,14 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Location of the presubmit bash script in Git.
+build_file: "hafnium/kokoro/ubuntu/build.sh"
+
+action {
+  define_artifacts {
+    regex: "git/hafnium/out/**/kokoro_log/**/*sponge_log.log"
+    regex: "git/hafnium/out/**/kokoro_log/**/*sponge_log.xml"
+    strip_prefix: "git/hafnium"
+  }
+}
+
+timeout_mins: 10
diff --git a/kokoro/ubuntu/test.sh b/kokoro/ubuntu/test.sh
new file mode 100755
index 0000000..0dcd187
--- /dev/null
+++ b/kokoro/ubuntu/test.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+#
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Note: this assumes that the images have all been built and the current working
+# directory is the root of the repo.
+
+# Fail on any error.
+set -e
+# Fail on any part of a pipeline failing.
+set -o pipefail
+# Treat unset variables as an error.
+set -u
+# Display commands being run.
+set -x
+
+USE_FVP=false
+SKIP_LONG_RUNNING_TESTS=false
+RUN_ALL_QEMU_CPUS=false
+
+while test $# -gt 0
+do
+  case "$1" in
+    --fvp) USE_FVP=true
+      ;;
+    --skip-long-running-tests) SKIP_LONG_RUNNING_TESTS=true
+      ;;
+    --run-all-qemu-cpus) RUN_ALL_QEMU_CPUS=true
+      ;;
+    *) echo "Unexpected argument $1"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+TIMEOUT=(timeout --foreground)
+PROJECT="${PROJECT:-reference}"
+OUT="out/${PROJECT}"
+LOG_DIR_BASE="${OUT}/kokoro_log"
+
+# Run the tests with a timeout so they can't loop forever.
+HFTEST=(${TIMEOUT[@]} 300s ./test/hftest/hftest.py)
+if [ $USE_FVP == true ]
+then
+  HFTEST+=(--fvp)
+  HFTEST+=(--out "$OUT/aem_v8a_fvp_clang")
+  HFTEST+=(--out_initrd "$OUT/aem_v8a_fvp_vm_clang")
+else
+  HFTEST+=(--out "$OUT/qemu_aarch64_clang")
+  HFTEST+=(--out_initrd "$OUT/qemu_aarch64_vm_clang")
+fi
+if [ $SKIP_LONG_RUNNING_TESTS == true ]
+then
+  HFTEST+=(--skip-long-running-tests)
+fi
+
+# Add prebuilt libc++ to the path.
+export LD_LIBRARY_PATH="$PWD/prebuilts/linux-x64/clang/lib64"
+
+# Run the host unit tests.
+mkdir -p "${LOG_DIR_BASE}/unit_tests"
+${TIMEOUT[@]} 30s "$OUT/host_fake_clang/unit_tests" \
+  --gtest_output="xml:${LOG_DIR_BASE}/unit_tests/sponge_log.xml" \
+  | tee "${LOG_DIR_BASE}/unit_tests/sponge_log.log"
+
+CPUS=("")
+
+if [ $RUN_ALL_QEMU_CPUS == true ]
+then
+  CPUS=("cortex-a53" "max")
+fi
+
+for CPU in "${CPUS[@]}"
+do
+  HFTEST_CPU=("${HFTEST[@]}")
+  if [ -n "$CPU" ]
+  then
+    # Per-CPU log directory to avoid filename conflicts.
+    HFTEST_CPU+=(--cpu "$CPU" --log "$LOG_DIR_BASE/$CPU")
+  else
+    HFTEST_CPU+=(--log "$LOG_DIR_BASE")
+  fi
+  "${HFTEST_CPU[@]}" arch_test
+  "${HFTEST_CPU[@]}" hafnium --initrd test/vmapi/arch/aarch64/aarch64_test
+  "${HFTEST_CPU[@]}" hafnium --initrd test/vmapi/arch/aarch64/gicv3/gicv3_test
+  "${HFTEST_CPU[@]}" hafnium --initrd test/vmapi/primary_only/primary_only_test
+  "${HFTEST_CPU[@]}" hafnium --initrd test/vmapi/primary_with_secondaries/primary_with_secondaries_test
+  "${HFTEST_CPU[@]}" hafnium --initrd test/linux/linux_test --vm_args "rdinit=/test_binary --"
+done
diff --git a/navbar.md b/navbar.md
new file mode 100644
index 0000000..f0835ab
--- /dev/null
+++ b/navbar.md
@@ -0,0 +1,8 @@
+# Hafnium
+
+*   [Getting started](/docs/GettingStarted.md)
+*   [Architecture](/docs/Architecture.md)
+*   [Coding style](/docs/StyleGuide.md)
+*   [Documentation index](/docs/README.md)
+
+[home]: /README.md
diff --git a/prebuilts b/prebuilts
new file mode 160000
index 0000000..6fcd301
--- /dev/null
+++ b/prebuilts
@@ -0,0 +1 @@
+Subproject commit 6fcd30188886b8cdd7cdcd1de8be813914eb9015
diff --git a/project/reference b/project/reference
new file mode 160000
index 0000000..e2a525d
--- /dev/null
+++ b/project/reference
@@ -0,0 +1 @@
+Subproject commit e2a525d24927401f8b2d190f234af6ea24c85664
diff --git a/src/BUILD.gn b/src/BUILD.gn
new file mode 100644
index 0000000..1437908
--- /dev/null
+++ b/src/BUILD.gn
@@ -0,0 +1,175 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+import("//build/toolchain/platform.gni")
+
+# The hypervisor image.
+hypervisor("hafnium") {
+  deps = [
+    ":layout",
+    ":src_not_testable_yet",
+  ]
+}
+
+# Hypervisor specific code that isn't. One day it will be testable and both the
+# src targets will merge!
+source_set("src_not_testable_yet") {
+  sources = [
+    "cpio.c",
+    "init.c",
+    "load.c",
+    "main.c",
+  ]
+  deps = [
+    ":src_testable",
+    "//project/${project}/${plat_name}",
+    plat_boot_flow,
+    plat_console,
+    plat_iommu,
+  ]
+}
+
+# One day, this will contain all the hypervisor's source but only once it can
+# all be built against the fake arch for unit tests. Utilities that are shared
+# e.g. with VM used in the VM tests have their own targets to facilitate
+# sharing.
+source_set("src_testable") {
+  sources = [
+    "abort.c",
+    "api.c",
+    "cpu.c",
+    "manifest.c",
+    "panic.c",
+    "spci_architected_message.c",
+    "string.c",
+    "vcpu.c",
+    "vm.c",
+  ]
+
+  deps = [
+    ":fdt",
+    ":fdt_handler",
+    ":memiter",
+    ":mm",
+    ":std",
+    "//src/arch/${plat_arch}/hypervisor",
+    plat_boot_flow,
+    plat_console,
+    plat_iommu,
+  ]
+
+  if (is_debug) {
+    deps += [ ":dlog" ]
+  }
+}
+
+source_set("layout") {
+  sources = [
+    "layout.c",
+  ]
+}
+
+source_set("mm") {
+  sources = [
+    "mm.c",
+    "mpool.c",
+  ]
+}
+
+# Standard library functions.
+source_set("std") {
+  sources = [
+    "std.c",
+  ]
+
+  deps = [
+    "//src/arch/${plat_arch}:std",
+  ]
+}
+
+# Debug code that is not specific to a certain image so can be shared.
+source_set("dlog") {
+  sources = [
+    "dlog.c",
+  ]
+
+  deps = [
+    ":std",
+    plat_console,
+  ]
+}
+
+source_set("fdt_handler") {
+  sources = [
+    "fdt_handler.c",
+  ]
+  deps = [
+    ":fdt",
+  ]
+
+  if (is_debug) {
+    deps += [ ":dlog" ]
+  }
+}
+
+# Flattened Device Tree (FDT) utilities.
+source_set("fdt") {
+  sources = [
+    "fdt.c",
+  ]
+
+  deps = [
+    ":std",
+  ]
+
+  if (is_debug) {
+    deps += [ ":dlog" ]
+  }
+}
+
+source_set("memiter") {
+  sources = [
+    "memiter.c",
+  ]
+}
+
+source_set("panic") {
+  sources = [
+    "panic.c",
+  ]
+}
+
+executable("unit_tests") {
+  testonly = true
+  sources = [
+    "api_test.cc",
+    "fdt_handler_test.cc",
+    "fdt_test.cc",
+    "manifest_test.cc",
+    "mm_test.cc",
+    "mpool_test.cc",
+    "string_test.cc",
+    "vm_test.cc",
+  ]
+  sources += [ "layout_fake.c" ]
+  cflags_cc = [
+    "-Wno-c99-extensions",
+    "-Wno-nested-anon-types",
+  ]
+  deps = [
+    ":src_testable",
+    "//third_party/googletest:gtest_main",
+  ]
+}
diff --git a/src/abort.c b/src/abort.c
new file mode 100644
index 0000000..8d9d723
--- /dev/null
+++ b/src/abort.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/abort.h"
+
+#include "hf/dlog.h"
+
+/**
+ * Causes execution to halt and prevent progress of the current and less
+ * privileged software components. This should be triggered when a
+ * non-recoverable event is identified which leaves the system in an
+ * inconsistent state.
+ *
+ * TODO: Should this also reset the system?
+ */
+noreturn void abort(void)
+{
+	/* TODO: Block all CPUs. */
+	for (;;) {
+		/* Prevent loop being optimized away. */
+		__asm__ volatile("nop");
+	}
+}
diff --git a/src/api.c b/src/api.c
new file mode 100644
index 0000000..d8ced97
--- /dev/null
+++ b/src/api.c
@@ -0,0 +1,1458 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/api.h"
+
+#include "hf/arch/cpu.h"
+#include "hf/arch/timer.h"
+
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/mm.h"
+#include "hf/plat/console.h"
+#include "hf/spci_internal.h"
+#include "hf/spinlock.h"
+#include "hf/static_assert.h"
+#include "hf/std.h"
+#include "hf/vm.h"
+
+#include "vmapi/hf/call.h"
+#include "vmapi/hf/spci.h"
+
+/*
+ * To eliminate the risk of deadlocks, we define a partial order for the
+ * acquisition of locks held concurrently by the same physical CPU. Our current
+ * ordering requirements are as follows:
+ *
+ * vm::lock -> vcpu::lock -> mm_stage1_lock -> dlog sl
+ *
+ * Locks of the same kind require the lock of lowest address to be locked first,
+ * see `sl_lock_both()`.
+ */
+
+static_assert(HF_MAILBOX_SIZE == PAGE_SIZE,
+	      "Currently, a page is mapped for the send and receive buffers so "
+	      "the maximum request is the size of a page.");
+
+static struct mpool api_page_pool;
+
+/**
+ * Initialises the API page pool by taking ownership of the contents of the
+ * given page pool.
+ */
+void api_init(struct mpool *ppool)
+{
+	mpool_init_from(&api_page_pool, ppool);
+}
+
+/**
+ * Switches the physical CPU back to the corresponding vCPU of the primary VM.
+ *
+ * This triggers the scheduling logic to run. Run in the context of secondary VM
+ * to cause SPCI_RUN to return and the primary VM to regain control of the CPU.
+ */
+static struct vcpu *api_switch_to_primary(struct vcpu *current,
+					  struct spci_value primary_ret,
+					  enum vcpu_state secondary_state)
+{
+	struct vm *primary = vm_find(HF_PRIMARY_VM_ID);
+	struct vcpu *next = vm_get_vcpu(primary, cpu_index(current->cpu));
+
+	/*
+	 * If the secondary is blocked but has a timer running, sleep until the
+	 * timer fires rather than indefinitely.
+	 */
+	switch (primary_ret.func) {
+	case HF_SPCI_RUN_WAIT_FOR_INTERRUPT:
+	case SPCI_MSG_WAIT_32: {
+		if (arch_timer_enabled_current()) {
+			uint64_t remaining_ns =
+				arch_timer_remaining_ns_current();
+
+			if (remaining_ns == 0) {
+				/*
+				 * Timer is pending, so the current vCPU should
+				 * be run again right away.
+				 */
+				primary_ret.func = SPCI_INTERRUPT_32;
+				/*
+				 * primary_ret.arg1 should already be set to the
+				 * current VM ID and vCPU ID.
+				 */
+				primary_ret.arg2 = 0;
+			} else {
+				primary_ret.arg2 = remaining_ns;
+			}
+		} else {
+			primary_ret.arg2 = SPCI_SLEEP_INDEFINITE;
+		}
+		break;
+	}
+
+	default:
+		/* Do nothing. */
+		break;
+	}
+
+	/* Set the return value for the primary VM's call to HF_VCPU_RUN. */
+	arch_regs_set_retval(&next->regs, primary_ret);
+
+	/* Mark the current vCPU as waiting. */
+	sl_lock(&current->lock);
+	current->state = secondary_state;
+	sl_unlock(&current->lock);
+
+	return next;
+}
+
+/**
+ * Returns to the primary VM and signals that the vCPU still has work to do so.
+ */
+struct vcpu *api_preempt(struct vcpu *current)
+{
+	struct spci_value ret = {
+		.func = SPCI_INTERRUPT_32,
+		.arg1 = spci_vm_vcpu(current->vm->id, vcpu_index(current)),
+	};
+
+	return api_switch_to_primary(current, ret, VCPU_STATE_READY);
+}
+
+/**
+ * Puts the current vCPU in wait for interrupt mode, and returns to the primary
+ * VM.
+ */
+struct vcpu *api_wait_for_interrupt(struct vcpu *current)
+{
+	struct spci_value ret = {
+		.func = HF_SPCI_RUN_WAIT_FOR_INTERRUPT,
+		.arg1 = spci_vm_vcpu(current->vm->id, vcpu_index(current)),
+	};
+
+	return api_switch_to_primary(current, ret,
+				     VCPU_STATE_BLOCKED_INTERRUPT);
+}
+
+/**
+ * Puts the current vCPU in off mode, and returns to the primary VM.
+ */
+struct vcpu *api_vcpu_off(struct vcpu *current)
+{
+	struct spci_value ret = {
+		.func = HF_SPCI_RUN_WAIT_FOR_INTERRUPT,
+		.arg1 = spci_vm_vcpu(current->vm->id, vcpu_index(current)),
+	};
+
+	/*
+	 * Disable the timer, so the scheduler doesn't get told to call back
+	 * based on it.
+	 */
+	arch_timer_disable_current();
+
+	return api_switch_to_primary(current, ret, VCPU_STATE_OFF);
+}
+
+/**
+ * Returns to the primary VM to allow this CPU to be used for other tasks as the
+ * vCPU does not have work to do at this moment. The current vCPU is marked as
+ * ready to be scheduled again.
+ */
+void api_yield(struct vcpu *current, struct vcpu **next)
+{
+	struct spci_value primary_ret = {
+		.func = SPCI_YIELD_32,
+		.arg1 = spci_vm_vcpu(current->vm->id, vcpu_index(current)),
+	};
+
+	if (current->vm->id == HF_PRIMARY_VM_ID) {
+		/* NOOP on the primary as it makes the scheduling decisions. */
+		return;
+	}
+
+	*next = api_switch_to_primary(current, primary_ret, VCPU_STATE_READY);
+}
+
+/**
+ * Switches to the primary so that it can switch to the target, or kick it if it
+ * is already running on a different physical CPU.
+ */
+struct vcpu *api_wake_up(struct vcpu *current, struct vcpu *target_vcpu)
+{
+	struct spci_value ret = {
+		.func = HF_SPCI_RUN_WAKE_UP,
+		.arg1 = spci_vm_vcpu(target_vcpu->vm->id,
+				     vcpu_index(target_vcpu)),
+	};
+	return api_switch_to_primary(current, ret, VCPU_STATE_READY);
+}
+
+/**
+ * Aborts the vCPU and triggers its VM to abort fully.
+ */
+struct vcpu *api_abort(struct vcpu *current)
+{
+	struct spci_value ret = spci_error(SPCI_ABORTED);
+
+	dlog("Aborting VM %u vCPU %u\n", current->vm->id, vcpu_index(current));
+
+	if (current->vm->id == HF_PRIMARY_VM_ID) {
+		/* TODO: what to do when the primary aborts? */
+		for (;;) {
+			/* Do nothing. */
+		}
+	}
+
+	atomic_store_explicit(&current->vm->aborting, true,
+			      memory_order_relaxed);
+
+	/* TODO: free resources once all vCPUs abort. */
+
+	return api_switch_to_primary(current, ret, VCPU_STATE_ABORTED);
+}
+
+/**
+ * Returns the ID of the VM.
+ */
+struct spci_value api_spci_id_get(const struct vcpu *current)
+{
+	return (struct spci_value){.func = SPCI_SUCCESS_32,
+				   .arg2 = current->vm->id};
+}
+
+/**
+ * Returns the number of VMs configured to run.
+ */
+spci_vm_count_t api_vm_get_count(void)
+{
+	return vm_get_count();
+}
+
+/**
+ * Returns the number of vCPUs configured in the given VM, or 0 if there is no
+ * such VM or the caller is not the primary VM.
+ */
+spci_vcpu_count_t api_vcpu_get_count(spci_vm_id_t vm_id,
+				     const struct vcpu *current)
+{
+	struct vm *vm;
+
+	/* Only the primary VM needs to know about vCPUs for scheduling. */
+	if (current->vm->id != HF_PRIMARY_VM_ID) {
+		return 0;
+	}
+
+	vm = vm_find(vm_id);
+	if (vm == NULL) {
+		return 0;
+	}
+
+	return vm->vcpu_count;
+}
+
+/**
+ * This function is called by the architecture-specific context switching
+ * function to indicate that register state for the given vCPU has been saved
+ * and can therefore be used by other pCPUs.
+ */
+void api_regs_state_saved(struct vcpu *vcpu)
+{
+	sl_lock(&vcpu->lock);
+	vcpu->regs_available = true;
+	sl_unlock(&vcpu->lock);
+}
+
+/**
+ * Retrieves the next waiter and removes it from the wait list if the VM's
+ * mailbox is in a writable state.
+ */
+static struct wait_entry *api_fetch_waiter(struct vm_locked locked_vm)
+{
+	struct wait_entry *entry;
+	struct vm *vm = locked_vm.vm;
+
+	if (vm->mailbox.state != MAILBOX_STATE_EMPTY ||
+	    vm->mailbox.recv == NULL || list_empty(&vm->mailbox.waiter_list)) {
+		/* The mailbox is not writable or there are no waiters. */
+		return NULL;
+	}
+
+	/* Remove waiter from the wait list. */
+	entry = CONTAINER_OF(vm->mailbox.waiter_list.next, struct wait_entry,
+			     wait_links);
+	list_remove(&entry->wait_links);
+	return entry;
+}
+
+/**
+ * Assuming that the arguments have already been checked by the caller, injects
+ * a virtual interrupt of the given ID into the given target vCPU. This doesn't
+ * cause the vCPU to actually be run immediately; it will be taken when the vCPU
+ * is next run, which is up to the scheduler.
+ *
+ * Returns:
+ *  - 0 on success if no further action is needed.
+ *  - 1 if it was called by the primary VM and the primary VM now needs to wake
+ *    up or kick the target vCPU.
+ */
+static int64_t internal_interrupt_inject(struct vcpu *target_vcpu,
+					 uint32_t intid, struct vcpu *current,
+					 struct vcpu **next)
+{
+	uint32_t intid_index = intid / INTERRUPT_REGISTER_BITS;
+	uint32_t intid_mask = 1U << (intid % INTERRUPT_REGISTER_BITS);
+	int64_t ret = 0;
+
+	sl_lock(&target_vcpu->lock);
+
+	/*
+	 * We only need to change state and (maybe) trigger a virtual IRQ if it
+	 * is enabled and was not previously pending. Otherwise we can skip
+	 * everything except setting the pending bit.
+	 *
+	 * If you change this logic make sure to update the need_vm_lock logic
+	 * above to match.
+	 */
+	if (!(target_vcpu->interrupts.interrupt_enabled[intid_index] &
+	      ~target_vcpu->interrupts.interrupt_pending[intid_index] &
+	      intid_mask)) {
+		goto out;
+	}
+
+	/* Increment the count. */
+	target_vcpu->interrupts.enabled_and_pending_count++;
+
+	/*
+	 * Only need to update state if there was not already an
+	 * interrupt enabled and pending.
+	 */
+	if (target_vcpu->interrupts.enabled_and_pending_count != 1) {
+		goto out;
+	}
+
+	if (current->vm->id == HF_PRIMARY_VM_ID) {
+		/*
+		 * If the call came from the primary VM, let it know that it
+		 * should run or kick the target vCPU.
+		 */
+		ret = 1;
+	} else if (current != target_vcpu && next != NULL) {
+		*next = api_wake_up(current, target_vcpu);
+	}
+
+out:
+	/* Either way, make it pending. */
+	target_vcpu->interrupts.interrupt_pending[intid_index] |= intid_mask;
+
+	sl_unlock(&target_vcpu->lock);
+
+	return ret;
+}
+
+/**
+ * Constructs an SPCI_MSG_SEND value to return from a successful SPCI_MSG_POLL
+ * or SPCI_MSG_WAIT call.
+ */
+static struct spci_value spci_msg_recv_return(const struct vm *receiver)
+{
+	return (struct spci_value){
+		.func = SPCI_MSG_SEND_32,
+		.arg1 = (receiver->mailbox.recv_sender << 16) | receiver->id,
+		.arg3 = receiver->mailbox.recv_size,
+		.arg4 = receiver->mailbox.recv_attributes};
+}
+
+/**
+ * Prepares the vCPU to run by updating its state and fetching whether a return
+ * value needs to be forced onto the vCPU.
+ */
+static bool api_vcpu_prepare_run(const struct vcpu *current, struct vcpu *vcpu,
+				 struct spci_value *run_ret)
+{
+	bool need_vm_lock;
+	bool ret;
+
+	/*
+	 * Check that the registers are available so that the vCPU can be run.
+	 *
+	 * The VM lock is not needed in the common case so it must only be taken
+	 * when it is going to be needed. This ensures there are no inter-vCPU
+	 * dependencies in the common run case meaning the sensitive context
+	 * switch performance is consistent.
+	 */
+	sl_lock(&vcpu->lock);
+
+	/* The VM needs to be locked to deliver mailbox messages. */
+	need_vm_lock = vcpu->state == VCPU_STATE_BLOCKED_MAILBOX;
+	if (need_vm_lock) {
+		sl_unlock(&vcpu->lock);
+		sl_lock(&vcpu->vm->lock);
+		sl_lock(&vcpu->lock);
+	}
+
+	/*
+	 * If the vCPU is already running somewhere then we can't run it here
+	 * simultaneously. While it is actually running then the state should be
+	 * `VCPU_STATE_RUNNING` and `regs_available` should be false. Once it
+	 * stops running but while Hafnium is in the process of switching back
+	 * to the primary there will be a brief period while the state has been
+	 * updated but `regs_available` is still false (until
+	 * `api_regs_state_saved` is called). We can't start running it again
+	 * until this has finished, so count this state as still running for the
+	 * purposes of this check.
+	 */
+	if (vcpu->state == VCPU_STATE_RUNNING || !vcpu->regs_available) {
+		/*
+		 * vCPU is running on another pCPU.
+		 *
+		 * It's okay not to return the sleep duration here because the
+		 * other physical CPU that is currently running this vCPU will
+		 * return the sleep duration if needed.
+		 */
+		*run_ret = spci_error(SPCI_BUSY);
+		ret = false;
+		goto out;
+	}
+
+	if (atomic_load_explicit(&vcpu->vm->aborting, memory_order_relaxed)) {
+		if (vcpu->state != VCPU_STATE_ABORTED) {
+			dlog("Aborting VM %u vCPU %u\n", vcpu->vm->id,
+			     vcpu_index(vcpu));
+			vcpu->state = VCPU_STATE_ABORTED;
+		}
+		ret = false;
+		goto out;
+	}
+
+	switch (vcpu->state) {
+	case VCPU_STATE_RUNNING:
+	case VCPU_STATE_OFF:
+	case VCPU_STATE_ABORTED:
+		ret = false;
+		goto out;
+
+	case VCPU_STATE_BLOCKED_MAILBOX:
+		/*
+		 * A pending message allows the vCPU to run so the message can
+		 * be delivered directly.
+		 */
+		if (vcpu->vm->mailbox.state == MAILBOX_STATE_RECEIVED) {
+			arch_regs_set_retval(&vcpu->regs,
+					     spci_msg_recv_return(vcpu->vm));
+			vcpu->vm->mailbox.state = MAILBOX_STATE_READ;
+			break;
+		}
+		/* Fall through. */
+	case VCPU_STATE_BLOCKED_INTERRUPT:
+		/* Allow virtual interrupts to be delivered. */
+		if (vcpu->interrupts.enabled_and_pending_count > 0) {
+			break;
+		}
+
+		if (arch_timer_enabled(&vcpu->regs)) {
+			uint64_t timer_remaining_ns =
+				arch_timer_remaining_ns(&vcpu->regs);
+
+			/*
+			 * The timer expired so allow the interrupt to be
+			 * delivered.
+			 */
+			if (timer_remaining_ns == 0) {
+				break;
+			}
+
+			/*
+			 * The vCPU is not ready to run, return the appropriate
+			 * code to the primary which called vcpu_run.
+			 */
+			run_ret->func =
+				vcpu->state == VCPU_STATE_BLOCKED_MAILBOX
+					? SPCI_MSG_WAIT_32
+					: HF_SPCI_RUN_WAIT_FOR_INTERRUPT;
+			run_ret->arg1 =
+				spci_vm_vcpu(vcpu->vm->id, vcpu_index(vcpu));
+			run_ret->arg2 = timer_remaining_ns;
+		}
+
+		ret = false;
+		goto out;
+
+	case VCPU_STATE_READY:
+		break;
+	}
+
+	/* It has been decided that the vCPU should be run. */
+	vcpu->cpu = current->cpu;
+	vcpu->state = VCPU_STATE_RUNNING;
+
+	/*
+	 * Mark the registers as unavailable now that we're about to reflect
+	 * them onto the real registers. This will also prevent another physical
+	 * CPU from trying to read these registers.
+	 */
+	vcpu->regs_available = false;
+
+	ret = true;
+
+out:
+	sl_unlock(&vcpu->lock);
+	if (need_vm_lock) {
+		sl_unlock(&vcpu->vm->lock);
+	}
+
+	return ret;
+}
+
+struct spci_value api_spci_run(spci_vm_id_t vm_id, spci_vcpu_index_t vcpu_idx,
+			       const struct vcpu *current, struct vcpu **next)
+{
+	struct vm *vm;
+	struct vcpu *vcpu;
+	struct spci_value ret = spci_error(SPCI_INVALID_PARAMETERS);
+
+	/* Only the primary VM can switch vCPUs. */
+	if (current->vm->id != HF_PRIMARY_VM_ID) {
+		ret.arg2 = SPCI_DENIED;
+		goto out;
+	}
+
+	/* Only secondary VM vCPUs can be run. */
+	if (vm_id == HF_PRIMARY_VM_ID) {
+		goto out;
+	}
+
+	/* The requested VM must exist. */
+	vm = vm_find(vm_id);
+	if (vm == NULL) {
+		goto out;
+	}
+
+	/* The requested vCPU must exist. */
+	if (vcpu_idx >= vm->vcpu_count) {
+		goto out;
+	}
+
+	/* Update state if allowed. */
+	vcpu = vm_get_vcpu(vm, vcpu_idx);
+	if (!api_vcpu_prepare_run(current, vcpu, &ret)) {
+		goto out;
+	}
+
+	/*
+	 * Inject timer interrupt if timer has expired. It's safe to access
+	 * vcpu->regs here because api_vcpu_prepare_run already made sure that
+	 * regs_available was true (and then set it to false) before returning
+	 * true.
+	 */
+	if (arch_timer_pending(&vcpu->regs)) {
+		/* Make virtual timer interrupt pending. */
+		internal_interrupt_inject(vcpu, HF_VIRTUAL_TIMER_INTID, vcpu,
+					  NULL);
+
+		/*
+		 * Set the mask bit so the hardware interrupt doesn't fire
+		 * again. Ideally we wouldn't do this because it affects what
+		 * the secondary vCPU sees, but if we don't then we end up with
+		 * a loop of the interrupt firing each time we try to return to
+		 * the secondary vCPU.
+		 */
+		arch_timer_mask(&vcpu->regs);
+	}
+
+	/* Switch to the vCPU. */
+	*next = vcpu;
+
+	/*
+	 * Set a placeholder return code to the scheduler. This will be
+	 * overwritten when the switch back to the primary occurs.
+	 */
+	ret.func = SPCI_INTERRUPT_32;
+	ret.arg1 = spci_vm_vcpu(vm_id, vcpu_idx);
+	ret.arg2 = 0;
+
+out:
+	return ret;
+}
+
+/**
+ * Check that the mode indicates memory that is valid, owned and exclusive.
+ */
+static bool api_mode_valid_owned_and_exclusive(uint32_t mode)
+{
+	return (mode & (MM_MODE_D | MM_MODE_INVALID | MM_MODE_UNOWNED |
+			MM_MODE_SHARED)) == 0;
+}
+
+/**
+ * Determines the value to be returned by api_vm_configure and spci_rx_release
+ * after they've succeeded. If a secondary VM is running and there are waiters,
+ * it also switches back to the primary VM for it to wake waiters up.
+ */
+static struct spci_value api_waiter_result(struct vm_locked locked_vm,
+					   struct vcpu *current,
+					   struct vcpu **next)
+{
+	struct vm *vm = locked_vm.vm;
+
+	if (list_empty(&vm->mailbox.waiter_list)) {
+		/* No waiters, nothing else to do. */
+		return (struct spci_value){.func = SPCI_SUCCESS_32};
+	}
+
+	if (vm->id == HF_PRIMARY_VM_ID) {
+		/* The caller is the primary VM. Tell it to wake up waiters. */
+		return (struct spci_value){.func = SPCI_RX_RELEASE_32};
+	}
+
+	/*
+	 * Switch back to the primary VM, informing it that there are waiters
+	 * that need to be notified.
+	 */
+	*next = api_switch_to_primary(
+		current, (struct spci_value){.func = SPCI_RX_RELEASE_32},
+		VCPU_STATE_READY);
+
+	return (struct spci_value){.func = SPCI_SUCCESS_32};
+}
+
+/**
+ * Configures the hypervisor's stage-1 view of the send and receive pages. The
+ * stage-1 page tables must be locked so memory cannot be taken by another core
+ * which could result in this transaction being unable to roll back in the case
+ * of an error.
+ */
+static bool api_vm_configure_stage1(struct vm_locked vm_locked,
+				    paddr_t pa_send_begin, paddr_t pa_send_end,
+				    paddr_t pa_recv_begin, paddr_t pa_recv_end,
+				    struct mpool *local_page_pool)
+{
+	bool ret;
+	struct mm_stage1_locked mm_stage1_locked = mm_lock_stage1();
+
+	/* Map the send page as read-only in the hypervisor address space. */
+	vm_locked.vm->mailbox.send =
+		mm_identity_map(mm_stage1_locked, pa_send_begin, pa_send_end,
+				MM_MODE_R, local_page_pool);
+	if (!vm_locked.vm->mailbox.send) {
+		/* TODO: partial defrag of failed range. */
+		/* Recover any memory consumed in failed mapping. */
+		mm_defrag(mm_stage1_locked, local_page_pool);
+		goto fail;
+	}
+
+	/*
+	 * Map the receive page as writable in the hypervisor address space. On
+	 * failure, unmap the send page before returning.
+	 */
+	vm_locked.vm->mailbox.recv =
+		mm_identity_map(mm_stage1_locked, pa_recv_begin, pa_recv_end,
+				MM_MODE_W, local_page_pool);
+	if (!vm_locked.vm->mailbox.recv) {
+		/* TODO: partial defrag of failed range. */
+		/* Recover any memory consumed in failed mapping. */
+		mm_defrag(mm_stage1_locked, local_page_pool);
+		goto fail_undo_send;
+	}
+
+	ret = true;
+	goto out;
+
+	/*
+	 * The following mappings will not require more memory than is available
+	 * in the local pool.
+	 */
+fail_undo_send:
+	vm_locked.vm->mailbox.send = NULL;
+	CHECK(mm_unmap(mm_stage1_locked, pa_send_begin, pa_send_end,
+		       local_page_pool));
+
+fail:
+	ret = false;
+
+out:
+	mm_unlock_stage1(&mm_stage1_locked);
+
+	return ret;
+}
+
+/**
+ * Configures the send and receive pages in the VM stage-2 and hypervisor
+ * stage-1 page tables. Locking of the page tables combined with a local memory
+ * pool ensures there will always be enough memory to recover from any errors
+ * that arise.
+ */
+static bool api_vm_configure_pages(struct vm_locked vm_locked,
+				   paddr_t pa_send_begin, paddr_t pa_send_end,
+				   uint32_t orig_send_mode,
+				   paddr_t pa_recv_begin, paddr_t pa_recv_end,
+				   uint32_t orig_recv_mode)
+{
+	bool ret;
+	struct mpool local_page_pool;
+
+	/*
+	 * Create a local pool so any freed memory can't be used by another
+	 * thread. This is to ensure the original mapping can be restored if any
+	 * stage of the process fails.
+	 */
+	mpool_init_with_fallback(&local_page_pool, &api_page_pool);
+
+	/* Take memory ownership away from the VM and mark as shared. */
+	if (!vm_identity_map(
+		    vm_locked, pa_send_begin, pa_send_end,
+		    MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R | MM_MODE_W,
+		    &local_page_pool, NULL)) {
+		goto fail;
+	}
+
+	if (!vm_identity_map(vm_locked, pa_recv_begin, pa_recv_end,
+			     MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R,
+			     &local_page_pool, NULL)) {
+		/* TODO: partial defrag of failed range. */
+		/* Recover any memory consumed in failed mapping. */
+		mm_vm_defrag(&vm_locked.vm->ptable, &local_page_pool);
+		goto fail_undo_send;
+	}
+
+	if (!api_vm_configure_stage1(vm_locked, pa_send_begin, pa_send_end,
+				     pa_recv_begin, pa_recv_end,
+				     &local_page_pool)) {
+		goto fail_undo_send_and_recv;
+	}
+
+	ret = true;
+	goto out;
+
+	/*
+	 * The following mappings will not require more memory than is available
+	 * in the local pool.
+	 */
+fail_undo_send_and_recv:
+	CHECK(vm_identity_map(vm_locked, pa_recv_begin, pa_recv_end,
+			      orig_recv_mode, &local_page_pool, NULL));
+
+fail_undo_send:
+	CHECK(vm_identity_map(vm_locked, pa_send_begin, pa_send_end,
+			      orig_send_mode, &local_page_pool, NULL));
+
+fail:
+	ret = false;
+
+out:
+	mpool_fini(&local_page_pool);
+
+	return ret;
+}
+
+/**
+ * Configures the VM to send/receive data through the specified pages. The pages
+ * must not be shared.
+ *
+ * Returns:
+ *  - SPCI_ERROR SPCI_INVALID_PARAMETERS if the given addresses are not properly
+ *    aligned or are the same.
+ *  - SPCI_ERROR SPCI_NO_MEMORY if the hypervisor was unable to map the buffers
+ *    due to insuffient page table memory.
+ *  - SPCI_ERROR SPCI_DENIED if the pages are already mapped or are not owned by
+ *    the caller.
+ *  - SPCI_SUCCESS on success if no further action is needed.
+ *  - SPCI_RX_RELEASE if it was called by the primary VM and the primary VM now
+ *    needs to wake up or kick waiters.
+ */
+struct spci_value api_spci_rxtx_map(ipaddr_t send, ipaddr_t recv,
+				    uint32_t page_count, struct vcpu *current,
+				    struct vcpu **next)
+{
+	struct vm *vm = current->vm;
+	struct vm_locked vm_locked;
+	paddr_t pa_send_begin;
+	paddr_t pa_send_end;
+	paddr_t pa_recv_begin;
+	paddr_t pa_recv_end;
+	uint32_t orig_send_mode;
+	uint32_t orig_recv_mode;
+	struct spci_value ret;
+
+	/* Hafnium only supports a fixed size of RX/TX buffers. */
+	if (page_count != HF_MAILBOX_SIZE / SPCI_PAGE_SIZE) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Fail if addresses are not page-aligned. */
+	if (!is_aligned(ipa_addr(send), PAGE_SIZE) ||
+	    !is_aligned(ipa_addr(recv), PAGE_SIZE)) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Convert to physical addresses. */
+	pa_send_begin = pa_from_ipa(send);
+	pa_send_end = pa_add(pa_send_begin, HF_MAILBOX_SIZE);
+
+	pa_recv_begin = pa_from_ipa(recv);
+	pa_recv_end = pa_add(pa_recv_begin, HF_MAILBOX_SIZE);
+
+	/* Fail if the same page is used for the send and receive pages. */
+	if (pa_addr(pa_send_begin) == pa_addr(pa_recv_begin)) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/*
+	 * The hypervisor's memory map must be locked for the duration of this
+	 * operation to ensure there will be sufficient memory to recover from
+	 * any failures.
+	 *
+	 * TODO: the scope of the can be reduced but will require restructuring
+	 *       to keep a single unlock point.
+	 */
+	vm_locked = vm_lock(vm);
+
+	/* We only allow these to be setup once. */
+	if (vm->mailbox.send || vm->mailbox.recv) {
+		ret = spci_error(SPCI_DENIED);
+		goto exit;
+	}
+
+	/*
+	 * Ensure the pages are valid, owned and exclusive to the VM and that
+	 * the VM has the required access to the memory.
+	 */
+	if (!mm_vm_get_mode(&vm->ptable, send, ipa_add(send, PAGE_SIZE),
+			    &orig_send_mode) ||
+	    !api_mode_valid_owned_and_exclusive(orig_send_mode) ||
+	    (orig_send_mode & MM_MODE_R) == 0 ||
+	    (orig_send_mode & MM_MODE_W) == 0) {
+		ret = spci_error(SPCI_DENIED);
+		goto exit;
+	}
+
+	if (!mm_vm_get_mode(&vm->ptable, recv, ipa_add(recv, PAGE_SIZE),
+			    &orig_recv_mode) ||
+	    !api_mode_valid_owned_and_exclusive(orig_recv_mode) ||
+	    (orig_recv_mode & MM_MODE_R) == 0) {
+		ret = spci_error(SPCI_DENIED);
+		goto exit;
+	}
+
+	if (!api_vm_configure_pages(vm_locked, pa_send_begin, pa_send_end,
+				    orig_send_mode, pa_recv_begin, pa_recv_end,
+				    orig_recv_mode)) {
+		ret = spci_error(SPCI_NO_MEMORY);
+		goto exit;
+	}
+
+	/* Tell caller about waiters, if any. */
+	ret = api_waiter_result(vm_locked, current, next);
+
+exit:
+	vm_unlock(&vm_locked);
+
+	return ret;
+}
+
+/**
+ * Checks whether the given `to` VM's mailbox is currently busy, and optionally
+ * registers the `from` VM to be notified when it becomes available.
+ */
+static bool msg_receiver_busy(struct vm_locked to, struct vm *from, bool notify)
+{
+	if (to.vm->mailbox.state != MAILBOX_STATE_EMPTY ||
+	    to.vm->mailbox.recv == NULL) {
+		/*
+		 * Fail if the receiver isn't currently ready to receive data,
+		 * setting up for notification if requested.
+		 */
+		if (notify) {
+			struct wait_entry *entry =
+				vm_get_wait_entry(from, to.vm->id);
+
+			/* Append waiter only if it's not there yet. */
+			if (list_empty(&entry->wait_links)) {
+				list_append(&to.vm->mailbox.waiter_list,
+					    &entry->wait_links);
+			}
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * Notifies the `to` VM about the message currently in its mailbox, possibly
+ * with the help of the primary VM.
+ */
+static void deliver_msg(struct vm_locked to, spci_vm_id_t from_id,
+			struct vcpu *current, struct vcpu **next)
+{
+	struct spci_value primary_ret = {
+		.func = SPCI_MSG_SEND_32,
+		.arg1 = ((uint32_t)from_id << 16) | to.vm->id,
+	};
+
+	/* Messages for the primary VM are delivered directly. */
+	if (to.vm->id == HF_PRIMARY_VM_ID) {
+		/*
+		 * Only tell the primary VM the size if the message is for it,
+		 * to avoid leaking data about messages for other VMs.
+		 */
+		primary_ret.arg3 = to.vm->mailbox.recv_size;
+		primary_ret.arg4 = to.vm->mailbox.recv_attributes;
+
+		to.vm->mailbox.state = MAILBOX_STATE_READ;
+		*next = api_switch_to_primary(current, primary_ret,
+					      VCPU_STATE_READY);
+		return;
+	}
+
+	to.vm->mailbox.state = MAILBOX_STATE_RECEIVED;
+
+	/* Return to the primary VM directly or with a switch. */
+	if (from_id != HF_PRIMARY_VM_ID) {
+		*next = api_switch_to_primary(current, primary_ret,
+					      VCPU_STATE_READY);
+	}
+}
+
+/**
+ * Copies data from the sender's send buffer to the recipient's receive buffer
+ * and notifies the recipient.
+ *
+ * If the recipient's receive buffer is busy, it can optionally register the
+ * caller to be notified when the recipient's receive buffer becomes available.
+ */
+struct spci_value api_spci_msg_send(spci_vm_id_t sender_vm_id,
+				    spci_vm_id_t receiver_vm_id, uint32_t size,
+				    uint32_t attributes, struct vcpu *current,
+				    struct vcpu **next)
+{
+	struct vm *from = current->vm;
+	struct vm *to;
+
+	struct two_vm_locked vm_to_from_lock;
+
+	const void *from_msg;
+
+	struct spci_value ret;
+	bool notify = (attributes & SPCI_MSG_SEND_NOTIFY_MASK) ==
+		      SPCI_MSG_SEND_NOTIFY;
+
+	/* Ensure sender VM ID corresponds to the current VM. */
+	if (sender_vm_id != from->id) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Disallow reflexive requests as this suggests an error in the VM. */
+	if (receiver_vm_id == from->id) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Limit the size of transfer. */
+	if (size > SPCI_MSG_PAYLOAD_MAX) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Ensure the receiver VM exists. */
+	to = vm_find(receiver_vm_id);
+	if (to == NULL) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/*
+	 * Check that the sender has configured its send buffer. If the tx
+	 * mailbox at from_msg is configured (i.e. from_msg != NULL) then it can
+	 * be safely accessed after releasing the lock since the tx mailbox
+	 * address can only be configured once.
+	 */
+	sl_lock(&from->lock);
+	from_msg = from->mailbox.send;
+	sl_unlock(&from->lock);
+
+	if (from_msg == NULL) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/*
+	 * Hafnium needs to hold the lock on <to> before the mailbox state is
+	 * checked. The lock on <to> must be held until the information is
+	 * copied to <to> Rx buffer. Since in
+	 * spci_msg_handle_architected_message we may call api_spci_share_memory
+	 * which must hold the <from> lock, we must hold the <from> lock at this
+	 * point to prevent a deadlock scenario.
+	 */
+	vm_to_from_lock = vm_lock_both(to, from);
+
+	if (msg_receiver_busy(vm_to_from_lock.vm1, from, notify)) {
+		ret = spci_error(SPCI_BUSY);
+		goto out;
+	}
+
+	/* Handle legacy memory sharing messages. */
+	if ((attributes & SPCI_MSG_SEND_LEGACY_MEMORY_MASK) != 0) {
+		/*
+		 * Buffer holding the internal copy of the shared memory
+		 * regions.
+		 */
+		uint8_t *message_replica = cpu_get_buffer(current->cpu->id);
+		uint32_t message_buffer_size =
+			cpu_get_buffer_size(current->cpu->id);
+
+		if (size > message_buffer_size) {
+			ret = spci_error(SPCI_INVALID_PARAMETERS);
+			goto out;
+		}
+
+		/* Copy the architected message into the internal buffer. */
+		memcpy_s(message_replica, message_buffer_size, from_msg, size);
+
+		/*
+		 * Note that architected_message_replica is passed as the third
+		 * parameter to spci_msg_handle_architected_message. The
+		 * execution flow commencing at
+		 * spci_msg_handle_architected_message will make several
+		 * accesses to fields in architected_message_replica. The memory
+		 * area architected_message_replica must be exclusively owned by
+		 * Hafnium so that TOCTOU issues do not arise.
+		 */
+		ret = spci_msg_handle_architected_message(
+			vm_to_from_lock.vm1, vm_to_from_lock.vm2,
+			(struct spci_memory_region *)message_replica, size,
+			attributes, &api_page_pool);
+
+		if (ret.func != SPCI_SUCCESS_32) {
+			goto out;
+		}
+	} else {
+		/* Copy data. */
+		memcpy_s(to->mailbox.recv, SPCI_MSG_PAYLOAD_MAX, from_msg,
+			 size);
+		to->mailbox.recv_size = size;
+		to->mailbox.recv_sender = sender_vm_id;
+		to->mailbox.recv_attributes = 0;
+		ret = (struct spci_value){.func = SPCI_SUCCESS_32};
+	}
+
+	deliver_msg(vm_to_from_lock.vm1, sender_vm_id, current, next);
+
+out:
+	vm_unlock(&vm_to_from_lock.vm1);
+	vm_unlock(&vm_to_from_lock.vm2);
+
+	return ret;
+}
+
+/**
+ * Checks whether the vCPU's attempt to block for a message has already been
+ * interrupted or whether it is allowed to block.
+ */
+bool api_spci_msg_recv_block_interrupted(struct vcpu *current)
+{
+	bool interrupted;
+
+	sl_lock(&current->lock);
+
+	/*
+	 * Don't block if there are enabled and pending interrupts, to match
+	 * behaviour of wait_for_interrupt.
+	 */
+	interrupted = (current->interrupts.enabled_and_pending_count > 0);
+
+	sl_unlock(&current->lock);
+
+	return interrupted;
+}
+
+/**
+ * Receives a message from the mailbox. If one isn't available, this function
+ * can optionally block the caller until one becomes available.
+ *
+ * No new messages can be received until the mailbox has been cleared.
+ */
+struct spci_value api_spci_msg_recv(bool block, struct vcpu *current,
+				    struct vcpu **next)
+{
+	struct vm *vm = current->vm;
+	struct spci_value return_code;
+
+	/*
+	 * The primary VM will receive messages as a status code from running
+	 * vCPUs and must not call this function.
+	 */
+	if (vm->id == HF_PRIMARY_VM_ID) {
+		return spci_error(SPCI_NOT_SUPPORTED);
+	}
+
+	sl_lock(&vm->lock);
+
+	/* Return pending messages without blocking. */
+	if (vm->mailbox.state == MAILBOX_STATE_RECEIVED) {
+		vm->mailbox.state = MAILBOX_STATE_READ;
+		return_code = spci_msg_recv_return(vm);
+		goto out;
+	}
+
+	/* No pending message so fail if not allowed to block. */
+	if (!block) {
+		return_code = spci_error(SPCI_RETRY);
+		goto out;
+	}
+
+	/*
+	 * From this point onward this call can only be interrupted or a message
+	 * received. If a message is received the return value will be set at
+	 * that time to SPCI_SUCCESS.
+	 */
+	return_code = spci_error(SPCI_INTERRUPTED);
+	if (api_spci_msg_recv_block_interrupted(current)) {
+		goto out;
+	}
+
+	/* Switch back to primary VM to block. */
+	{
+		struct spci_value run_return = {
+			.func = SPCI_MSG_WAIT_32,
+			.arg1 = spci_vm_vcpu(vm->id, vcpu_index(current)),
+		};
+
+		*next = api_switch_to_primary(current, run_return,
+					      VCPU_STATE_BLOCKED_MAILBOX);
+	}
+out:
+	sl_unlock(&vm->lock);
+
+	return return_code;
+}
+
+/**
+ * Retrieves the next VM whose mailbox became writable. For a VM to be notified
+ * by this function, the caller must have called api_mailbox_send before with
+ * the notify argument set to true, and this call must have failed because the
+ * mailbox was not available.
+ *
+ * It should be called repeatedly to retrieve a list of VMs.
+ *
+ * Returns -1 if no VM became writable, or the id of the VM whose mailbox
+ * became writable.
+ */
+int64_t api_mailbox_writable_get(const struct vcpu *current)
+{
+	struct vm *vm = current->vm;
+	struct wait_entry *entry;
+	int64_t ret;
+
+	sl_lock(&vm->lock);
+	if (list_empty(&vm->mailbox.ready_list)) {
+		ret = -1;
+		goto exit;
+	}
+
+	entry = CONTAINER_OF(vm->mailbox.ready_list.next, struct wait_entry,
+			     ready_links);
+	list_remove(&entry->ready_links);
+	ret = vm_id_for_wait_entry(vm, entry);
+
+exit:
+	sl_unlock(&vm->lock);
+	return ret;
+}
+
+/**
+ * Retrieves the next VM waiting to be notified that the mailbox of the
+ * specified VM became writable. Only primary VMs are allowed to call this.
+ *
+ * Returns -1 on failure or if there are no waiters; the VM id of the next
+ * waiter otherwise.
+ */
+int64_t api_mailbox_waiter_get(spci_vm_id_t vm_id, const struct vcpu *current)
+{
+	struct vm *vm;
+	struct vm_locked locked;
+	struct wait_entry *entry;
+	struct vm *waiting_vm;
+
+	/* Only primary VMs are allowed to call this function. */
+	if (current->vm->id != HF_PRIMARY_VM_ID) {
+		return -1;
+	}
+
+	vm = vm_find(vm_id);
+	if (vm == NULL) {
+		return -1;
+	}
+
+	/* Check if there are outstanding notifications from given VM. */
+	locked = vm_lock(vm);
+	entry = api_fetch_waiter(locked);
+	vm_unlock(&locked);
+
+	if (entry == NULL) {
+		return -1;
+	}
+
+	/* Enqueue notification to waiting VM. */
+	waiting_vm = entry->waiting_vm;
+
+	sl_lock(&waiting_vm->lock);
+	if (list_empty(&entry->ready_links)) {
+		list_append(&waiting_vm->mailbox.ready_list,
+			    &entry->ready_links);
+	}
+	sl_unlock(&waiting_vm->lock);
+
+	return waiting_vm->id;
+}
+
+/**
+ * Releases the caller's mailbox so that a new message can be received. The
+ * caller must have copied out all data they wish to preserve as new messages
+ * will overwrite the old and will arrive asynchronously.
+ *
+ * Returns:
+ *  - SPCI_ERROR SPCI_DENIED on failure, if the mailbox hasn't been read.
+ *  - SPCI_SUCCESS on success if no further action is needed.
+ *  - SPCI_RX_RELEASE if it was called by the primary VM and the primary VM now
+ *    needs to wake up or kick waiters. Waiters should be retrieved by calling
+ *    hf_mailbox_waiter_get.
+ */
+struct spci_value api_spci_rx_release(struct vcpu *current, struct vcpu **next)
+{
+	struct vm *vm = current->vm;
+	struct vm_locked locked;
+	struct spci_value ret;
+
+	locked = vm_lock(vm);
+	switch (vm->mailbox.state) {
+	case MAILBOX_STATE_EMPTY:
+	case MAILBOX_STATE_RECEIVED:
+		ret = spci_error(SPCI_DENIED);
+		break;
+
+	case MAILBOX_STATE_READ:
+		ret = api_waiter_result(locked, current, next);
+		vm->mailbox.state = MAILBOX_STATE_EMPTY;
+		break;
+	}
+	vm_unlock(&locked);
+
+	return ret;
+}
+
+/**
+ * Enables or disables a given interrupt ID for the calling vCPU.
+ *
+ * Returns 0 on success, or -1 if the intid is invalid.
+ */
+int64_t api_interrupt_enable(uint32_t intid, bool enable, struct vcpu *current)
+{
+	uint32_t intid_index = intid / INTERRUPT_REGISTER_BITS;
+	uint32_t intid_mask = 1U << (intid % INTERRUPT_REGISTER_BITS);
+
+	if (intid >= HF_NUM_INTIDS) {
+		return -1;
+	}
+
+	sl_lock(&current->lock);
+	if (enable) {
+		/*
+		 * If it is pending and was not enabled before, increment the
+		 * count.
+		 */
+		if (current->interrupts.interrupt_pending[intid_index] &
+		    ~current->interrupts.interrupt_enabled[intid_index] &
+		    intid_mask) {
+			current->interrupts.enabled_and_pending_count++;
+		}
+		current->interrupts.interrupt_enabled[intid_index] |=
+			intid_mask;
+	} else {
+		/*
+		 * If it is pending and was enabled before, decrement the count.
+		 */
+		if (current->interrupts.interrupt_pending[intid_index] &
+		    current->interrupts.interrupt_enabled[intid_index] &
+		    intid_mask) {
+			current->interrupts.enabled_and_pending_count--;
+		}
+		current->interrupts.interrupt_enabled[intid_index] &=
+			~intid_mask;
+	}
+
+	sl_unlock(&current->lock);
+	return 0;
+}
+
+/**
+ * Returns the ID of the next pending interrupt for the calling vCPU, and
+ * acknowledges it (i.e. marks it as no longer pending). Returns
+ * HF_INVALID_INTID if there are no pending interrupts.
+ */
+uint32_t api_interrupt_get(struct vcpu *current)
+{
+	uint8_t i;
+	uint32_t first_interrupt = HF_INVALID_INTID;
+
+	/*
+	 * Find the first enabled and pending interrupt ID, return it, and
+	 * deactivate it.
+	 */
+	sl_lock(&current->lock);
+	for (i = 0; i < HF_NUM_INTIDS / INTERRUPT_REGISTER_BITS; ++i) {
+		uint32_t enabled_and_pending =
+			current->interrupts.interrupt_enabled[i] &
+			current->interrupts.interrupt_pending[i];
+
+		if (enabled_and_pending != 0) {
+			uint8_t bit_index = ctz(enabled_and_pending);
+			/*
+			 * Mark it as no longer pending and decrement the count.
+			 */
+			current->interrupts.interrupt_pending[i] &=
+				~(1U << bit_index);
+			current->interrupts.enabled_and_pending_count--;
+			first_interrupt =
+				i * INTERRUPT_REGISTER_BITS + bit_index;
+			break;
+		}
+	}
+
+	sl_unlock(&current->lock);
+	return first_interrupt;
+}
+
+/**
+ * Returns whether the current vCPU is allowed to inject an interrupt into the
+ * given VM and vCPU.
+ */
+static inline bool is_injection_allowed(uint32_t target_vm_id,
+					struct vcpu *current)
+{
+	uint32_t current_vm_id = current->vm->id;
+
+	/*
+	 * The primary VM is allowed to inject interrupts into any VM. Secondary
+	 * VMs are only allowed to inject interrupts into their own vCPUs.
+	 */
+	return current_vm_id == HF_PRIMARY_VM_ID ||
+	       current_vm_id == target_vm_id;
+}
+
+/**
+ * Injects a virtual interrupt of the given ID into the given target vCPU.
+ * This doesn't cause the vCPU to actually be run immediately; it will be taken
+ * when the vCPU is next run, which is up to the scheduler.
+ *
+ * Returns:
+ *  - -1 on failure because the target VM or vCPU doesn't exist, the interrupt
+ *    ID is invalid, or the current VM is not allowed to inject interrupts to
+ *    the target VM.
+ *  - 0 on success if no further action is needed.
+ *  - 1 if it was called by the primary VM and the primary VM now needs to wake
+ *    up or kick the target vCPU.
+ */
+int64_t api_interrupt_inject(spci_vm_id_t target_vm_id,
+			     spci_vcpu_index_t target_vcpu_idx, uint32_t intid,
+			     struct vcpu *current, struct vcpu **next)
+{
+	struct vcpu *target_vcpu;
+	struct vm *target_vm = vm_find(target_vm_id);
+
+	if (intid >= HF_NUM_INTIDS) {
+		return -1;
+	}
+
+	if (target_vm == NULL) {
+		return -1;
+	}
+
+	if (target_vcpu_idx >= target_vm->vcpu_count) {
+		/* The requested vCPU must exist. */
+		return -1;
+	}
+
+	if (!is_injection_allowed(target_vm_id, current)) {
+		return -1;
+	}
+
+	target_vcpu = vm_get_vcpu(target_vm, target_vcpu_idx);
+
+	dlog("Injecting IRQ %d for VM %d vCPU %d from VM %d vCPU %d\n", intid,
+	     target_vm_id, target_vcpu_idx, current->vm->id, current->cpu->id);
+	return internal_interrupt_inject(target_vcpu, intid, current, next);
+}
+
+/** Returns the version of the implemented SPCI specification. */
+struct spci_value api_spci_version(void)
+{
+	/*
+	 * Ensure that both major and minor revision representation occupies at
+	 * most 15 bits.
+	 */
+	static_assert(0x8000 > SPCI_VERSION_MAJOR,
+		      "Major revision representation take more than 15 bits.");
+	static_assert(0x10000 > SPCI_VERSION_MINOR,
+		      "Minor revision representation take more than 16 bits.");
+
+	struct spci_value ret = {
+		.func = SPCI_SUCCESS_32,
+		.arg2 = (SPCI_VERSION_MAJOR << SPCI_VERSION_MAJOR_OFFSET) |
+			SPCI_VERSION_MINOR};
+	return ret;
+}
+
+int64_t api_debug_log(char c, struct vcpu *current)
+{
+	bool flush;
+	struct vm *vm = current->vm;
+	struct vm_locked vm_locked = vm_lock(vm);
+
+	if (c == '\n' || c == '\0') {
+		flush = true;
+	} else {
+		vm->log_buffer[vm->log_buffer_length++] = c;
+		flush = (vm->log_buffer_length == sizeof(vm->log_buffer));
+	}
+
+	if (flush) {
+		dlog_flush_vm_buffer(vm->id, vm->log_buffer,
+				     vm->log_buffer_length);
+		vm->log_buffer_length = 0;
+	}
+
+	vm_unlock(&vm_locked);
+
+	return 0;
+}
+
+/**
+ * Discovery function returning information about the implementation of optional
+ * SPCI interfaces.
+ */
+struct spci_value api_spci_features(uint32_t function_id)
+{
+	switch (function_id) {
+	case SPCI_ERROR_32:
+	case SPCI_SUCCESS_32:
+	case SPCI_ID_GET_32:
+	case SPCI_YIELD_32:
+	case SPCI_VERSION_32:
+	case SPCI_FEATURES_32:
+	case SPCI_MSG_SEND_32:
+	case SPCI_MSG_POLL_32:
+	case SPCI_MSG_WAIT_32:
+		return (struct spci_value){.func = SPCI_SUCCESS_32};
+	default:
+		return spci_error(SPCI_NOT_SUPPORTED);
+	}
+}
diff --git a/src/api_test.cc b/src/api_test.cc
new file mode 100644
index 0000000..0a80156
--- /dev/null
+++ b/src/api_test.cc
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This order of headers works around a libc++ issue which prevents
+ * "atomic" being included before "stdatomic.h".
+ */
+#include <gmock/gmock.h>
+
+extern "C" {
+#include "hf/api.h"
+}
+
+namespace
+{
+using ::testing::Eq;
+
+TEST(api, vm_get_count)
+{
+	EXPECT_THAT(api_vm_get_count(), Eq(0));
+}
+
+} /* namespace */
diff --git a/src/arch/aarch64/BUILD.gn b/src/arch/aarch64/BUILD.gn
new file mode 100644
index 0000000..93569ac
--- /dev/null
+++ b/src/arch/aarch64/BUILD.gn
@@ -0,0 +1,47 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+config("config") {
+  include_dirs = [ "." ]
+}
+
+# Implementation of the arch interface for aarch64.
+source_set("arch") {
+  sources = [
+    "irq.c",
+    "mm.c",
+    "timer.c",
+  ]
+}
+
+source_set("std") {
+  sources = [
+    "stack_protector.c",
+    "std.c",
+  ]
+}
+
+# Entry code to prepare the loaded image to be run.
+source_set("entry") {
+  sources = [
+    "entry.S",
+  ]
+}
+
+# Make a call to the secure monitor.
+source_set("smc") {
+  sources = [
+    "smc.c",
+  ]
+}
diff --git a/src/arch/aarch64/args.gni b/src/arch/aarch64/args.gni
new file mode 100644
index 0000000..1582b34
--- /dev/null
+++ b/src/arch/aarch64/args.gni
@@ -0,0 +1,18 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare_args() {
+  # SMC hooks to be used for the platform, specified as build target.
+  plat_smc = "//src/arch/aarch64/smc:absent"
+}
diff --git a/src/arch/aarch64/boot_flow/BUILD.gn b/src/arch/aarch64/boot_flow/BUILD.gn
new file mode 100644
index 0000000..d709a9e
--- /dev/null
+++ b/src/arch/aarch64/boot_flow/BUILD.gn
@@ -0,0 +1,25 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source_set("android") {
+  sources = [
+    "android.S",
+  ]
+}
+
+source_set("linux") {
+  sources = [
+    "linux.S",
+  ]
+}
diff --git a/src/arch/aarch64/boot_flow/android.S b/src/arch/aarch64/boot_flow/android.S
new file mode 100644
index 0000000..13f0530
--- /dev/null
+++ b/src/arch/aarch64/boot_flow/android.S
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.section .init.plat_boot_flow_hook, "ax"
+.global plat_boot_flow_hook
+plat_boot_flow_hook:
+	/* Do nothing. */
+	ret
diff --git a/src/arch/aarch64/boot_flow/linux.S b/src/arch/aarch64/boot_flow/linux.S
new file mode 100644
index 0000000..419f9a4
--- /dev/null
+++ b/src/arch/aarch64/boot_flow/linux.S
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.section .init.plat_boot_flow_hook, "ax"
+.global plat_boot_flow_hook
+plat_boot_flow_hook:
+	/* Save the FDT pointer to a global variable. */
+	adrp x25, plat_boot_flow_fdt_addr
+	add x25, x25, :lo12:plat_boot_flow_fdt_addr
+	str x0, [x25]
+	ret
diff --git a/src/arch/aarch64/entry.S b/src/arch/aarch64/entry.S
new file mode 100644
index 0000000..3d3725e
--- /dev/null
+++ b/src/arch/aarch64/entry.S
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This is a generic entry point for an image. It carries out the operations
+ * required to prepare the loaded image to be run. Specifically, it performs
+ * relocations and zeroing of the bss section using registers x25 and above.
+ */
+.section .init.entry, "ax"
+.global entry
+entry:
+	/* Linux aarch64 image header. */
+	b 0f
+	.word 0
+	.quad 0x1000      /* text_offset */
+	.quad image_size  /* image_size */
+	.quad 0           /* flags */
+	.quad 0           /* res2 */
+	.quad 0           /* res3 */
+	.quad 0           /* res4 */
+	.word 0x644d5241  /* magic */
+	.word 0
+
+	/*
+	 * Calculate the difference between the actual load address and the
+	 * preferred one. We'll use this to relocate.
+	 */
+0:	adrp x25, entry
+	add x25, x25, :lo12:entry
+
+	ldr w29, =ORIGIN_ADDRESS
+
+	sub x25, x25, x29
+
+	/* Find where the relocations begin and end. */
+	adrp x29, rela_begin
+	add x29, x29, :lo12:rela_begin
+
+	adrp x30, rela_end
+	add x30, x30, :lo12:rela_end
+
+	/* Iterate over all relocations. */
+1:	cmp x29, x30
+	b.eq 2f
+
+	ldp x26, x27, [x29], #16
+	ldr x28, [x29], #8
+
+	cmp w27, #1027 /* R_AARCH64_RELATIVE */
+#	b.ne 1b
+	b.ne .
+
+	add x28, x28, x25
+	str x28, [x26, x25]
+	b 1b
+
+	/* Zero out the bss section. */
+2:	adrp x29, bss_begin
+	add x29, x29, :lo12:bss_begin
+
+	adrp x30, bss_end
+	add x30, x30, :lo12:bss_end
+
+3:	cmp x29, x30
+	b.hs 4f
+
+	stp xzr, xzr, [x29], #16
+	b 3b
+
+	/* Branch to the entry point for the specific image. */
+4:	b image_entry
diff --git a/src/arch/aarch64/exception_macros.S b/src/arch/aarch64/exception_macros.S
new file mode 100644
index 0000000..fee454a
--- /dev/null
+++ b/src/arch/aarch64/exception_macros.S
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * From Linux commit 679db70801da9fda91d26caf13bf5b5ccc74e8e8:
+ * "Some CPUs can speculate past an ERET instruction and potentially perform
+ * speculative accesses to memory before processing the exception return.
+ * Since the register state is often controlled by a lower privilege level
+ * at the point of an ERET, this could potentially be used as part of a
+ * side-channel attack."
+ *
+ * This macro emits a speculation barrier after the ERET to prevent the CPU
+ * from speculating past the exception return.
+ *
+ * ARMv8.5 introduces a dedicated SB speculative barrier instruction.
+ * Use a DSB/ISB pair on older platforms.
+ */
+.macro eret_with_sb
+	eret
+	dsb	nsh
+	isb
+.endm
+
+/**
+ * Saves the volatile registers onto the stack. This currently takes 14
+ * instructions, so it can be used in exception handlers with 18 instructions
+ * left, 2 of which in the same cache line (assuming a 16-byte cache line).
+ *
+ * On return, x0 and x1 are initialised to elr_el2 and spsr_el2 respectively,
+ * which can be used as the first and second arguments of a subsequent call.
+ */
+.macro save_volatile_to_stack elx:req
+	/* Reserve stack space and save registers x0-x18, x29 & x30. */
+	stp x0, x1, [sp, #-(8 * 24)]!
+	stp x2, x3, [sp, #8 * 2]
+	stp x4, x5, [sp, #8 * 4]
+	stp x6, x7, [sp, #8 * 6]
+	stp x8, x9, [sp, #8 * 8]
+	stp x10, x11, [sp, #8 * 10]
+	stp x12, x13, [sp, #8 * 12]
+	stp x14, x15, [sp, #8 * 14]
+	stp x16, x17, [sp, #8 * 16]
+	str x18, [sp, #8 * 18]
+	stp x29, x30, [sp, #8 * 20]
+
+	/*
+	 * Save elr_elx & spsr_elx. This such that we can take nested exception
+	 * and still be able to unwind.
+	 */
+	mrs x0, elr_\elx
+	mrs x1, spsr_\elx
+	stp x0, x1, [sp, #8 * 22]
+.endm
+
+/**
+ * Restores the volatile registers from the stack. This currently takes 14
+ * instructions, so it can be used in exception handlers while still leaving 18
+ * instructions left; if paired with save_volatile_to_stack, there are 4
+ * instructions to spare.
+ */
+.macro restore_volatile_from_stack elx:req
+	/* Restore registers x2-x18, x29 & x30. */
+	ldp x2, x3, [sp, #8 * 2]
+	ldp x4, x5, [sp, #8 * 4]
+	ldp x6, x7, [sp, #8 * 6]
+	ldp x8, x9, [sp, #8 * 8]
+	ldp x10, x11, [sp, #8 * 10]
+	ldp x12, x13, [sp, #8 * 12]
+	ldp x14, x15, [sp, #8 * 14]
+	ldp x16, x17, [sp, #8 * 16]
+	ldr x18, [sp, #8 * 18]
+	ldp x29, x30, [sp, #8 * 20]
+
+	/* Restore registers elr_elx & spsr_elx, using x0 & x1 as scratch. */
+	ldp x0, x1, [sp, #8 * 22]
+	msr elr_\elx, x0
+	msr spsr_\elx, x1
+
+	/* Restore x0 & x1, and release stack space. */
+	ldp x0, x1, [sp], #8 * 24
+.endm
+
+/**
+ * This is a generic handler for exceptions taken at the current EL while using
+ * SP0. It behaves similarly to the SPx case by first switching to SPx, doing
+ * the work, then switching back to SP0 before returning.
+ *
+ * Switching to SPx and calling the C handler takes 16 instructions, so it's not
+ * possible to add a branch to a common exit path without going into the next
+ * cache line (assuming 16-byte cache lines). Additionally, to restore and
+ * return we need an additional 16 instructions, so we could implement the whole
+ * handler within the allotted 32 instructions. However, since we want to emit
+ * a speculation barrier after each ERET, we are forced to move the ERET to
+ * a shared exit path.
+ */
+.macro current_exception_sp0 elx:req handler:req eret_label:req
+	msr spsel, #1
+	save_volatile_to_stack \elx
+	bl \handler
+	restore_volatile_from_stack \elx
+	msr spsel, #0
+	b \eret_label
+.endm
+
+/**
+ * Variant of current_exception_sp0 which assumes the handler never returns.
+ */
+.macro noreturn_current_exception_sp0 elx:req handler:req
+	msr spsel, #1
+	save_volatile_to_stack \elx
+	b \handler
+.endm
+
+/**
+ * This is a generic handler for exceptions taken at the current EL while using
+ * SPx. It saves volatile registers, calls the C handler, restores volatile
+ * registers, then returns.
+ *
+ * Saving state and jumping to C handler takes 15 instructions. We add an extra
+ * branch to a common exit path. So each handler takes up one unique cache line
+ * and one shared cache line (assuming 16-byte cache lines).
+ */
+.macro current_exception_spx elx:req handler:req
+	save_volatile_to_stack \elx
+	bl \handler
+	b restore_from_stack_and_return
+.endm
+
+/**
+ * Variant of current_exception_spx which assumes the handler never returns.
+ */
+.macro noreturn_current_exception_spx elx:req handler:req
+	save_volatile_to_stack \elx
+	b \handler
+.endm
diff --git a/src/arch/aarch64/hftest/BUILD.gn b/src/arch/aarch64/hftest/BUILD.gn
new file mode 100644
index 0000000..a801bd1
--- /dev/null
+++ b/src/arch/aarch64/hftest/BUILD.gn
@@ -0,0 +1,92 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These components are only used by VMs for aarch64 specific actions.
+
+# Implements image_entry for a simple VM kernel.
+source_set("entry") {
+  sources = [
+    "entry.S",
+  ]
+}
+
+# Shutdown the system or exit emulation, start/stop CPUs.
+source_set("power_mgmt") {
+  testonly = true
+  public_configs = [ "//src/arch/aarch64:config" ]
+  sources = [
+    "cpu_entry.S",
+    "power_mgmt.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64:smc",
+  ]
+}
+
+# Exception handlers for interrupts.
+source_set("interrupts") {
+  testonly = true
+  public_configs = [
+    "//src/arch/aarch64:config",
+    "//test/hftest:hftest_config",
+  ]
+  sources = [
+    "events.c",
+    "exceptions.S",
+    "interrupts.c",
+  ]
+}
+
+# GICv3 EL1 driver.
+source_set("interrupts_gicv3") {
+  testonly = true
+  public_configs = [ "//src/arch/aarch64:config" ]
+  sources = [
+    "interrupts_gicv3.c",
+  ]
+}
+
+# Get/set CPU state.
+source_set("state") {
+  testonly = true
+  public_configs = [ "//src/arch/aarch64:config" ]
+  sources = [
+    "state.c",
+  ]
+}
+
+# Interact directly with registers.
+source_set("registers") {
+  testonly = true
+  sources = [
+    "registers.c",
+  ]
+}
+
+source_set("console") {
+  sources = [
+    "console.c",
+  ]
+
+  deps = [
+    "//vmlib/aarch64:call",
+  ]
+}
+
+source_set("mm") {
+  sources = [
+    "mm.c",
+  ]
+}
diff --git a/src/arch/aarch64/hftest/console.c b/src/arch/aarch64/hftest/console.c
new file mode 100644
index 0000000..4e2f4e8
--- /dev/null
+++ b/src/arch/aarch64/hftest/console.c
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/plat/console.h"
+
+#include "vmapi/hf/call.h"
+
+void plat_console_putchar(char c)
+{
+	hf_debug_log(c);
+}
diff --git a/src/arch/aarch64/hftest/cpu_entry.S b/src/arch/aarch64/hftest/cpu_entry.S
new file mode 100644
index 0000000..726b042
--- /dev/null
+++ b/src/arch/aarch64/hftest/cpu_entry.S
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.global vm_cpu_entry
+vm_cpu_entry:
+	/* Disable trapping floating point access in EL1. */
+	mov x1, #(0x3 << 20)
+	msr cpacr_el1, x1
+	isb
+
+	/* Initialise stack from the cpu_start_state struct. */
+	ldr x1, [x0]
+	mov sp, x1
+
+	/* Load entry function pointer and its argument. */
+	ldr x1, [x0, 8]
+	ldr x0, [x0, 16]
+
+	/* Branch to entry function. */
+	blr x1
+
+	/* Entry function should not return, but if it does, spin. */
+	b .
diff --git a/src/arch/aarch64/hftest/entry.S b/src/arch/aarch64/hftest/entry.S
new file mode 100644
index 0000000..5fbe110
--- /dev/null
+++ b/src/arch/aarch64/hftest/entry.S
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.section .init.image_entry, "ax"
+.global image_entry
+image_entry:
+	/* Prepare the stack. */
+	adr x30, kstack + 4096
+	mov sp, x30
+
+	/* Disable trapping floating point access in EL1. */
+	mov x30, #(0x3 << 20)
+	msr cpacr_el1, x30
+	isb
+
+	/* Call into C code. */
+	bl kmain
+
+	/* If the VM returns, shutdown the system. */
+	bl arch_power_off
+
+	/* Loop forever waiting for interrupts. */
+0:	wfi
+	b 0b
diff --git a/src/arch/aarch64/hftest/events.c b/src/arch/aarch64/hftest/events.c
new file mode 100644
index 0000000..435a74d
--- /dev/null
+++ b/src/arch/aarch64/hftest/events.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/events.h"
+
+void event_wait(void)
+{
+	__asm__ volatile("wfe");
+}
+
+void event_send_local(void)
+{
+	__asm__ volatile("sevl");
+}
diff --git a/src/arch/aarch64/hftest/exceptions.S b/src/arch/aarch64/hftest/exceptions.S
new file mode 100644
index 0000000..0203110
--- /dev/null
+++ b/src/arch/aarch64/hftest/exceptions.S
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "exception_macros.S"
+
+.section .text.vector_table_el1, "ax"
+.global vector_table_el1
+.balign 0x800
+vector_table_el1:
+sync_cur_sp0:
+	b .
+
+.balign 0x80
+irq_cur_sp0:
+	current_exception_sp0 el1 irq_current exception_handler_return
+
+.balign 0x80
+fiq_cur_sp0:
+	b .
+
+.balign 0x80
+serr_cur_sp0:
+	b .
+
+.balign 0x80
+sync_cur_spx:
+	current_exception_spx el1 sync_exception_current
+
+.balign 0x80
+irq_cur_spx:
+	current_exception_spx el1 irq_current
+
+.balign 0x80
+fiq_cur_spx:
+	b .
+
+.balign 0x80
+serr_cur_spx:
+	b .
+
+.balign 0x80
+sync_lower_64:
+	b .
+
+.balign 0x80
+irq_lower_64:
+	b .
+
+.balign 0x80
+fiq_lower_64:
+	b .
+
+.balign 0x80
+serr_lower_64:
+	b .
+
+.balign 0x80
+sync_lower_32:
+	b .
+
+.balign 0x80
+irq_lower_32:
+	b .
+
+.balign 0x80
+fiq_lower_32:
+	b .
+
+.balign 0x80
+serr_lower_32:
+	b .
+
+.balign 0x40
+/**
+ * Restores the volatile registers from the stack.
+
+ * Register x0: if false restores elr_el1, if true retains the value of elr_el1.
+ * This enables exception handlers to indicate whether they have changed the
+ * value of elr_el1 (e.g., to skip the faulting instruction).
+ */
+restore_from_stack_and_return:
+	/* Restore registers x2-x18, x29 & x30. */
+	ldp x2, x3, [sp, #8 * 2]
+	ldp x4, x5, [sp, #8 * 4]
+	ldp x6, x7, [sp, #8 * 6]
+	ldp x8, x9, [sp, #8 * 8]
+	ldp x10, x11, [sp, #8 * 10]
+	ldp x12, x13, [sp, #8 * 12]
+	ldp x14, x15, [sp, #8 * 14]
+	ldp x16, x17, [sp, #8 * 16]
+	ldr x18, [sp, #8 * 18]
+	ldp x29, x30, [sp, #8 * 20]
+
+	cbnz x0, skip_elr
+
+	/* Restore register elr_el1 using x1 as scratch. */
+	ldr x1, [sp, #8 * 22]
+	msr elr_el1, x1
+
+skip_elr:
+	/* Restore register spsr_el1 using x1 as scratch. */
+	ldr x1, [sp, #8 * 23]
+	msr spsr_el1, x1
+
+	/* Restore x0 & x1, and release stack space. */
+	ldp x0, x1, [sp], #8 * 24
+
+exception_handler_return:
+	eret_with_sb
diff --git a/src/arch/aarch64/hftest/interrupts.c b/src/arch/aarch64/hftest/interrupts.c
new file mode 100644
index 0000000..a7eb401
--- /dev/null
+++ b/src/arch/aarch64/hftest/interrupts.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts.h"
+
+#include <stdint.h>
+
+#include "hf/dlog.h"
+
+#include "msr.h"
+#include "test/hftest.h"
+
+extern uint8_t vector_table_el1;
+static void (*irq_callback)(void);
+static bool (*exception_callback)(void);
+
+void irq_current(void)
+{
+	if (irq_callback != NULL) {
+		irq_callback();
+	} else {
+		FAIL("Got unexpected interrupt.\n");
+	}
+}
+
+noreturn static bool default_sync_current_exception(void)
+{
+	uintreg_t esr = read_msr(esr_el1);
+	uintreg_t elr = read_msr(elr_el1);
+
+	switch (esr >> 26) {
+	case 0x25: /* EC = 100101, Data abort. */
+		dlog("Data abort: pc=%#x, esr=%#x, ec=%#x", elr, esr,
+		     esr >> 26);
+		if (!(esr & (1U << 10))) { /* Check FnV bit. */
+			dlog(", far=%#x", read_msr(far_el1));
+		} else {
+			dlog(", far=invalid");
+		}
+
+		dlog("\n");
+		break;
+
+	default:
+		dlog("Unknown current sync exception pc=%#x, esr=%#x, "
+		     "ec=%#x\n",
+		     elr, esr, esr >> 26);
+	}
+
+	for (;;) {
+		/* do nothing */
+	}
+}
+
+bool sync_exception_current(void)
+{
+	if (exception_callback != NULL) {
+		return exception_callback();
+	}
+	return default_sync_current_exception();
+}
+
+void exception_setup(void (*irq)(void), bool (*exception)(void))
+{
+	irq_callback = irq;
+	exception_callback = exception;
+
+	/* Set exception vector table. */
+	write_msr(VBAR_EL1, &vector_table_el1);
+}
+
+void interrupt_wait(void)
+{
+	__asm__ volatile("wfi");
+}
diff --git a/src/arch/aarch64/hftest/interrupts_gicv3.c b/src/arch/aarch64/hftest/interrupts_gicv3.c
new file mode 100644
index 0000000..9010ce1
--- /dev/null
+++ b/src/arch/aarch64/hftest/interrupts_gicv3.c
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts_gicv3.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "hf/dlog.h"
+
+#include "msr.h"
+
+void interrupt_gic_setup(void)
+{
+	uint32_t ctlr = 1U << 4    /* Enable affinity routing. */
+			| 1U << 1; /* Enable group 1 non-secure interrupts. */
+
+	write_msr(ICC_CTLR_EL1, 0);
+
+	io_write32(GICD_CTLR, ctlr);
+
+	/* Mark CPU as awake. */
+	io_write32(GICR_WAKER, io_read32(GICR_WAKER) & ~(1U << 1));
+	while ((io_read32(GICR_WAKER) & (1U << 2)) != 0) {
+		dlog("Waiting for ChildrenAsleep==0\n");
+	}
+
+	/* Put interrupts into non-secure group 1. */
+	dlog("GICR_IGROUPR0 was %x\n", 0xffffffff, io_read32(GICR_IGROUPR0));
+	io_write32(GICR_IGROUPR0, 0xffffffff);
+	dlog("wrote %x to GICR_IGROUPR0, got back %x\n", 0xffffffff,
+	     io_read32(GICR_IGROUPR0));
+	/* Enable non-secure group 1. */
+	write_msr(ICC_IGRPEN1_EL1, 0x00000001);
+	dlog("wrote %x to ICC_IGRPEN1_EL1, got back %x\n", 0x00000001,
+	     read_msr(ICC_IGRPEN1_EL1));
+}
+
+void interrupt_enable(uint32_t intid, bool enable)
+{
+	uint32_t index = intid / 32;
+	uint32_t bit = 1U << (intid % 32);
+
+	if (enable) {
+		io_write32_array(GICD_ISENABLER, index, bit);
+		if (intid < 32) {
+			io_write32(GICR_ISENABLER0, bit);
+		}
+	} else {
+		io_write32_array(GICD_ICENABLER, index, bit);
+		if (intid < 32) {
+			io_write32(GICR_ICENABLER0, bit);
+		}
+	}
+}
+
+void interrupt_enable_all(bool enable)
+{
+	uint32_t i;
+
+	if (enable) {
+		io_write32(GICR_ISENABLER0, 0xffffffff);
+		for (i = 0; i < 32; ++i) {
+			io_write32_array(GICD_ISENABLER, i, 0xffffffff);
+		}
+	} else {
+		io_write32(GICR_ISENABLER0, 0);
+		for (i = 0; i < 32; ++i) {
+			io_write32_array(GICD_ISENABLER, i, 0);
+		}
+	}
+}
+
+void interrupt_set_priority_mask(uint8_t min_priority)
+{
+	write_msr(ICC_PMR_EL1, min_priority);
+}
+
+void interrupt_set_priority(uint32_t intid, uint8_t priority)
+{
+	io_write8_array(GICD_IPRIORITYR, intid, priority);
+}
+
+void interrupt_set_edge_triggered(uint32_t intid, bool edge_triggered)
+{
+	uint32_t index = intid / 16;
+	uint32_t bit = 1U << (((intid % 16) * 2) + 1);
+
+	if (intid < 32) {
+		uint32_t v = io_read32_array(GICR_ICFGR, index);
+
+		if (edge_triggered) {
+			io_write32_array(GICR_ICFGR, index, v | bit);
+		} else {
+			io_write32_array(GICR_ICFGR, index, v & ~bit);
+		}
+	} else {
+		uint32_t v = io_read32_array(GICD_ICFGR, index);
+
+		if (edge_triggered) {
+			io_write32_array(GICD_ICFGR, index, v | bit);
+		} else {
+			io_write32_array(GICD_ICFGR, index, v & ~bit);
+		}
+	}
+}
+
+void interrupt_send_sgi(uint8_t intid, bool irm, uint8_t affinity3,
+			uint8_t affinity2, uint8_t affinity1,
+			uint16_t target_list)
+{
+	uint64_t sgi_register =
+		((uint64_t)target_list) | ((uint64_t)affinity1 << 16) |
+		(((uint64_t)intid & 0x0f) << 24) | ((uint64_t)affinity2 << 32) |
+		((uint64_t)irm << 40) | ((uint64_t)affinity3 << 48);
+
+	write_msr(ICC_SGI1R_EL1, sgi_register);
+}
+
+uint32_t interrupt_get_and_acknowledge(void)
+{
+	return read_msr(ICC_IAR1_EL1);
+}
+
+void interrupt_end(uint32_t intid)
+{
+	write_msr(ICC_EOIR1_EL1, intid);
+}
diff --git a/src/arch/aarch64/hftest/mm.c b/src/arch/aarch64/hftest/mm.c
new file mode 100644
index 0000000..86f27d8
--- /dev/null
+++ b/src/arch/aarch64/hftest/mm.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/mm.h"
+
+#include "hf/arch/barriers.h"
+#include "hf/arch/vm/mm.h"
+
+#include "hf/dlog.h"
+
+#include "../msr.h"
+
+#define STAGE1_DEVICEINDX UINT64_C(0)
+#define STAGE1_NORMALINDX UINT64_C(1)
+
+static uintreg_t mm_mair_el1;
+static uintreg_t mm_tcr_el1;
+static uintreg_t mm_sctlr_el1;
+
+/**
+ * Initialize MMU for a test running in EL1.
+ */
+bool arch_vm_mm_init(void)
+{
+	static const int pa_bits_table[16] = {32, 36, 40, 42, 44, 48};
+	uint64_t features = read_msr(id_aa64mmfr0_el1);
+	int pa_bits = pa_bits_table[features & 0xf];
+
+	/* Check that 4KB granules are supported. */
+	if ((features >> 28) & 0xf) {
+		dlog("4KB granules are not supported\n");
+		return false;
+	}
+
+	/* Check the physical address range. */
+	if (!pa_bits) {
+		dlog("Unsupported value of id_aa64mmfr0_el1.PARange: %x\n",
+		     features & 0xf);
+		return false;
+	}
+
+	/*
+	 * 0    -> Device-nGnRnE memory
+	 * 0xff -> Normal memory, Inner/Outer Write-Back Non-transient,
+	 *         Write-Alloc, Read-Alloc.
+	 */
+	mm_mair_el1 = (0 << (8 * STAGE1_DEVICEINDX)) |
+		      (0xff << (8 * STAGE1_NORMALINDX));
+
+	mm_tcr_el1 = (1 << 20) |		/* TBI, top byte ignored. */
+		     ((features & 0xf) << 16) | /* PS. */
+		     (0 << 14) |		/* TG0, granule size, 4KB. */
+		     (3 << 12) |		/* SH0, inner shareable. */
+		     (1 << 10) | /* ORGN0, normal mem, WB RA WA Cacheable. */
+		     (1 << 8) |  /* IRGN0, normal mem, WB RA WA Cacheable. */
+		     (25 << 0) | /* T0SZ, input address is 2^39 bytes. */
+		     0;
+
+	mm_sctlr_el1 = (1 << 0) |  /* M, enable stage 1 EL2 MMU. */
+		       (1 << 1) |  /* A, enable alignment check faults. */
+		       (1 << 2) |  /* C, data cache enable. */
+		       (1 << 3) |  /* SA, enable stack alignment check. */
+		       (3 << 4) |  /* RES1 bits. */
+		       (1 << 11) | /* RES1 bit. */
+		       (1 << 12) | /* I, instruction cache enable. */
+		       (1 << 16) | /* RES1 bit. */
+		       (1 << 18) | /* RES1 bit. */
+		       (0 << 19) | /* WXN bit, writable execute never. */
+		       (3 << 22) | /* RES1 bits. */
+		       (3 << 28) | /* RES1 bits. */
+		       0;
+
+	return true;
+}
+
+void arch_vm_mm_enable(paddr_t table)
+{
+	/* Configure translation management registers. */
+	write_msr(ttbr0_el1, pa_addr(table));
+	write_msr(mair_el1, mm_mair_el1);
+	write_msr(tcr_el1, mm_tcr_el1);
+
+	/* Configure sctlr_el1 to enable MMU and cache. */
+	dsb(sy);
+	isb();
+	write_msr(sctlr_el1, mm_sctlr_el1);
+	isb();
+}
diff --git a/src/arch/aarch64/hftest/power_mgmt.c b/src/arch/aarch64/hftest/power_mgmt.c
new file mode 100644
index 0000000..cfa3e08
--- /dev/null
+++ b/src/arch/aarch64/hftest/power_mgmt.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include "hf/static_assert.h"
+
+#include "vmapi/hf/call.h"
+
+#include "psci.h"
+#include "smc.h"
+
+/**
+ * Starts the CPU with the given ID. It will set the stack pointer according to
+ * the provided `state` and jump to the entry point with the given argument
+ * specified in it.
+ *
+ * Note: The caller of this function must guarantee that the contents of `state`
+ * do not change until the new CPU has branched to the given entry point, and
+ * that it was written-back to memory (that it is not waiting in a data cache)
+ * because the new CPU is started with caching disabled.
+ */
+bool arch_cpu_start(uintptr_t id, struct arch_cpu_start_state *state)
+{
+	void vm_cpu_entry(uintptr_t arg);
+	struct spci_value smc_res;
+
+	/* Try to start the CPU. */
+	smc_res = smc64(PSCI_CPU_ON, id, (uintptr_t)&vm_cpu_entry,
+			(uintptr_t)state, 0, 0, 0, SMCCC_CALLER_HYPERVISOR);
+
+	return smc_res.func == PSCI_RETURN_SUCCESS;
+}
+
+/**
+ * Stops the current CPU.
+ */
+noreturn void arch_cpu_stop(void)
+{
+	smc32(PSCI_CPU_OFF, 0, 0, 0, 0, 0, 0, SMCCC_CALLER_HYPERVISOR);
+	for (;;) {
+		/* This should never be reached. */
+	}
+}
+
+static_assert(POWER_STATUS_ON == PSCI_RETURN_ON,
+	      "power_status enum values must match PSCI return values.");
+static_assert(POWER_STATUS_OFF == PSCI_RETURN_OFF,
+	      "power_status enum values must match PSCI return values.");
+static_assert(POWER_STATUS_ON_PENDING == PSCI_RETURN_ON_PENDING,
+	      "power_status enum values must match PSCI return values.");
+
+/**
+ * Returns the power status of the given CPU.
+ */
+enum power_status arch_cpu_status(cpu_id_t cpu_id)
+{
+	uint32_t lowest_affinity_level = 0;
+	struct spci_value smc_res;
+
+	/*
+	 * This works because the power_status enum values happen to be the same
+	 * as the PSCI_RETURN_* values. The static_asserts above validate that
+	 * this is the case.
+	 */
+	smc_res = smc32(PSCI_AFFINITY_INFO, cpu_id, lowest_affinity_level, 0, 0,
+			0, 0, SMCCC_CALLER_HYPERVISOR);
+	return smc_res.func;
+}
+
+/**
+ * Shuts down the system or exits emulation.
+ */
+noreturn void arch_power_off(void)
+{
+	smc32(PSCI_SYSTEM_OFF, 0, 0, 0, 0, 0, 0, SMCCC_CALLER_HYPERVISOR);
+	for (;;) {
+		/* This should never be reached. */
+	}
+}
diff --git a/src/arch/aarch64/hftest/registers.c b/src/arch/aarch64/hftest/registers.c
new file mode 100644
index 0000000..15b3409
--- /dev/null
+++ b/src/arch/aarch64/hftest/registers.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/registers.h"
+
+#define read_fp_register(name)                                   \
+	__extension__({                                          \
+		double __v;                                      \
+		__asm__ volatile("fmov %0, " #name : "=r"(__v)); \
+		__v;                                             \
+	})
+
+#define write_fp_register(name, value)                  \
+	__extension__({                                 \
+		__asm__ volatile("fmov " #name ", %0"   \
+				 :                      \
+				 : "r"((double)(value)) \
+				 : #name);              \
+	})
+
+#define move_fp_register(dest, source)                                    \
+	__extension__({                                                   \
+		__asm__ volatile("fmov " #dest ", " #source : : : #dest); \
+	})
+
+void fill_fp_registers(double value)
+{
+	write_fp_register(d0, value);
+	move_fp_register(d1, d0);
+	move_fp_register(d2, d0);
+	move_fp_register(d3, d0);
+	move_fp_register(d4, d0);
+	move_fp_register(d5, d0);
+	move_fp_register(d6, d0);
+	move_fp_register(d7, d0);
+	move_fp_register(d8, d0);
+	move_fp_register(d9, d0);
+	move_fp_register(d10, d0);
+	move_fp_register(d11, d0);
+	move_fp_register(d12, d0);
+	move_fp_register(d13, d0);
+	move_fp_register(d14, d0);
+	move_fp_register(d15, d0);
+	move_fp_register(d16, d0);
+	move_fp_register(d17, d0);
+	move_fp_register(d18, d0);
+	move_fp_register(d19, d0);
+	move_fp_register(d20, d0);
+	move_fp_register(d21, d0);
+	move_fp_register(d22, d0);
+	move_fp_register(d23, d0);
+	move_fp_register(d24, d0);
+	move_fp_register(d25, d0);
+	move_fp_register(d26, d0);
+	move_fp_register(d27, d0);
+	move_fp_register(d28, d0);
+	move_fp_register(d29, d0);
+	move_fp_register(d30, d0);
+	move_fp_register(d31, d0);
+}
+
+bool check_fp_register(double value)
+{
+	bool result = true;
+
+	result = result && (read_fp_register(d0) == value);
+	result = result && (read_fp_register(d1) == value);
+	result = result && (read_fp_register(d2) == value);
+	result = result && (read_fp_register(d3) == value);
+	result = result && (read_fp_register(d4) == value);
+	result = result && (read_fp_register(d5) == value);
+	result = result && (read_fp_register(d6) == value);
+	result = result && (read_fp_register(d7) == value);
+	result = result && (read_fp_register(d8) == value);
+	result = result && (read_fp_register(d9) == value);
+	result = result && (read_fp_register(d10) == value);
+	result = result && (read_fp_register(d11) == value);
+	result = result && (read_fp_register(d12) == value);
+	result = result && (read_fp_register(d13) == value);
+	result = result && (read_fp_register(d14) == value);
+	result = result && (read_fp_register(d15) == value);
+	result = result && (read_fp_register(d16) == value);
+	result = result && (read_fp_register(d17) == value);
+	result = result && (read_fp_register(d18) == value);
+	result = result && (read_fp_register(d19) == value);
+	result = result && (read_fp_register(d20) == value);
+	result = result && (read_fp_register(d21) == value);
+	result = result && (read_fp_register(d21) == value);
+	result = result && (read_fp_register(d23) == value);
+	result = result && (read_fp_register(d24) == value);
+	result = result && (read_fp_register(d25) == value);
+	result = result && (read_fp_register(d26) == value);
+	result = result && (read_fp_register(d27) == value);
+	result = result && (read_fp_register(d28) == value);
+	result = result && (read_fp_register(d29) == value);
+	result = result && (read_fp_register(d30) == value);
+	result = result && (read_fp_register(d31) == value);
+	return result;
+}
diff --git a/src/arch/aarch64/hftest/state.c b/src/arch/aarch64/hftest/state.c
new file mode 100644
index 0000000..53471f2
--- /dev/null
+++ b/src/arch/aarch64/hftest/state.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/state.h"
+
+#include "msr.h"
+
+void per_cpu_ptr_set(uintptr_t v)
+{
+	write_msr(tpidr_el1, v);
+}
+
+uintptr_t per_cpu_ptr_get(void)
+{
+	return read_msr(tpidr_el1);
+}
diff --git a/src/arch/aarch64/hypervisor/BUILD.gn b/src/arch/aarch64/hypervisor/BUILD.gn
new file mode 100644
index 0000000..8a057e9
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/BUILD.gn
@@ -0,0 +1,52 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/offset_size_header.gni")
+import("//src/arch/aarch64/args.gni")
+
+offset_size_header("offsets") {
+  sources = [
+    "offsets.c",
+  ]
+  path = "hf/arch/offsets.h"
+}
+
+# Hypervisor specific code.
+source_set("hypervisor") {
+  public_configs = [ "//src/arch/aarch64:config" ]
+  sources = [
+    "exceptions.S",
+    "hypervisor_entry.S",
+    "plat_entry.S",
+  ]
+
+  sources += [
+    "cpu.c",
+    "debug_el1.c",
+    "feature_id.c",
+    "handler.c",
+    "perfmon.c",
+    "psci_handler.c",
+    "sysregs.c",
+    "vm.c",
+  ]
+
+  deps = [
+    ":offsets",
+    "//src/arch/aarch64:arch",
+    "//src/arch/aarch64:entry",
+    "//src/arch/aarch64:smc",
+    plat_smc,
+  ]
+}
diff --git a/src/arch/aarch64/hypervisor/cpu.c b/src/arch/aarch64/hypervisor/cpu.c
new file mode 100644
index 0000000..346c6b6
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/cpu.c
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/cpu.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hf/addr.h"
+#include "hf/spci.h"
+#include "hf/std.h"
+#include "hf/vm.h"
+
+#include "feature_id.h"
+#include "msr.h"
+#include "perfmon.h"
+#include "sysregs.h"
+
+/**
+ * The LO field indicates whether LORegions are supported.
+ */
+#define ID_AA64MMFR1_EL1_LO (UINT64_C(1) << 16)
+
+static void lor_disable(void)
+{
+	/*
+	 * Accesses to LORC_EL1 are undefined if LORegions are not supported.
+	 */
+	if (read_msr(ID_AA64MMFR1_EL1) & ID_AA64MMFR1_EL1_LO) {
+		write_msr(MSR_LORC_EL1, 0);
+	}
+}
+
+static void gic_regs_reset(struct arch_regs *r, bool is_primary)
+{
+#if GIC_VERSION == 3 || GIC_VERSION == 4
+	uint32_t ich_hcr = 0;
+	uint32_t icc_sre_el2 =
+		(1U << 0) | /* SRE, enable ICH_* and ICC_* at EL2. */
+		(0x3 << 1); /* DIB and DFB, disable IRQ/FIQ bypass. */
+
+	if (is_primary) {
+		icc_sre_el2 |= 1U << 3; /* Enable EL1 access to ICC_SRE_EL1. */
+	} else {
+		/* Trap EL1 access to GICv3 system registers. */
+		ich_hcr =
+			(0x1fU << 10); /* TDIR, TSEI, TALL1, TALL0, TC bits. */
+	}
+	r->gic.ich_hcr_el2 = ich_hcr;
+	r->gic.icc_sre_el2 = icc_sre_el2;
+#endif
+}
+
+void arch_regs_reset(struct vcpu *vcpu)
+{
+	spci_vm_id_t vm_id = vcpu->vm->id;
+	bool is_primary = vm_id == HF_PRIMARY_VM_ID;
+	cpu_id_t vcpu_id = vcpu_index(vcpu);
+	paddr_t table = vcpu->vm->ptable.root;
+	struct arch_regs *r = &vcpu->regs;
+	uintreg_t pc = r->pc;
+	uintreg_t arg = r->r[0];
+	uintreg_t cnthctl;
+
+	memset_s(r, sizeof(*r), 0, sizeof(*r));
+
+	r->pc = pc;
+	r->r[0] = arg;
+
+	cnthctl = 0;
+
+	if (is_primary) {
+		cnthctl |=
+			(1U << 0) | /* EL1PCTEN, don't trap phys cnt access. */
+			(1U << 1);  /* EL1PCEN, don't trap phys timer access. */
+	}
+
+	r->lazy.hcr_el2 = get_hcr_el2_value(vm_id);
+	r->lazy.cnthctl_el2 = cnthctl;
+	r->lazy.vttbr_el2 = pa_addr(table) | ((uint64_t)vm_id << 48);
+	r->lazy.vmpidr_el2 = vcpu_id;
+	/* Mask (disable) interrupts and run in EL1h mode. */
+	r->spsr = PSR_D | PSR_A | PSR_I | PSR_F | PSR_PE_MODE_EL1H;
+
+	r->lazy.mdcr_el2 = get_mdcr_el2_value();
+
+	/*
+	 * NOTE: It is important that MDSCR_EL1.MDE (bit 15) is set to 0 for
+	 * secondary VMs as long as Hafnium does not support debug register
+	 * access for secondary VMs. If adding Hafnium support for secondary VM
+	 * debug register accesses, then on context switches Hafnium needs to
+	 * save/restore EL1 debug register state that either might change, or
+	 * that needs to be protected.
+	 */
+	r->lazy.mdscr_el1 = 0x0U & ~(0x1U << 15);
+
+	/* Disable cycle counting on initialization. */
+	r->lazy.pmccfiltr_el0 = perfmon_get_pmccfiltr_el0_init_value(vm_id);
+
+	/* Set feature-specific register values. */
+	feature_set_traps(vcpu->vm, r);
+
+	gic_regs_reset(r, is_primary);
+}
+
+void arch_regs_set_pc_arg(struct arch_regs *r, ipaddr_t pc, uintreg_t arg)
+{
+	r->pc = ipa_addr(pc);
+	r->r[0] = arg;
+}
+
+void arch_regs_set_retval(struct arch_regs *r, struct spci_value v)
+{
+	r->r[0] = v.func;
+	r->r[1] = v.arg1;
+	r->r[2] = v.arg2;
+	r->r[3] = v.arg3;
+	r->r[4] = v.arg4;
+	r->r[5] = v.arg5;
+	r->r[6] = v.arg6;
+	r->r[7] = v.arg7;
+}
+
+void arch_cpu_init(void)
+{
+	/*
+	 * Linux expects LORegions to be disabled, hence if the current system
+	 * supports them, Hafnium ensures that they are disabled.
+	 */
+	lor_disable();
+
+	write_msr(CPTR_EL2, get_cptr_el2_value());
+}
diff --git a/src/arch/aarch64/hypervisor/debug_el1.c b/src/arch/aarch64/hypervisor/debug_el1.c
new file mode 100644
index 0000000..45e69b0
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/debug_el1.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug_el1.h"
+
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/types.h"
+
+#include "msr.h"
+#include "sysregs.h"
+
+/* clang-format off */
+
+/**
+ * Definitions of read-only debug registers' encodings.
+ * See Arm Architecture Reference Manual Armv8-A, D12.2.
+ * NAME, op0, op1, crn, crm, op2
+ */
+#define EL1_DEBUG_REGISTERS_READ	      \
+	X(MDRAR_EL1	    , 2, 0, 1,  0, 0) \
+	X(OSLSR_EL1	    , 2, 0, 1,  1, 4) \
+	X(DBGAUTHSTATUS_EL1 , 2, 0, 7, 14, 6) \
+
+/**
+ * Definitions of write-only debug registers' encodings.
+ * See Arm Architecture Reference Manual Armv8-A, D12.2.
+ * NAME, op0, op1, crn, crm, op2
+ */
+#define EL1_DEBUG_REGISTERS_WRITE	      \
+	X(OSLAR_EL1	    , 2, 0, 1,  0, 4) \
+
+/**
+ * Definitions of readable and writeable debug registers' encodings.
+ * See Arm Architecture Reference Manual Armv8-A, D12.2.
+ * NAME, op0, op1, crn, crm, op2
+ */
+#define EL1_DEBUG_REGISTERS_READ_WRITE        \
+	X(OSDTRRX_EL1	    , 2, 0, 0,  0, 2) \
+	X(MDCCINT_EL1	    , 2, 0, 0,  2, 0) \
+	X(MDSCR_EL1	    , 2, 0, 0,  2, 2) \
+	X(OSDTRTX_EL1	    , 2, 0, 0,  3, 2) \
+	X(OSECCR_EL1	    , 2, 0, 0,  6, 2) \
+	X(DBGBVR0_EL1	    , 2, 0, 0,  0, 4) \
+	X(DBGBVR1_EL1	    , 2, 0, 0,  1, 4) \
+	X(DBGBVR2_EL1	    , 2, 0, 0,  2, 4) \
+	X(DBGBVR3_EL1	    , 2, 0, 0,  3, 4) \
+	X(DBGBVR4_EL1	    , 2, 0, 0,  4, 4) \
+	X(DBGBVR5_EL1	    , 2, 0, 0,  5, 4) \
+	X(DBGBVR6_EL1	    , 2, 0, 0,  6, 4) \
+	X(DBGBVR7_EL1	    , 2, 0, 0,  7, 4) \
+	X(DBGBVR8_EL1	    , 2, 0, 0,  8, 4) \
+	X(DBGBVR9_EL1	    , 2, 0, 0,  9, 4) \
+	X(DBGBVR10_EL1	    , 2, 0, 0, 10, 4) \
+	X(DBGBVR11_EL1	    , 2, 0, 0, 11, 4) \
+	X(DBGBVR12_EL1	    , 2, 0, 0, 12, 4) \
+	X(DBGBVR13_EL1	    , 2, 0, 0, 13, 4) \
+	X(DBGBVR14_EL1	    , 2, 0, 0, 14, 4) \
+	X(DBGBVR15_EL1	    , 2, 0, 0, 15, 4) \
+	X(DBGBCR0_EL1       , 2, 0, 0,  0, 5) \
+	X(DBGBCR1_EL1       , 2, 0, 0,  1, 5) \
+	X(DBGBCR2_EL1       , 2, 0, 0,  2, 5) \
+	X(DBGBCR3_EL1       , 2, 0, 0,  3, 5) \
+	X(DBGBCR4_EL1       , 2, 0, 0,  4, 5) \
+	X(DBGBCR5_EL1       , 2, 0, 0,  5, 5) \
+	X(DBGBCR6_EL1       , 2, 0, 0,  6, 5) \
+	X(DBGBCR7_EL1       , 2, 0, 0,  7, 5) \
+	X(DBGBCR8_EL1       , 2, 0, 0,  8, 5) \
+	X(DBGBCR9_EL1       , 2, 0, 0,  9, 5) \
+	X(DBGBCR10_EL1      , 2, 0, 0, 10, 5) \
+	X(DBGBCR11_EL1      , 2, 0, 0, 11, 5) \
+	X(DBGBCR12_EL1      , 2, 0, 0, 12, 5) \
+	X(DBGBCR13_EL1      , 2, 0, 0, 13, 5) \
+	X(DBGBCR14_EL1      , 2, 0, 0, 14, 5) \
+	X(DBGBCR15_EL1      , 2, 0, 0, 15, 5) \
+	X(DBGWVR0_EL1       , 2, 0, 0,  0, 6) \
+	X(DBGWVR1_EL1       , 2, 0, 0,  1, 6) \
+	X(DBGWVR2_EL1       , 2, 0, 0,  2, 6) \
+	X(DBGWVR3_EL1       , 2, 0, 0,  3, 6) \
+	X(DBGWVR4_EL1       , 2, 0, 0,  4, 6) \
+	X(DBGWVR5_EL1       , 2, 0, 0,  5, 6) \
+	X(DBGWVR6_EL1       , 2, 0, 0,  6, 6) \
+	X(DBGWVR7_EL1       , 2, 0, 0,  7, 6) \
+	X(DBGWVR8_EL1       , 2, 0, 0,  8, 6) \
+	X(DBGWVR9_EL1       , 2, 0, 0,  9, 6) \
+	X(DBGWVR10_EL1      , 2, 0, 0, 10, 6) \
+	X(DBGWVR11_EL1      , 2, 0, 0, 11, 6) \
+	X(DBGWVR12_EL1      , 2, 0, 0, 12, 6) \
+	X(DBGWVR13_EL1      , 2, 0, 0, 13, 6) \
+	X(DBGWVR14_EL1      , 2, 0, 0, 14, 6) \
+	X(DBGWVR15_EL1      , 2, 0, 0, 15, 6) \
+	X(DBGWCR0_EL1       , 2, 0, 0,  0, 7) \
+	X(DBGWCR1_EL1       , 2, 0, 0,  1, 7) \
+	X(DBGWCR2_EL1       , 2, 0, 0,  2, 7) \
+	X(DBGWCR3_EL1       , 2, 0, 0,  3, 7) \
+	X(DBGWCR4_EL1       , 2, 0, 0,  4, 7) \
+	X(DBGWCR5_EL1       , 2, 0, 0,  5, 7) \
+	X(DBGWCR6_EL1       , 2, 0, 0,  6, 7) \
+	X(DBGWCR7_EL1       , 2, 0, 0,  7, 7) \
+	X(DBGWCR8_EL1       , 2, 0, 0,  8, 7) \
+	X(DBGWCR9_EL1       , 2, 0, 0,  9, 7) \
+	X(DBGWCR10_EL1      , 2, 0, 0, 10, 7) \
+	X(DBGWCR11_EL1      , 2, 0, 0, 11, 7) \
+	X(DBGWCR12_EL1      , 2, 0, 0, 12, 7) \
+	X(DBGWCR13_EL1      , 2, 0, 0, 13, 7) \
+	X(DBGWCR14_EL1      , 2, 0, 0, 14, 7) \
+	X(DBGWCR15_EL1      , 2, 0, 0, 15, 7) \
+	X(OSDLR_EL1         , 2, 0, 1,  3, 4) \
+	X(DBGPRCR_EL1       , 2, 0, 1,  4, 4) \
+	X(DBGCLAIMSET_EL1   , 2, 0, 7,  8, 6) \
+	X(DBGCLAIMCLR_EL1   , 2, 0, 7,  9, 6)
+
+/* clang-format on */
+
+/**
+ * Returns true if the ESR register shows an access to an EL1 debug register.
+ */
+bool debug_el1_is_register_access(uintreg_t esr)
+{
+	/*
+	 * Architecture Reference Manual D12.2: op0 == 2 is for debug and trace
+	 * system registers, op1 == 1 for trace, remaining are debug.
+	 */
+	return GET_ISS_OP0(esr) == 2 && GET_ISS_OP1(esr) != 1;
+}
+
+/**
+ * Processes an access (msr, mrs) to an EL1 debug register.
+ * Returns true if the access was allowed and performed, false otherwise.
+ */
+bool debug_el1_process_access(struct vcpu *vcpu, spci_vm_id_t vm_id,
+			      uintreg_t esr)
+{
+	/*
+	 * For now, debug registers are not supported by secondary VMs.
+	 * Disallow accesses to them.
+	 */
+	if (vm_id != HF_PRIMARY_VM_ID) {
+		return false;
+	}
+
+	uintreg_t sys_register = GET_ISS_SYSREG(esr);
+	uintreg_t rt_register = GET_ISS_RT(esr);
+	uintreg_t value;
+
+	/* +1 because Rt can access register XZR */
+	CHECK(rt_register < NUM_GP_REGS + 1);
+
+	if (ISS_IS_READ(esr)) {
+		switch (sys_register) {
+#define X(reg_name, op0, op1, crn, crm, op2)              \
+	case (GET_ISS_ENCODING(op0, op1, crn, crm, op2)): \
+		value = read_msr(reg_name);               \
+		break;
+			EL1_DEBUG_REGISTERS_READ
+			EL1_DEBUG_REGISTERS_READ_WRITE
+#undef X
+		default:
+			value = vcpu->regs.r[rt_register];
+			dlog("Unsupported debug system register read: "
+			     "op0=%d, op1=%d, crn=%d, crm=%d, op2=%d, rt=%d.\n",
+			     GET_ISS_OP0(esr), GET_ISS_OP1(esr),
+			     GET_ISS_CRN(esr), GET_ISS_CRM(esr),
+			     GET_ISS_OP2(esr), GET_ISS_RT(esr));
+			break;
+		}
+		if (rt_register != RT_REG_XZR) {
+			vcpu->regs.r[rt_register] = value;
+		}
+	} else {
+		if (rt_register != RT_REG_XZR) {
+			value = vcpu->regs.r[rt_register];
+		} else {
+			value = 0;
+		}
+		switch (sys_register) {
+#define X(reg_name, op0, op1, crn, crm, op2)              \
+	case (GET_ISS_ENCODING(op0, op1, crn, crm, op2)): \
+		write_msr(reg_name, value);               \
+		break;
+			EL1_DEBUG_REGISTERS_WRITE
+			EL1_DEBUG_REGISTERS_READ_WRITE
+#undef X
+		default:
+			dlog("Unsupported debug system register write: "
+			     "op0=%d, op1=%d, crn=%d, crm=%d, op2=%d, rt=%d.\n",
+			     GET_ISS_OP0(esr), GET_ISS_OP1(esr),
+			     GET_ISS_CRN(esr), GET_ISS_CRM(esr),
+			     GET_ISS_OP2(esr), GET_ISS_RT(esr));
+			break;
+		}
+	}
+
+	return true;
+}
diff --git a/src/arch/aarch64/hypervisor/debug_el1.h b/src/arch/aarch64/hypervisor/debug_el1.h
new file mode 100644
index 0000000..9dc1ef6
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/debug_el1.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/arch/types.h"
+
+#include "hf/cpu.h"
+
+#include "vmapi/hf/spci.h"
+
+bool debug_el1_is_register_access(uintreg_t esr_el2);
+
+bool debug_el1_process_access(struct vcpu *vcpu, spci_vm_id_t vm_id,
+			      uintreg_t esr_el2);
diff --git a/src/arch/aarch64/hypervisor/exceptions.S b/src/arch/aarch64/hypervisor/exceptions.S
new file mode 100644
index 0000000..17a5b63
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/exceptions.S
@@ -0,0 +1,529 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/offsets.h"
+#include "exception_macros.S"
+
+/**
+ * Saves the volatile registers into the register buffer of the current vCPU.
+ */
+.macro save_volatile_to_vcpu
+	/*
+	 * Save x18 since we're about to clobber it. We subtract 16 instead of
+	 * 8 from the stack pointer to keep it 16-byte aligned.
+	 */
+	str x18, [sp, #-16]!
+
+	/* Get the current vCPU. */
+	mrs x18, tpidr_el2
+	stp x0, x1, [x18, #VCPU_REGS + 8 * 0]
+	stp x2, x3, [x18, #VCPU_REGS + 8 * 2]
+	stp x4, x5, [x18, #VCPU_REGS + 8 * 4]
+	stp x6, x7, [x18, #VCPU_REGS + 8 * 6]
+	stp x8, x9, [x18, #VCPU_REGS + 8 * 8]
+	stp x10, x11, [x18, #VCPU_REGS + 8 * 10]
+	stp x12, x13, [x18, #VCPU_REGS + 8 * 12]
+	stp x14, x15, [x18, #VCPU_REGS + 8 * 14]
+	stp x16, x17, [x18, #VCPU_REGS + 8 * 16]
+	stp x29, x30, [x18, #VCPU_REGS + 8 * 29]
+
+	/* x18 was saved on the stack, so we move it to vCPU regs buffer. */
+	ldr x0, [sp], #16
+	str x0, [x18, #VCPU_REGS + 8 * 18]
+
+	/* Save return address & mode. */
+	mrs x1, elr_el2
+	mrs x2, spsr_el2
+	stp x1, x2, [x18, #VCPU_REGS + 8 * 31]
+.endm
+
+/**
+ * This is a generic handler for exceptions taken at a lower EL. It saves the
+ * volatile registers to the current vCPU and calls the C handler, which can
+ * select one of two paths: (a) restore volatile registers and return, or
+ * (b) switch to a different vCPU. In the latter case, the handler needs to save
+ * all non-volatile registers (they haven't been saved yet), then restore all
+ * registers from the new vCPU.
+ */
+.macro lower_exception handler:req
+	save_volatile_to_vcpu
+
+	/* Call C handler. */
+	bl \handler
+
+	/* Switch vCPU if requested by handler. */
+	cbnz x0, vcpu_switch
+
+	/* vCPU is not changing. */
+	mrs x0, tpidr_el2
+	b vcpu_restore_volatile_and_run
+.endm
+
+/**
+ * This is the handler for a sync exception taken at a lower EL.
+ */
+.macro lower_sync_exception
+	save_volatile_to_vcpu
+
+	/* Extract the exception class (EC) from exception syndrome register. */
+	mrs x18, esr_el2
+	lsr x18, x18, #26
+
+	/* Take the system register path for EC 0x18. */
+	sub x18, x18, #0x18
+	cbz x18, system_register_access
+
+	/* Read syndrome register and call C handler. */
+	mrs x0, esr_el2
+	bl sync_lower_exception
+
+	/* Switch vCPU if requested by handler. */
+	cbnz x0, vcpu_switch
+
+	/* vCPU is not changing. */
+	mrs x0, tpidr_el2
+	b vcpu_restore_volatile_and_run
+.endm
+
+/**
+ * The following is the exception table. A pointer to it will be stored in
+ * register vbar_el2.
+ */
+.section .text.vector_table_el2, "ax"
+.global vector_table_el2
+.balign 0x800
+vector_table_el2:
+sync_cur_sp0:
+	noreturn_current_exception_sp0 el2 sync_current_exception_noreturn
+
+.balign 0x80
+irq_cur_sp0:
+	noreturn_current_exception_sp0 el2 irq_current_exception_noreturn
+
+.balign 0x80
+fiq_cur_sp0:
+	noreturn_current_exception_sp0 el2 fiq_current_exception_noreturn
+
+.balign 0x80
+serr_cur_sp0:
+	noreturn_current_exception_sp0 el2 serr_current_exception_noreturn
+
+.balign 0x80
+sync_cur_spx:
+	noreturn_current_exception_spx el2 sync_current_exception_noreturn
+
+.balign 0x80
+irq_cur_spx:
+	noreturn_current_exception_spx el2 irq_current_exception_noreturn
+
+.balign 0x80
+fiq_cur_spx:
+	noreturn_current_exception_spx el2 fiq_current_exception_noreturn
+
+.balign 0x80
+serr_cur_spx:
+	noreturn_current_exception_spx el2 serr_current_exception_noreturn
+
+.balign 0x80
+sync_lower_64:
+	lower_sync_exception
+
+.balign 0x80
+irq_lower_64:
+	lower_exception irq_lower
+
+.balign 0x80
+fiq_lower_64:
+	lower_exception fiq_lower
+
+.balign 0x80
+serr_lower_64:
+	lower_exception serr_lower
+
+.balign 0x80
+sync_lower_32:
+	lower_sync_exception
+
+.balign 0x80
+irq_lower_32:
+	lower_exception irq_lower
+
+.balign 0x80
+fiq_lower_32:
+	lower_exception fiq_lower
+
+.balign 0x80
+serr_lower_32:
+	lower_exception serr_lower
+
+.balign 0x40
+
+/**
+ * Handle accesses to system registers (EC=0x18) and return to original caller.
+ */
+system_register_access:
+	/*
+	 * Non-volatile registers are (conservatively) saved because the handler
+	 * can clobber non-volatile registers that are used by the msr/mrs,
+	 * which results in the wrong value being read or written.
+	 */
+	/* Get the current vCPU. */
+	mrs x18, tpidr_el2
+	stp x19, x20, [x18, #VCPU_REGS + 8 * 19]
+	stp x21, x22, [x18, #VCPU_REGS + 8 * 21]
+	stp x23, x24, [x18, #VCPU_REGS + 8 * 23]
+	stp x25, x26, [x18, #VCPU_REGS + 8 * 25]
+	stp x27, x28, [x18, #VCPU_REGS + 8 * 27]
+
+	/* Read syndrome register and call C handler. */
+	mrs x0, esr_el2
+	bl handle_system_register_access
+	cbnz x0, vcpu_switch
+
+	/* vCPU is not changing. */
+	mrs x0, tpidr_el2
+	b vcpu_restore_nonvolatile_and_run
+
+/**
+ * Switch to a new vCPU.
+ *
+ * All volatile registers from the old vCPU have already been saved. We need
+ * to save only non-volatile ones from the old vCPU, and restore all from the
+ * new one.
+ *
+ * x0 is a pointer to the new vCPU.
+ */
+vcpu_switch:
+	/* Save non-volatile registers. */
+	mrs x1, tpidr_el2
+	stp x19, x20, [x1, #VCPU_REGS + 8 * 19]
+	stp x21, x22, [x1, #VCPU_REGS + 8 * 21]
+	stp x23, x24, [x1, #VCPU_REGS + 8 * 23]
+	stp x25, x26, [x1, #VCPU_REGS + 8 * 25]
+	stp x27, x28, [x1, #VCPU_REGS + 8 * 27]
+
+	/* Save lazy state. */
+	/* Use x28 as the base */
+	add x28, x1, #VCPU_LAZY
+
+	mrs x24, vmpidr_el2
+	mrs x25, csselr_el1
+	stp x24, x25, [x28], #16
+
+	mrs x2, sctlr_el1
+	mrs x3, actlr_el1
+	stp x2, x3, [x28], #16
+
+	mrs x4, cpacr_el1
+	mrs x5, ttbr0_el1
+	stp x4, x5, [x28], #16
+
+	mrs x6, ttbr1_el1
+	mrs x7, tcr_el1
+	stp x6, x7, [x28], #16
+
+	mrs x8, esr_el1
+	mrs x9, afsr0_el1
+	stp x8, x9, [x28], #16
+
+	mrs x10, afsr1_el1
+	mrs x11, far_el1
+	stp x10, x11, [x28], #16
+
+	mrs x12, mair_el1
+	mrs x13, vbar_el1
+	stp x12, x13, [x28], #16
+
+	mrs x14, contextidr_el1
+	mrs x15, tpidr_el0
+	stp x14, x15, [x28], #16
+
+	mrs x16, tpidrro_el0
+	mrs x17, tpidr_el1
+	stp x16, x17, [x28], #16
+
+	mrs x18, amair_el1
+	mrs x19, cntkctl_el1
+	stp x18, x19, [x28], #16
+
+	mrs x20, sp_el0
+	mrs x21, sp_el1
+	stp x20, x21, [x28], #16
+
+	mrs x22, elr_el1
+	mrs x23, spsr_el1
+	stp x22, x23, [x28], #16
+
+	mrs x24, par_el1
+	mrs x25, hcr_el2
+	stp x24, x25, [x28], #16
+
+	mrs x26, cnthctl_el2
+	mrs x27, vttbr_el2
+	stp x26, x27, [x28], #16
+
+	mrs x4, mdcr_el2
+	mrs x5, mdscr_el1
+	stp x4, x5, [x28], #16
+
+	mrs x6, pmccfiltr_el0
+	mrs x7, pmcr_el0
+	stp x6, x7, [x28], #16
+
+	mrs x8, pmcntenset_el0
+	mrs x9, pmintenset_el1
+	stp x8, x9, [x28], #16
+
+	/* Save GIC registers. */
+#if GIC_VERSION == 3 || GIC_VERSION == 4
+	/* Offset is too large, so start from a new base. */
+	add x2, x1, #VCPU_GIC
+
+	mrs x3, ich_hcr_el2
+	mrs x4, icc_sre_el2
+	stp x3, x4, [x2, #16 * 0]
+#endif
+
+	/* Save floating point registers. */
+	/* Use x28 as the base. */
+	add x28, x1, #VCPU_FREGS
+	stp q0, q1, [x28], #32
+	stp q2, q3, [x28], #32
+	stp q4, q5, [x28], #32
+	stp q6, q7, [x28], #32
+	stp q8, q9, [x28], #32
+	stp q10, q11, [x28], #32
+	stp q12, q13, [x28], #32
+	stp q14, q15, [x28], #32
+	stp q16, q17, [x28], #32
+	stp q18, q19, [x28], #32
+	stp q20, q21, [x28], #32
+	stp q22, q23, [x28], #32
+	stp q24, q25, [x28], #32
+	stp q26, q27, [x28], #32
+	stp q28, q29, [x28], #32
+	stp q30, q31, [x28], #32
+	mrs x3, fpsr
+	mrs x4, fpcr
+	stp x3, x4, [x28], #32
+
+	/* Save new vCPU pointer in non-volatile register. */
+	mov x19, x0
+
+	/*
+	 * Save peripheral registers, and inform the arch-independent sections
+	 * that registers have been saved.
+	 */
+	mov x0, x1
+	bl complete_saving_state
+	mov x0, x19
+
+	/* Intentional fallthrough. */
+.global vcpu_restore_all_and_run
+vcpu_restore_all_and_run:
+	/* Update pointer to current vCPU. */
+	msr tpidr_el2, x0
+
+	/* Restore peripheral registers. */
+	mov x19, x0
+	bl begin_restoring_state
+	mov x0, x19
+
+	/*
+	 * Restore floating point registers.
+	 *
+	 * Offset is too large, so start from a new base.
+	 */
+	add x2, x0, #VCPU_FREGS
+	ldp q0, q1, [x2, #32 * 0]
+	ldp q2, q3, [x2, #32 * 1]
+	ldp q4, q5, [x2, #32 * 2]
+	ldp q6, q7, [x2, #32 * 3]
+	ldp q8, q9, [x2, #32 * 4]
+	ldp q10, q11, [x2, #32 * 5]
+	ldp q12, q13, [x2, #32 * 6]
+	ldp q14, q15, [x2, #32 * 7]
+	ldp q16, q17, [x2, #32 * 8]
+	ldp q18, q19, [x2, #32 * 9]
+	ldp q20, q21, [x2, #32 * 10]
+	ldp q22, q23, [x2, #32 * 11]
+	ldp q24, q25, [x2, #32 * 12]
+	ldp q26, q27, [x2, #32 * 13]
+	ldp q28, q29, [x2, #32 * 14]
+	/* Offset becomes too large, so move the base. */
+	ldp q30, q31, [x2, #32 * 15]!
+	ldp x3, x4, [x2, #32 * 1]
+	msr fpsr, x3
+
+	/*
+	 * Only restore FPCR if changed, to avoid expensive
+	 * self-synchronising operation where possible.
+	 */
+	mrs x5, fpcr
+	cmp x5, x4
+	b.eq vcpu_restore_lazy_and_run
+	msr fpcr, x4
+	/* Intentional fallthrough. */
+
+vcpu_restore_lazy_and_run:
+	/* Restore lazy registers. */
+	/* Use x28 as the base. */
+	add x28, x0, #VCPU_LAZY
+
+	ldp x24, x25, [x28], #16
+	msr vmpidr_el2, x24
+	msr csselr_el1, x25
+
+	ldp x2, x3, [x28], #16
+	msr sctlr_el1, x2
+	msr actlr_el1, x3
+
+	ldp x4, x5, [x28], #16
+	msr cpacr_el1, x4
+	msr ttbr0_el1, x5
+
+	ldp x6, x7, [x28], #16
+	msr ttbr1_el1, x6
+	msr tcr_el1, x7
+
+	ldp x8, x9, [x28], #16
+	msr esr_el1, x8
+	msr afsr0_el1, x9
+
+	ldp x10, x11, [x28], #16
+	msr afsr1_el1, x10
+	msr far_el1, x11
+
+	ldp x12, x13, [x28], #16
+	msr mair_el1, x12
+	msr vbar_el1, x13
+
+	ldp x14, x15, [x28], #16
+	msr contextidr_el1, x14
+	msr tpidr_el0, x15
+
+	ldp x16, x17, [x28], #16
+	msr tpidrro_el0, x16
+	msr tpidr_el1, x17
+
+	ldp x18, x19, [x28], #16
+	msr amair_el1, x18
+	msr cntkctl_el1, x19
+
+	ldp x20, x21, [x28], #16
+	msr sp_el0, x20
+	msr sp_el1, x21
+
+	ldp x22, x23, [x28], #16
+	msr elr_el1, x22
+	msr spsr_el1, x23
+
+	ldp x24, x25, [x28], #16
+	msr par_el1, x24
+	msr hcr_el2, x25
+
+	ldp x26, x27, [x28], #16
+	msr cnthctl_el2, x26
+	msr vttbr_el2, x27
+
+	ldp x4, x5, [x28], #16
+	msr mdcr_el2, x4
+	msr mdscr_el1, x5
+
+	ldp x6, x7, [x28], #16
+	msr pmccfiltr_el0, x6
+	msr pmcr_el0, x7
+
+	ldp x8, x9, [x28], #16
+	/*
+	 * NOTE: Writing 0s to pmcntenset_el0's bits do not alter their values.
+	 * To reset them, clear the register by writing to pmcntenclr_el0.
+	 */
+	mov x27, #0xffffffff
+	msr pmcntenclr_el0, x27
+	msr pmcntenset_el0, x8
+
+	/*
+	 * NOTE: Writing 0s to pmintenset_el1's bits do not alter their values.
+	 * To reset them, clear the register by writing to pmintenclr_el1.
+	 */
+	msr pmintenclr_el1, x27
+	msr pmintenset_el1, x9
+
+	/* Restore GIC registers. */
+#if GIC_VERSION == 3 || GIC_VERSION == 4
+	/* Offset is too large, so start from a new base. */
+	add x2, x0, #VCPU_GIC
+
+	ldp x3, x4, [x2, #16 * 0]
+	msr ich_hcr_el2, x3
+	msr icc_sre_el2, x4
+#endif
+
+	/*
+	 * If a different vCPU is being run on this physical CPU to the last one
+	 * which was run for this VM, invalidate the TLB. This must be called
+	 * after vttbr_el2 has been updated, so that we have the page table and
+	 * VMID of the vCPU to which we are switching.
+	 */
+	mov x19, x0
+	bl maybe_invalidate_tlb
+	mov x0, x19
+
+	/* Intentional fallthrough. */
+
+vcpu_restore_nonvolatile_and_run:
+	/* Restore non-volatile registers. */
+	ldp x19, x20, [x0, #VCPU_REGS + 8 * 19]
+	ldp x21, x22, [x0, #VCPU_REGS + 8 * 21]
+	ldp x23, x24, [x0, #VCPU_REGS + 8 * 23]
+	ldp x25, x26, [x0, #VCPU_REGS + 8 * 25]
+	ldp x27, x28, [x0, #VCPU_REGS + 8 * 27]
+
+	/* Intentional fallthrough. */
+/**
+ * Restore volatile registers and run the given vCPU.
+ *
+ * x0 is a pointer to the target vCPU.
+ */
+vcpu_restore_volatile_and_run:
+	ldp x4, x5, [x0, #VCPU_REGS + 8 * 4]
+	ldp x6, x7, [x0, #VCPU_REGS + 8 * 6]
+	ldp x8, x9, [x0, #VCPU_REGS + 8 * 8]
+	ldp x10, x11, [x0, #VCPU_REGS + 8 * 10]
+	ldp x12, x13, [x0, #VCPU_REGS + 8 * 12]
+	ldp x14, x15, [x0, #VCPU_REGS + 8 * 14]
+	ldp x16, x17, [x0, #VCPU_REGS + 8 * 16]
+	ldr x18, [x0, #VCPU_REGS + 8 * 18]
+	ldp x29, x30, [x0, #VCPU_REGS + 8 * 29]
+
+	/* Restore return address & mode. */
+	ldp x1, x2, [x0, #VCPU_REGS + 8 * 31]
+	msr elr_el2, x1
+	msr spsr_el2, x2
+
+	/* Restore x0..x3, which we have used as scratch before. */
+	ldp x2, x3, [x0, #VCPU_REGS + 8 * 2]
+	ldp x0, x1, [x0, #VCPU_REGS + 8 * 0]
+	eret_with_sb
+
+.balign 0x40
+/**
+ * Restore volatile registers from stack and return to original caller.
+ */
+restore_from_stack_and_return:
+	restore_volatile_from_stack el2
+	eret_with_sb
diff --git a/src/arch/aarch64/hypervisor/feature_id.c b/src/arch/aarch64/hypervisor/feature_id.c
new file mode 100644
index 0000000..20e6f55
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/feature_id.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "feature_id.h"
+
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/types.h"
+#include "hf/vm.h"
+
+#include "msr.h"
+#include "sysregs.h"
+
+/* clang-format off */
+
+/**
+ * Definitions of read-only feature ID (group 3) registers' encodings.
+ * See Arm Architecture Reference Manual Armv8-A, Table D1-52 and D12-2.
+ * NAME, op0, op1, crn, crm, op2
+ */
+#define FEATURE_ID_REGISTERS_READ              \
+	X(ID_PFR0_EL1       , 3, 0,  0,  1, 0) \
+	X(ID_PFR1_EL1       , 3, 0,  0,  1, 1) \
+	X(ID_DFR0_EL1       , 3, 0,  0,  1, 2) \
+	X(ID_AFR0_EL1       , 3, 0,  0,  1, 3) \
+	X(ID_MMFR0_EL1      , 3, 0,  0,  1, 4) \
+	X(ID_MMFR1_EL1      , 3, 0,  0,  1, 5) \
+	X(ID_MMFR2_EL1      , 3, 0,  0,  1, 6) \
+	X(ID_MMFR3_EL1      , 3, 0,  0,  1, 7) \
+	X(ID_ISAR0_EL1      , 3, 0,  0,  2, 0) \
+	X(ID_ISAR1_EL1      , 3, 0,  0,  2, 1) \
+	X(ID_ISAR2_EL1      , 3, 0,  0,  2, 2) \
+	X(ID_ISAR3_EL1      , 3, 0,  0,  2, 3) \
+	X(ID_ISAR4_EL1      , 3, 0,  0,  2, 4) \
+	X(ID_ISAR5_EL1      , 3, 0,  0,  2, 5) \
+	X(ID_MMFR4_EL1      , 3, 0,  0,  2, 6) \
+	\
+	X(MVFR0_EL1         , 3, 0,  0,  3, 0) \
+	X(MVFR1_EL1         , 3, 0,  0,  3, 1) \
+	X(MVFR2_EL1         , 3, 0,  0,  3, 2) \
+	\
+	X(ID_AA64PFR0_EL1   , 3, 0,  0,  4, 0) \
+	X(ID_AA64PFR1_EL1   , 3, 0,  0,  4, 1) \
+	\
+	X(ID_AA64DFR0_EL1   , 3, 0,  0,  5, 0) \
+	X(ID_AA64DFR1_EL1   , 3, 0,  0,  5, 1) \
+	\
+	X(ID_AA64AFR0_EL1   , 3, 0,  0,  5, 4) \
+	X(ID_AA64AFR1_EL1   , 3, 0,  0,  5, 5) \
+	\
+	X(ID_AA64ISAR0_EL1  , 3, 0,  0,  6, 0) \
+	X(ID_AA64ISAR1_EL1  , 3, 0,  0,  6, 1) \
+	\
+	X(ID_AA64MMFR0_EL1  , 3, 0,  0,  7, 0) \
+	X(ID_AA64MMFR1_EL1  , 3, 0,  0,  7, 1) \
+	X(ID_AA64MMFR2_EL1  , 3, 0,  0,  7, 2)
+
+/* clang-format on */
+
+enum {
+#define X(reg_name, op0, op1, crn, crm, op2) \
+	reg_name##_ENC = GET_ISS_ENCODING(op0, op1, crn, crm, op2),
+	FEATURE_ID_REGISTERS_READ
+#undef X
+};
+
+/**
+ * Returns true if the ESR register shows an access to a feature ID group 3
+ * register.
+ */
+bool feature_id_is_register_access(uintreg_t esr)
+{
+	uintreg_t op0 = GET_ISS_OP0(esr);
+	uintreg_t op1 = GET_ISS_OP1(esr);
+	uintreg_t crn = GET_ISS_CRN(esr);
+	uintreg_t crm = GET_ISS_CRM(esr);
+
+	/* From the Arm Architecture Reference Manual Table D12-2. */
+	return op0 == 3 && op1 == 0 && crn == 0 && crm >= 1 && crm <= 7;
+}
+
+/**
+ * RAS-related. RES0 when RAS is not implemented.
+ */
+#define ID_AA64MMFR1_EL1_SPEC_SEI (UINT64_C(0xf) << 24)
+
+/**
+ * Indicates support for LORegions.
+ */
+#define ID_AA64MMFR1_EL1_LO (UINT64_C(0xf) << 24)
+
+/**
+ * RAS Extension version.
+ */
+#define ID_AA64PFR0_EL1_RAS (UINT64_C(0xf) << 28)
+
+/**
+ * Self-hosted Trace Extension Version
+ */
+#define ID_AA64DFR0_EL1_TRACE_FILT (UINT64_C(0xf) << 40)
+
+/**
+ * OS Double Lock implemented.
+ */
+#define ID_AA64DFR0_EL1_DOUBLE_LOCK (UINT64_C(0xf) << 36)
+
+/**
+ * Statistical Profiling Extension version.
+ */
+#define ID_AA64DFR0_EL1_PMS_VER (UINT64_C(0xf) << 32)
+
+/**
+ * Performance Monitors Extension version.
+ */
+#define ID_AA64DFR0_EL1_PMU_VER (UINT64_C(0xf) << 8)
+
+/**
+ * Indicates whether System register interface to trace unit is implemented.
+ */
+#define ID_AA64DFR0_EL1_TRACE_VER (UINT64_C(0xf) << 4)
+
+/**
+ * Debug architecture version.
+ */
+#define ID_AA64DFR0_EL1_DEBUG_VER (UINT64_C(0xf))
+
+/**
+ * PAuth: whether an implementation defined algorithm for generic code
+ * authentication is implemented.
+ */
+#define ID_AA64ISAR1_EL1_GPI (UINT64_C(0xf) << 28)
+
+/**
+ * PAuth: whether QARMA or Architected algorithm for generic code authentication
+ * is implemented.
+ */
+#define ID_AA64ISAR1_EL1_GPA (UINT64_C(0xf) << 24)
+
+/**
+ * PAuth: whether an implementation defined algorithm for address authentication
+ * is implemented.
+ */
+#define ID_AA64ISAR1_EL1_API (UINT64_C(0xf) << 8)
+
+/**
+ * PAuth: whether QARMA or Architected algorithm for address authentication is
+ * implemented.
+ */
+#define ID_AA64ISAR1_EL1_APA (UINT64_C(0xf) << 24)
+
+void feature_set_traps(struct vm *vm, struct arch_regs *regs)
+{
+	arch_features_t features = vm->arch.trapped_features;
+
+	if (features & ~HF_FEATURE_ALL) {
+		panic("features has undefined bits 0x%x", features);
+	}
+
+	/* By default do not mask out any features. */
+	vm->arch.tid3_masks.id_aa64mmfr1_el1 = ~0ULL;
+	vm->arch.tid3_masks.id_aa64pfr0_el1 = ~0ULL;
+	vm->arch.tid3_masks.id_aa64dfr0_el1 = ~0ULL;
+	vm->arch.tid3_masks.id_aa64isar1_el1 = ~0ULL;
+
+	if (features & HF_FEATURE_RAS) {
+		regs->lazy.hcr_el2 |= HCR_EL2_TERR;
+		vm->arch.tid3_masks.id_aa64mmfr1_el1 &=
+			~ID_AA64MMFR1_EL1_SPEC_SEI;
+		vm->arch.tid3_masks.id_aa64pfr0_el1 &= ~ID_AA64PFR0_EL1_RAS;
+	}
+
+	if (features & HF_FEATURE_SPE) {
+		/*
+		 * Trap VM accesses to Statistical Profiling Extension (SPE)
+		 * registers.
+		 */
+		regs->lazy.mdcr_el2 |= MDCR_EL2_TPMS;
+
+		/*
+		 * Set E2PB to 0b00. This ensures that accesses to Profiling
+		 * Buffer controls at EL1 are trapped to EL2.
+		 */
+		regs->lazy.mdcr_el2 &= ~MDCR_EL2_E2PB;
+
+		vm->arch.tid3_masks.id_aa64dfr0_el1 &= ~ID_AA64DFR0_EL1_PMS_VER;
+	}
+
+	if (features & HF_FEATURE_DEBUG) {
+		regs->lazy.mdcr_el2 |=
+			MDCR_EL2_TDRA | MDCR_EL2_TDOSA | MDCR_EL2_TDA;
+
+		vm->arch.tid3_masks.id_aa64dfr0_el1 &=
+			~ID_AA64DFR0_EL1_DOUBLE_LOCK;
+	}
+
+	if (features & HF_FEATURE_TRACE) {
+		regs->lazy.mdcr_el2 |= MDCR_EL2_TTRF;
+
+		vm->arch.tid3_masks.id_aa64dfr0_el1 &=
+			~ID_AA64DFR0_EL1_TRACE_FILT;
+		vm->arch.tid3_masks.id_aa64dfr0_el1 &=
+			~ID_AA64DFR0_EL1_TRACE_VER;
+	}
+
+	if (features & HF_FEATURE_PERFMON) {
+		regs->lazy.mdcr_el2 |= MDCR_EL2_TPM | MDCR_EL2_TPMCR;
+
+		vm->arch.tid3_masks.id_aa64dfr0_el1 &= ~ID_AA64DFR0_EL1_PMU_VER;
+	}
+
+	if (features & HF_FEATURE_LOR) {
+		regs->lazy.hcr_el2 |= HCR_EL2_TLOR;
+
+		vm->arch.tid3_masks.id_aa64mmfr1_el1 &= ~ID_AA64MMFR1_EL1_LO;
+	}
+
+	if (features & HF_FEATURE_PAUTH) {
+		/* APK and API bits *enable* trapping when cleared.  */
+		regs->lazy.hcr_el2 &= ~(HCR_EL2_APK | HCR_EL2_API);
+
+		vm->arch.tid3_masks.id_aa64isar1_el1 &= ~ID_AA64ISAR1_EL1_GPI;
+		vm->arch.tid3_masks.id_aa64isar1_el1 &= ~ID_AA64ISAR1_EL1_GPA;
+		vm->arch.tid3_masks.id_aa64isar1_el1 &= ~ID_AA64ISAR1_EL1_API;
+		vm->arch.tid3_masks.id_aa64isar1_el1 &= ~ID_AA64ISAR1_EL1_APA;
+	}
+}
+
+/**
+ * Processes an access (mrs) to a feature ID register.
+ * Returns true if the access was allowed and performed, false otherwise.
+ */
+bool feature_id_process_access(struct vcpu *vcpu, uintreg_t esr)
+{
+	const struct vm *vm = vcpu->vm;
+	uintreg_t sys_register = GET_ISS_SYSREG(esr);
+	uintreg_t rt_register = GET_ISS_RT(esr);
+	uintreg_t value;
+
+	/* +1 because Rt can access register XZR */
+	CHECK(rt_register < NUM_GP_REGS + 1);
+
+	if (!ISS_IS_READ(esr)) {
+		dlog("Unsupported feature ID register write: "
+		     "op0=%d, op1=%d, crn=%d, crm=%d, op2=%d, rt=%d.\n",
+		     GET_ISS_OP0(esr), GET_ISS_OP1(esr), GET_ISS_CRN(esr),
+		     GET_ISS_CRM(esr), GET_ISS_OP2(esr), GET_ISS_RT(esr));
+		return true;
+	}
+
+	switch (sys_register) {
+#define X(reg_name, op0, op1, crn, crm, op2)              \
+	case (GET_ISS_ENCODING(op0, op1, crn, crm, op2)): \
+		value = read_msr(reg_name);               \
+		break;
+		FEATURE_ID_REGISTERS_READ
+#undef X
+	default:
+		/* Reserved registers should be read as zero (raz). */
+		value = 0;
+		dlog("Unsupported feature ID register read: "
+		     "op0=%d, op1=%d, crn=%d, crm=%d, op2=%d, rt=%d.\n",
+		     GET_ISS_OP0(esr), GET_ISS_OP1(esr), GET_ISS_CRN(esr),
+		     GET_ISS_CRM(esr), GET_ISS_OP2(esr), GET_ISS_RT(esr));
+		break;
+	}
+
+	/* Mask values for features Hafnium might restrict. */
+	switch (sys_register) {
+	case ID_AA64MMFR1_EL1_ENC:
+		value &= vm->arch.tid3_masks.id_aa64mmfr1_el1;
+		break;
+	case ID_AA64PFR0_EL1_ENC:
+		value &= vm->arch.tid3_masks.id_aa64pfr0_el1;
+		break;
+	case ID_AA64DFR0_EL1_ENC:
+		value &= vm->arch.tid3_masks.id_aa64dfr0_el1;
+		break;
+	case ID_AA64ISAR1_EL1_ENC:
+		value &= vm->arch.tid3_masks.id_aa64isar1_el1;
+		break;
+	default:
+		break;
+	}
+
+	if (rt_register != RT_REG_XZR) {
+		vcpu->regs.r[rt_register] = value;
+	}
+
+	return true;
+}
diff --git a/src/arch/aarch64/hypervisor/feature_id.h b/src/arch/aarch64/hypervisor/feature_id.h
new file mode 100644
index 0000000..86c7c01
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/feature_id.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/arch/types.h"
+
+#include "hf/cpu.h"
+
+#include "vmapi/hf/spci.h"
+
+#define HF_FEATURE_NONE UINT64_C(0)
+
+/*  Reliability, Availability, and Serviceability (RAS) Extension Features */
+#define HF_FEATURE_RAS UINT64_C(1)
+
+/* Limited Ordering Regions */
+#define HF_FEATURE_LOR (UINT64_C(1) << 1)
+
+/* Performance Monitor */
+#define HF_FEATURE_PERFMON (UINT64_C(1) << 2)
+
+/* Debug Registers */
+#define HF_FEATURE_DEBUG (UINT64_C(1) << 3)
+
+/* Statistical Profiling Extension (SPE) */
+#define HF_FEATURE_SPE (UINT64_C(1) << 4)
+
+/* Self-hosted Trace */
+#define HF_FEATURE_TRACE (UINT64_C(1) << 5)
+
+/* Pointer Authentication (PAuth) */
+#define HF_FEATURE_PAUTH (UINT64_C(1) << 6)
+
+/*
+ * NOTE: This should be based on the last (highest value) defined feature.
+ * Adjust if adding more features.
+ */
+#define HF_FEATURE_ALL ((HF_FEATURE_PAUTH << 1) - 1)
+
+bool feature_id_is_register_access(uintreg_t esr_el2);
+
+bool feature_id_process_access(struct vcpu *vcpu, uintreg_t esr_el2);
+
+void feature_set_traps(struct vm *vm, struct arch_regs *regs);
diff --git a/src/arch/aarch64/hypervisor/handler.c b/src/arch/aarch64/hypervisor/handler.c
new file mode 100644
index 0000000..65aef52
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/handler.c
@@ -0,0 +1,767 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdnoreturn.h>
+
+#include "hf/arch/barriers.h"
+#include "hf/arch/init.h"
+#include "hf/arch/mm.h"
+#include "hf/arch/plat/smc.h"
+
+#include "hf/api.h"
+#include "hf/check.h"
+#include "hf/cpu.h"
+#include "hf/dlog.h"
+#include "hf/panic.h"
+#include "hf/spci.h"
+#include "hf/vm.h"
+
+#include "vmapi/hf/call.h"
+
+#include "debug_el1.h"
+#include "feature_id.h"
+#include "msr.h"
+#include "perfmon.h"
+#include "psci.h"
+#include "psci_handler.h"
+#include "smc.h"
+#include "sysregs.h"
+
+/**
+ * Gets the Exception Class from the ESR.
+ */
+#define GET_ESR_EC(esr) ((esr) >> 26)
+
+/**
+ * Gets the Instruction Length bit for the synchronous exception
+ */
+#define GET_ESR_IL(esr) ((esr) & (1 << 25))
+
+/**
+ * Gets the value to increment for the next PC.
+ * The ESR encodes whether the instruction is 2 bytes or 4 bytes long.
+ */
+#define GET_NEXT_PC_INC(esr) (GET_ESR_IL(esr) ? 4 : 2)
+
+/**
+ * The Client ID field within X7 for an SMC64 call.
+ */
+#define CLIENT_ID_MASK UINT64_C(0xffff)
+
+/**
+ * Returns a reference to the currently executing vCPU.
+ */
+static struct vcpu *current(void)
+{
+	return (struct vcpu *)read_msr(tpidr_el2);
+}
+
+/**
+ * Saves the state of per-vCPU peripherals, such as the virtual timer, and
+ * informs the arch-independent sections that registers have been saved.
+ */
+void complete_saving_state(struct vcpu *vcpu)
+{
+	vcpu->regs.peripherals.cntv_cval_el0 = read_msr(cntv_cval_el0);
+	vcpu->regs.peripherals.cntv_ctl_el0 = read_msr(cntv_ctl_el0);
+
+	api_regs_state_saved(vcpu);
+
+	/*
+	 * If switching away from the primary, copy the current EL0 virtual
+	 * timer registers to the corresponding EL2 physical timer registers.
+	 * This is used to emulate the virtual timer for the primary in case it
+	 * should fire while the secondary is running.
+	 */
+	if (vcpu->vm->id == HF_PRIMARY_VM_ID) {
+		/*
+		 * Clear timer control register before copying compare value, to
+		 * avoid a spurious timer interrupt. This could be a problem if
+		 * the interrupt is configured as edge-triggered, as it would
+		 * then be latched in.
+		 */
+		write_msr(cnthp_ctl_el2, 0);
+		write_msr(cnthp_cval_el2, read_msr(cntv_cval_el0));
+		write_msr(cnthp_ctl_el2, read_msr(cntv_ctl_el0));
+	}
+}
+
+/**
+ * Restores the state of per-vCPU peripherals, such as the virtual timer.
+ */
+void begin_restoring_state(struct vcpu *vcpu)
+{
+	/*
+	 * Clear timer control register before restoring compare value, to avoid
+	 * a spurious timer interrupt. This could be a problem if the interrupt
+	 * is configured as edge-triggered, as it would then be latched in.
+	 */
+	write_msr(cntv_ctl_el0, 0);
+	write_msr(cntv_cval_el0, vcpu->regs.peripherals.cntv_cval_el0);
+	write_msr(cntv_ctl_el0, vcpu->regs.peripherals.cntv_ctl_el0);
+
+	/*
+	 * If we are switching (back) to the primary, disable the EL2 physical
+	 * timer which was being used to emulate the EL0 virtual timer, as the
+	 * virtual timer is now running for the primary again.
+	 */
+	if (vcpu->vm->id == HF_PRIMARY_VM_ID) {
+		write_msr(cnthp_ctl_el2, 0);
+		write_msr(cnthp_cval_el2, 0);
+	}
+}
+
+/**
+ * Invalidate all stage 1 TLB entries on the current (physical) CPU for the
+ * current VMID.
+ */
+static void invalidate_vm_tlb(void)
+{
+	/*
+	 * Ensure that the last VTTBR write has taken effect so we invalidate
+	 * the right set of TLB entries.
+	 */
+	isb();
+
+	__asm__ volatile("tlbi vmalle1");
+
+	/*
+	 * Ensure that no instructions are fetched for the VM until after the
+	 * TLB invalidation has taken effect.
+	 */
+	isb();
+
+	/*
+	 * Ensure that no data reads or writes for the VM happen until after the
+	 * TLB invalidation has taken effect. Non-shareable is enough because
+	 * the TLB is local to the CPU.
+	 */
+	dsb(nsh);
+}
+
+/**
+ * Invalidates the TLB if a different vCPU is being run than the last vCPU of
+ * the same VM which was run on the current pCPU.
+ *
+ * This is necessary because VMs may (contrary to the architecture
+ * specification) use inconsistent ASIDs across vCPUs. c.f. KVM's similar
+ * workaround:
+ * https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=94d0e5980d6791b9
+ */
+void maybe_invalidate_tlb(struct vcpu *vcpu)
+{
+	size_t current_cpu_index = cpu_index(vcpu->cpu);
+	spci_vcpu_index_t new_vcpu_index = vcpu_index(vcpu);
+
+	if (vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] !=
+	    new_vcpu_index) {
+		/*
+		 * The vCPU has changed since the last time this VM was run on
+		 * this pCPU, so we need to invalidate the TLB.
+		 */
+		invalidate_vm_tlb();
+
+		/* Record the fact that this vCPU is now running on this CPU. */
+		vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] =
+			new_vcpu_index;
+	}
+}
+
+noreturn void irq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
+{
+	(void)elr;
+	(void)spsr;
+
+	panic("IRQ from current");
+}
+
+noreturn void fiq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
+{
+	(void)elr;
+	(void)spsr;
+
+	panic("FIQ from current");
+}
+
+noreturn void serr_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
+{
+	(void)elr;
+	(void)spsr;
+
+	panic("SERR from current");
+}
+
+noreturn void sync_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
+{
+	uintreg_t esr = read_msr(esr_el2);
+	uintreg_t ec = GET_ESR_EC(esr);
+
+	(void)spsr;
+
+	switch (ec) {
+	case 0x25: /* EC = 100101, Data abort. */
+		dlog("Data abort: pc=%#x, esr=%#x, ec=%#x", elr, esr, ec);
+		if (!(esr & (1U << 10))) { /* Check FnV bit. */
+			dlog(", far=%#x", read_msr(far_el2));
+		} else {
+			dlog(", far=invalid");
+		}
+
+		dlog("\n");
+		break;
+
+	default:
+		dlog("Unknown current sync exception pc=%#x, esr=%#x, "
+		     "ec=%#x\n",
+		     elr, esr, ec);
+		break;
+	}
+
+	panic("EL2 exception");
+}
+
+/**
+ * Sets or clears the VI bit in the HCR_EL2 register saved in the given
+ * arch_regs.
+ */
+static void set_virtual_interrupt(struct arch_regs *r, bool enable)
+{
+	if (enable) {
+		r->lazy.hcr_el2 |= HCR_EL2_VI;
+	} else {
+		r->lazy.hcr_el2 &= ~HCR_EL2_VI;
+	}
+}
+
+/**
+ * Sets or clears the VI bit in the HCR_EL2 register.
+ */
+static void set_virtual_interrupt_current(bool enable)
+{
+	uintreg_t hcr_el2 = read_msr(hcr_el2);
+
+	if (enable) {
+		hcr_el2 |= HCR_EL2_VI;
+	} else {
+		hcr_el2 &= ~HCR_EL2_VI;
+	}
+	write_msr(hcr_el2, hcr_el2);
+}
+
+/**
+ * Checks whether to block an SMC being forwarded from a VM.
+ */
+static bool smc_is_blocked(const struct vm *vm, uint32_t func)
+{
+	bool block_by_default = !vm->smc_whitelist.permissive;
+
+	for (size_t i = 0; i < vm->smc_whitelist.smc_count; ++i) {
+		if (func == vm->smc_whitelist.smcs[i]) {
+			return false;
+		}
+	}
+
+	dlog("SMC %#010x attempted from VM %d, blocked=%d\n", func, vm->id,
+	     block_by_default);
+
+	/* Access is still allowed in permissive mode. */
+	return block_by_default;
+}
+
+/**
+ * Applies SMC access control according to manifest and forwards the call if
+ * access is granted.
+ */
+static void smc_forwarder(const struct vm *vm, struct spci_value *args)
+{
+	struct spci_value ret;
+	uint32_t client_id = vm->id;
+	uintreg_t arg7 = args->arg7;
+
+	if (smc_is_blocked(vm, args->func)) {
+		args->func = SMCCC_ERROR_UNKNOWN;
+		return;
+	}
+
+	/*
+	 * Set the Client ID but keep the existing Secure OS ID and anything
+	 * else (currently unspecified) that the client may have passed in the
+	 * upper bits.
+	 */
+	args->arg7 = client_id | (arg7 & ~CLIENT_ID_MASK);
+	ret = smc_forward(args->func, args->arg1, args->arg2, args->arg3,
+			  args->arg4, args->arg5, args->arg6, args->arg7);
+
+	/*
+	 * Preserve the value passed by the caller, rather than the generated
+	 * client_id. Note that this would also overwrite any return value that
+	 * may be in x7, but the SMCs that we are forwarding are legacy calls
+	 * from before SMCCC 1.2 so won't have more than 4 return values anyway.
+	 */
+	ret.arg7 = arg7;
+
+	plat_smc_post_forward(*args, &ret);
+
+	*args = ret;
+}
+
+static bool spci_handler(struct spci_value *args, struct vcpu **next)
+{
+	/*
+	 * NOTE: When adding new methods to this handler update
+	 * api_spci_features accordingly.
+	 */
+	switch (args->func & ~SMCCC_CONVENTION_MASK) {
+	case SPCI_VERSION_32:
+		*args = api_spci_version();
+		return true;
+	case SPCI_ID_GET_32:
+		*args = api_spci_id_get(current());
+		return true;
+	case SPCI_FEATURES_32:
+		*args = api_spci_features(args->arg1);
+		return true;
+	case SPCI_RX_RELEASE_32:
+		*args = api_spci_rx_release(current(), next);
+		return true;
+	case SPCI_RXTX_MAP_32:
+		*args = api_spci_rxtx_map(ipa_init(args->arg1),
+					  ipa_init(args->arg2), args->arg3,
+					  current(), next);
+		return true;
+	case SPCI_YIELD_32:
+		api_yield(current(), next);
+
+		/* SPCI_YIELD always returns SPCI_SUCCESS. */
+		*args = (struct spci_value){.func = SPCI_SUCCESS_32};
+
+		return true;
+	case SPCI_MSG_SEND_32:
+		*args = api_spci_msg_send(spci_msg_send_sender(*args),
+					  spci_msg_send_receiver(*args),
+					  spci_msg_send_size(*args),
+					  spci_msg_send_attributes(*args),
+					  current(), next);
+		return true;
+	case SPCI_MSG_WAIT_32:
+		*args = api_spci_msg_recv(true, current(), next);
+		return true;
+	case SPCI_MSG_POLL_32:
+		*args = api_spci_msg_recv(false, current(), next);
+		return true;
+	case SPCI_RUN_32:
+		*args = api_spci_run(spci_vm_id(*args), spci_vcpu_index(*args),
+				     current(), next);
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * Set or clear VI bit according to pending interrupts.
+ */
+static void update_vi(struct vcpu *next)
+{
+	if (next == NULL) {
+		/*
+		 * Not switching vCPUs, set the bit for the current vCPU
+		 * directly in the register.
+		 */
+		struct vcpu *vcpu = current();
+
+		sl_lock(&vcpu->lock);
+		set_virtual_interrupt_current(
+			vcpu->interrupts.enabled_and_pending_count > 0);
+		sl_unlock(&vcpu->lock);
+	} else {
+		/*
+		 * About to switch vCPUs, set the bit for the vCPU to which we
+		 * are switching in the saved copy of the register.
+		 */
+		sl_lock(&next->lock);
+		set_virtual_interrupt(
+			&next->regs,
+			next->interrupts.enabled_and_pending_count > 0);
+		sl_unlock(&next->lock);
+	}
+}
+
+/**
+ * Processes SMC instruction calls.
+ */
+static struct vcpu *smc_handler(struct vcpu *vcpu)
+{
+	struct spci_value args = {
+		.func = vcpu->regs.r[0],
+		.arg1 = vcpu->regs.r[1],
+		.arg2 = vcpu->regs.r[2],
+		.arg3 = vcpu->regs.r[3],
+		.arg4 = vcpu->regs.r[4],
+		.arg5 = vcpu->regs.r[5],
+		.arg6 = vcpu->regs.r[6],
+		.arg7 = vcpu->regs.r[7],
+	};
+	struct vcpu *next = NULL;
+
+	if (psci_handler(vcpu, args.func, args.arg1, args.arg2, args.arg3,
+			 &vcpu->regs.r[0], &next)) {
+		return next;
+	}
+
+	if (spci_handler(&args, &next)) {
+		arch_regs_set_retval(&vcpu->regs, args);
+		update_vi(next);
+		return next;
+	}
+
+	switch (args.func & ~SMCCC_CONVENTION_MASK) {
+	case HF_DEBUG_LOG:
+		vcpu->regs.r[0] = api_debug_log(args.arg1, vcpu);
+		return NULL;
+	}
+
+	smc_forwarder(vcpu->vm, &args);
+	arch_regs_set_retval(&vcpu->regs, args);
+	return NULL;
+}
+
+/*
+ * Exception vector offsets.
+ * See Arm Architecture Reference Manual Armv8-A, D1.10.2.
+ */
+
+/**
+ * Offset for synchronous exceptions at current EL with SPx.
+ */
+#define OFFSET_CURRENT_SPX UINT64_C(0x200)
+
+/**
+ * Offset for synchronous exceptions at lower EL using AArch64.
+ */
+#define OFFSET_LOWER_EL_64 UINT64_C(0x400)
+
+/**
+ * Offset for synchronous exceptions at lower EL using AArch32.
+ */
+#define OFFSET_LOWER_EL_32 UINT64_C(0x600)
+
+/**
+ * Returns the address for the exception handler at EL1.
+ */
+static uintreg_t get_el1_exception_handler_addr(const struct vcpu *vcpu)
+{
+	uintreg_t base_addr = read_msr(vbar_el1);
+	uintreg_t pe_mode = vcpu->regs.spsr & PSR_PE_MODE_MASK;
+	bool is_arch32 = vcpu->regs.spsr & PSR_ARCH_MODE_32;
+
+	if (pe_mode == PSR_PE_MODE_EL0T) {
+		if (is_arch32) {
+			base_addr += OFFSET_LOWER_EL_32;
+		} else {
+			base_addr += OFFSET_LOWER_EL_64;
+		}
+	} else {
+		CHECK(!is_arch32);
+		base_addr += OFFSET_CURRENT_SPX;
+	}
+
+	return base_addr;
+}
+
+/**
+ * Injects an exception with an unknown reason (EC=0x0) to the EL1.
+ * See Arm Architecture Reference Manual Armv8-A, page D13-2924.
+ *
+ * NOTE: This function assumes that the lazy registers haven't been saved, and
+ * writes to the lazy registers of the CPU directly instead of the vCPU.
+ */
+static struct vcpu *inject_el1_unknown_exception(struct vcpu *vcpu,
+						 uintreg_t esr_el2)
+{
+	uintreg_t esr_el1_value = GET_ESR_IL(esr_el2);
+	uintreg_t handler_address = get_el1_exception_handler_addr(vcpu);
+	char *direction_str;
+
+	/* Update the CPU state to inject the exception. */
+	write_msr(esr_el1, esr_el1_value);
+	write_msr(elr_el1, vcpu->regs.pc);
+	write_msr(spsr_el1, vcpu->regs.spsr);
+
+	/*
+	 * Mask (disable) interrupts and run in EL1h mode.
+	 * EL1h mode is used because by default, taking an exception selects the
+	 * stack pointer for the target Exception level. The software can change
+	 * that later in the handler if needed.
+	 * See Arm Architecture Reference Manual Armv8-A, page D13-2924
+	 */
+	vcpu->regs.spsr = PSR_D | PSR_A | PSR_I | PSR_F | PSR_PE_MODE_EL1H;
+
+	/* Transfer control to the exception hander. */
+	vcpu->regs.pc = handler_address;
+
+	direction_str = ISS_IS_READ(esr_el2) ? "read" : "write";
+	dlog("Trapped access to system register %s: op0=%d, op1=%d, crn=%d, "
+	     "crm=%d, op2=%d, rt=%d.\n",
+	     direction_str, GET_ISS_OP0(esr_el2), GET_ISS_OP1(esr_el2),
+	     GET_ISS_CRN(esr_el2), GET_ISS_CRM(esr_el2), GET_ISS_OP2(esr_el2),
+	     GET_ISS_RT(esr_el2));
+
+	dlog("Injecting Unknown Reason exception into VM%d.\n", vcpu->vm->id);
+	dlog("Exception handler address 0x%x\n", handler_address);
+
+	/* Schedule the same VM to continue running. */
+	return NULL;
+}
+
+struct vcpu *hvc_handler(struct vcpu *vcpu)
+{
+	struct spci_value args = {
+		.func = vcpu->regs.r[0],
+		.arg1 = vcpu->regs.r[1],
+		.arg2 = vcpu->regs.r[2],
+		.arg3 = vcpu->regs.r[3],
+		.arg4 = vcpu->regs.r[4],
+		.arg5 = vcpu->regs.r[5],
+		.arg6 = vcpu->regs.r[6],
+		.arg7 = vcpu->regs.r[7],
+	};
+	struct vcpu *next = NULL;
+
+	if (psci_handler(vcpu, args.func, args.arg1, args.arg2, args.arg3,
+			 &vcpu->regs.r[0], &next)) {
+		return next;
+	}
+
+	if (spci_handler(&args, &next)) {
+		arch_regs_set_retval(&vcpu->regs, args);
+		update_vi(next);
+		return next;
+	}
+
+	switch (args.func) {
+	case HF_VM_GET_COUNT:
+		vcpu->regs.r[0] = api_vm_get_count();
+		break;
+
+	case HF_VCPU_GET_COUNT:
+		vcpu->regs.r[0] = api_vcpu_get_count(args.arg1, vcpu);
+		break;
+
+	case HF_MAILBOX_WRITABLE_GET:
+		vcpu->regs.r[0] = api_mailbox_writable_get(vcpu);
+		break;
+
+	case HF_MAILBOX_WAITER_GET:
+		vcpu->regs.r[0] = api_mailbox_waiter_get(args.arg1, vcpu);
+		break;
+
+	case HF_INTERRUPT_ENABLE:
+		vcpu->regs.r[0] =
+			api_interrupt_enable(args.arg1, args.arg2, vcpu);
+		break;
+
+	case HF_INTERRUPT_GET:
+		vcpu->regs.r[0] = api_interrupt_get(vcpu);
+		break;
+
+	case HF_INTERRUPT_INJECT:
+		vcpu->regs.r[0] = api_interrupt_inject(args.arg1, args.arg2,
+						       args.arg3, vcpu, &next);
+		break;
+
+	case HF_DEBUG_LOG:
+		vcpu->regs.r[0] = api_debug_log(args.arg1, vcpu);
+		break;
+
+	default:
+		vcpu->regs.r[0] = SMCCC_ERROR_UNKNOWN;
+	}
+
+	update_vi(next);
+
+	return next;
+}
+
+struct vcpu *irq_lower(void)
+{
+	/*
+	 * Switch back to primary VM, interrupts will be handled there.
+	 *
+	 * If the VM has aborted, this vCPU will be aborted when the scheduler
+	 * tries to run it again. This means the interrupt will not be delayed
+	 * by the aborted VM.
+	 *
+	 * TODO: Only switch when the interrupt isn't for the current VM.
+	 */
+	return api_preempt(current());
+}
+
+struct vcpu *fiq_lower(void)
+{
+	return irq_lower();
+}
+
+struct vcpu *serr_lower(void)
+{
+	dlog("SERR from lower\n");
+	return api_abort(current());
+}
+
+/**
+ * Initialises a fault info structure. It assumes that an FnV bit exists at
+ * bit offset 10 of the ESR, and that it is only valid when the bottom 6 bits of
+ * the ESR (the fault status code) are 010000; this is the case for both
+ * instruction and data aborts, but not necessarily for other exception reasons.
+ */
+static struct vcpu_fault_info fault_info_init(uintreg_t esr,
+					      const struct vcpu *vcpu,
+					      uint32_t mode)
+{
+	uint32_t fsc = esr & 0x3f;
+	struct vcpu_fault_info r;
+
+	r.mode = mode;
+	r.pc = va_init(vcpu->regs.pc);
+
+	/*
+	 * Check the FnV bit, which is only valid if dfsc/ifsc is 010000. It
+	 * indicates that we cannot rely on far_el2.
+	 */
+	if (fsc == 0x10 && esr & (1U << 10)) {
+		r.vaddr = va_init(0);
+		r.ipaddr = ipa_init(read_msr(hpfar_el2) << 8);
+	} else {
+		r.vaddr = va_init(read_msr(far_el2));
+		r.ipaddr = ipa_init((read_msr(hpfar_el2) << 8) |
+				    (read_msr(far_el2) & (PAGE_SIZE - 1)));
+	}
+
+	return r;
+}
+
+struct vcpu *sync_lower_exception(uintreg_t esr)
+{
+	struct vcpu *vcpu = current();
+	struct vcpu_fault_info info;
+	struct vcpu *new_vcpu;
+	uintreg_t ec = GET_ESR_EC(esr);
+
+	switch (ec) {
+	case 0x01: /* EC = 000001, WFI or WFE. */
+		/* Skip the instruction. */
+		vcpu->regs.pc += GET_NEXT_PC_INC(esr);
+		/* Check TI bit of ISS, 0 = WFI, 1 = WFE. */
+		if (esr & 1) {
+			/* WFE */
+			/*
+			 * TODO: consider giving the scheduler more context,
+			 * somehow.
+			 */
+			api_yield(vcpu, &new_vcpu);
+			return new_vcpu;
+		}
+		/* WFI */
+		return api_wait_for_interrupt(vcpu);
+
+	case 0x24: /* EC = 100100, Data abort. */
+		info = fault_info_init(
+			esr, vcpu, (esr & (1U << 6)) ? MM_MODE_W : MM_MODE_R);
+		if (vcpu_handle_page_fault(vcpu, &info)) {
+			return NULL;
+		}
+		break;
+
+	case 0x20: /* EC = 100000, Instruction abort. */
+		info = fault_info_init(esr, vcpu, MM_MODE_X);
+		if (vcpu_handle_page_fault(vcpu, &info)) {
+			return NULL;
+		}
+		break;
+
+	case 0x16: /* EC = 010110, HVC instruction */
+		return hvc_handler(vcpu);
+
+	case 0x17: /* EC = 010111, SMC instruction. */ {
+		uintreg_t smc_pc = vcpu->regs.pc;
+		struct vcpu *next = smc_handler(vcpu);
+
+		/* Skip the SMC instruction. */
+		vcpu->regs.pc = smc_pc + GET_NEXT_PC_INC(esr);
+
+		return next;
+	}
+
+	/*
+	 * EC = 011000, MSR, MRS or System instruction execution that is not
+	 * reported using EC 000000, 000001 or 000111.
+	 */
+	case 0x18:
+		/*
+		 * NOTE: This should never be reached because it goes through a
+		 * separate path handled by handle_system_register_access().
+		 */
+		panic("Handled by handle_system_register_access().");
+
+	default:
+		dlog("Unknown lower sync exception pc=%#x, esr=%#x, "
+		     "ec=%#x\n",
+		     vcpu->regs.pc, esr, ec);
+		break;
+	}
+
+	/*
+	 * The exception wasn't handled. Inject to the VM to give it chance to
+	 * handle as an unknown exception.
+	 */
+	return inject_el1_unknown_exception(vcpu, esr);
+}
+
+/**
+ * Handles EC = 011000, MSR, MRS instruction traps.
+ * Returns non-null ONLY if the access failed and the vCPU is changing.
+ */
+struct vcpu *handle_system_register_access(uintreg_t esr_el2)
+{
+	struct vcpu *vcpu = current();
+	spci_vm_id_t vm_id = vcpu->vm->id;
+	uintreg_t ec = GET_ESR_EC(esr_el2);
+
+	CHECK(ec == 0x18);
+	/*
+	 * Handle accesses to debug and performance monitor registers.
+	 * Inject an exception for unhandled/unsupported registers.
+	 */
+	if (debug_el1_is_register_access(esr_el2)) {
+		if (!debug_el1_process_access(vcpu, vm_id, esr_el2)) {
+			return inject_el1_unknown_exception(vcpu, esr_el2);
+		}
+	} else if (perfmon_is_register_access(esr_el2)) {
+		if (!perfmon_process_access(vcpu, vm_id, esr_el2)) {
+			return inject_el1_unknown_exception(vcpu, esr_el2);
+		}
+	} else if (feature_id_is_register_access(esr_el2)) {
+		if (!feature_id_process_access(vcpu, esr_el2)) {
+			return inject_el1_unknown_exception(vcpu, esr_el2);
+		}
+	} else {
+		return inject_el1_unknown_exception(vcpu, esr_el2);
+	}
+
+	/* Instruction was fulfilled. Skip it and run the next one. */
+	vcpu->regs.pc += GET_NEXT_PC_INC(esr_el2);
+	return NULL;
+}
diff --git a/src/arch/aarch64/hypervisor/hypervisor_entry.S b/src/arch/aarch64/hypervisor/hypervisor_entry.S
new file mode 100644
index 0000000..c6fcb62
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/hypervisor_entry.S
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/offsets.h"
+
+/**
+ * Called only on first boot after the image has been relocated and BSS zeroed.
+ *
+ * It is required that caches be clean and invalid.
+ */
+.section .init.image_entry, "ax"
+.global image_entry
+image_entry:
+	/* Interpret the registers passed from the loader. */
+	bl plat_boot_flow_hook
+
+	/* Get pointer to first CPU. */
+	adrp x28, cpus
+	add x28, x28, :lo12:cpus
+
+	/* Set the ID of this CPU from the affinity bits of mpidr. */
+	mrs x30, mpidr_el1
+	ubfx x29, x30, 0, 24
+	ubfx x30, x30, 32, 8
+	orr x30, x29, x30
+	str x30, [x28, CPU_ID]
+
+	mov x0, x28
+	bl prepare_for_c
+
+	/*
+	 * Call into C to initialize the memory management configuration with
+	 * MMU and caches disabled. Result will be stored in `arch_mm_config`.
+	 */
+	bl one_time_init_mm
+
+	/* Enable MMU and caches before running the rest of initialization. */
+	bl mm_enable
+	bl one_time_init
+
+	/* Begin steady state operation. */
+	mov x0, x28
+	b cpu_init
+
+/**
+ * Entry point for all cases other than the first boot e.g. secondary CPUs and
+ * resuming from suspend.
+ *
+ * It is required that caches be coherent but not necessarily clean or invalid.
+ *
+ * x0 points to the current CPU.
+ */
+.section .text.entry, "ax"
+.global cpu_entry
+cpu_entry:
+	bl mm_enable
+	bl prepare_for_c
+
+	/* Intentional fallthrough. */
+
+cpu_init:
+	/* Call into C code, x0 holds the CPU pointer. */
+	bl cpu_main
+
+	/* Run the vCPU returned by cpu_main. */
+	bl vcpu_restore_all_and_run
+
+	/* Loop forever waiting for interrupts. */
+0:	wfi
+	b 0b
+
+/**
+ * Set up CPU environment for executing C code. This is called on first boot
+ * with caches disabled but subsequent calls will have caches enabled.
+ *
+ * x0 points to the current CPU on entry and exit.
+ */
+prepare_for_c:
+	/* Use SPx (instead of SP0). */
+	msr spsel, #1
+
+	/* Prepare the stack. */
+	ldr x1, [x0, #CPU_STACK_BOTTOM]
+	mov sp, x1
+
+	/* Configure exception handlers. */
+	adr x2, vector_table_el2
+	msr vbar_el2, x2
+	ret
+
+/**
+ * Applies the memory management configuration to the CPU, preserving x0 along
+ * the way.
+ */
+mm_enable:
+	/*
+	 * Invalidate any potentially stale local TLB entries for the
+	 * hypervisor's stage-1 and the VM's stage-2 before they start being
+	 * used. The VM's stage-1 is invalidated as a side effect but it wasn't
+	 * using it yet anyway.
+	 */
+	tlbi alle2
+	tlbi vmalls12e1
+
+	/*
+	 * Load and apply the memory management configuration. Order depends on
+	 * `struct arch_mm_config.
+	 */
+	adrp x6, arch_mm_config
+	add x6, x6, :lo12:arch_mm_config
+
+	ldp x1, x2, [x6]
+	ldp x3, x4, [x6, #16]
+	ldr x5, [x6, #32]
+
+	msr ttbr0_el2, x1
+	msr vtcr_el2, x2
+	msr mair_el2, x3
+	msr tcr_el2, x4
+
+	/* Ensure everything before this point has completed. */
+	dsb sy
+	isb
+
+	/*
+	 * Configure sctlr_el2 to enable MMU and cache and don't proceed until
+	 * this has completed.
+	 */
+	msr sctlr_el2, x5
+	isb
+	ret
diff --git a/src/arch/aarch64/hypervisor/offsets.c b/src/arch/aarch64/hypervisor/offsets.c
new file mode 100644
index 0000000..b99273c
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/offsets.c
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/cpu.h"
+#include "hf/offset_size_header.h"
+
+DEFINE_OFFSETOF(CPU_ID, struct cpu, id)
+DEFINE_OFFSETOF(CPU_STACK_BOTTOM, struct cpu, stack_bottom)
+DEFINE_OFFSETOF(VCPU_REGS, struct vcpu, regs)
+DEFINE_OFFSETOF(VCPU_LAZY, struct vcpu, regs.lazy)
+DEFINE_OFFSETOF(VCPU_FREGS, struct vcpu, regs.fp)
+
+#if GIC_VERSION == 3 || GIC_VERSION == 4
+DEFINE_OFFSETOF(VCPU_GIC, struct vcpu, regs.gic)
+#endif
diff --git a/src/arch/aarch64/hypervisor/perfmon.c b/src/arch/aarch64/hypervisor/perfmon.c
new file mode 100644
index 0000000..5be27ae
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/perfmon.c
@@ -0,0 +1,237 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "perfmon.h"
+
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/types.h"
+
+#include "msr.h"
+#include "sysregs.h"
+
+/* clang-format off */
+
+/**
+ * Definitions of read-only performance monitor registers' encodings.
+ * See Arm Architecture Reference Manual Armv8-A, D12.3.1.
+ * NAME, op0, op1, crn, crm, op2
+ */
+#define PERFMON_REGISTERS_READ                 \
+	X(PMCEID0_EL0       , 3, 3,  9, 12, 6) \
+	X(PMCEID1_EL0       , 3, 3,  9, 12, 7) \
+	X(PMEVCNTR0_EL0     , 3, 3, 14,  8, 0) \
+	X(PMEVCNTR1_EL0     , 3, 3, 14,  8, 1) \
+	X(PMEVCNTR2_EL0     , 3, 3, 14,  8, 2) \
+	X(PMEVCNTR3_EL0     , 3, 3, 14,  8, 3) \
+	X(PMEVCNTR4_EL0     , 3, 3, 14,  8, 4) \
+	X(PMEVCNTR5_EL0     , 3, 3, 14,  8, 5) \
+	X(PMEVCNTR6_EL0     , 3, 3, 14,  8, 6) \
+	X(PMEVCNTR7_EL0     , 3, 3, 14,  8, 7) \
+	X(PMEVCNTR8_EL0     , 3, 3, 14,  9, 0) \
+	X(PMEVCNTR9_EL0     , 3, 3, 14,  9, 1) \
+	X(PMEVCNTR10_EL0    , 3, 3, 14,  9, 2) \
+	X(PMEVCNTR11_EL0    , 3, 3, 14,  9, 3) \
+	X(PMEVCNTR12_EL0    , 3, 3, 14,  9, 4) \
+	X(PMEVCNTR13_EL0    , 3, 3, 14,  9, 5) \
+	X(PMEVCNTR14_EL0    , 3, 3, 14,  9, 6) \
+	X(PMEVCNTR15_EL0    , 3, 3, 14,  9, 7) \
+	X(PMEVCNTR16_EL0    , 3, 3, 14, 10, 0) \
+	X(PMEVCNTR17_EL0    , 3, 3, 14, 10, 1) \
+	X(PMEVCNTR18_EL0    , 3, 3, 14, 10, 2) \
+	X(PMEVCNTR19_EL0    , 3, 3, 14, 10, 3) \
+	X(PMEVCNTR20_EL0    , 3, 3, 14, 10, 4) \
+	X(PMEVCNTR21_EL0    , 3, 3, 14, 10, 5) \
+	X(PMEVCNTR22_EL0    , 3, 3, 14, 10, 6) \
+	X(PMEVCNTR23_EL0    , 3, 3, 14, 10, 7) \
+	X(PMEVCNTR24_EL0    , 3, 3, 14, 11, 0) \
+	X(PMEVCNTR25_EL0    , 3, 3, 14, 11, 1) \
+	X(PMEVCNTR26_EL0    , 3, 3, 14, 11, 2) \
+	X(PMEVCNTR27_EL0    , 3, 3, 14, 11, 3) \
+	X(PMEVCNTR28_EL0    , 3, 3, 14, 11, 4) \
+	X(PMEVCNTR29_EL0    , 3, 3, 14, 11, 5) \
+	X(PMEVCNTR30_EL0    , 3, 3, 14, 11, 6) \
+
+/**
+ * Definitions of write-only performance monitor registers' encodings.
+ * See Arm Architecture Reference Manual Armv8-A, D12.3.1.
+ * NAME, op0, op1, crn, crm, op2
+ */
+#define PERFMON_REGISTERS_WRITE                \
+	X(PMSWINC_EL0       , 3, 3,  9, 12, 4) \
+
+/**
+ * Definitions of readable and writeable performance monitor registers' encodings.
+ * See Arm Architecture Reference Manual Armv8-A, D12.3.1.
+ * NAME, op0, op1, crn, crm, op2
+ */
+#define PERFMON_REGISTERS_READ_WRITE           \
+	X(PMINTENSET_EL1    , 3, 0,  9, 14, 1) \
+	X(PMINTENCLR_EL1    , 3, 0,  9, 14, 2) \
+	X(PMCR_EL0          , 3, 3,  9, 12, 0) \
+	X(PMCNTENSET_EL0    , 3, 3,  9, 12, 1) \
+	X(PMCNTENCLR_EL0    , 3, 3,  9, 12, 2) \
+	X(PMOVSCLR_EL0      , 3, 3,  9, 12, 3) \
+	X(PMSELR_EL0        , 3, 3,  9, 12, 5) \
+	X(PMCCNTR_EL0       , 3, 3,  9, 13, 0) \
+	X(PMXEVTYPER_EL0    , 3, 3,  9, 13, 1) \
+	X(PMXEVCNTR_EL0     , 3, 3,  9, 13, 2) \
+	X(PMUSERENR_EL0     , 3, 3,  9, 14, 0) \
+	X(PMOVSSET_EL0      , 3, 3,  9, 14, 3) \
+	X(PMEVTYPER0_EL0    , 3, 3, 14, 12, 0) \
+	X(PMEVTYPER1_EL0    , 3, 3, 14, 12, 1) \
+	X(PMEVTYPER2_EL0    , 3, 3, 14, 12, 2) \
+	X(PMEVTYPER3_EL0    , 3, 3, 14, 12, 3) \
+	X(PMEVTYPER4_EL0    , 3, 3, 14, 12, 4) \
+	X(PMEVTYPER5_EL0    , 3, 3, 14, 12, 5) \
+	X(PMEVTYPER6_EL0    , 3, 3, 14, 12, 6) \
+	X(PMEVTYPER7_EL0    , 3, 3, 14, 12, 7) \
+	X(PMEVTYPER8_EL0    , 3, 3, 14, 13, 0) \
+	X(PMEVTYPER9_EL0    , 3, 3, 14, 13, 1) \
+	X(PMEVTYPER10_EL0   , 3, 3, 14, 13, 2) \
+	X(PMEVTYPER11_EL0   , 3, 3, 14, 13, 3) \
+	X(PMEVTYPER12_EL0   , 3, 3, 14, 13, 4) \
+	X(PMEVTYPER13_EL0   , 3, 3, 14, 13, 5) \
+	X(PMEVTYPER14_EL0   , 3, 3, 14, 13, 6) \
+	X(PMEVTYPER15_EL0   , 3, 3, 14, 13, 7) \
+	X(PMEVTYPER16_EL0   , 3, 3, 14, 14, 0) \
+	X(PMEVTYPER17_EL0   , 3, 3, 14, 14, 1) \
+	X(PMEVTYPER18_EL0   , 3, 3, 14, 14, 2) \
+	X(PMEVTYPER19_EL0   , 3, 3, 14, 14, 3) \
+	X(PMEVTYPER20_EL0   , 3, 3, 14, 14, 4) \
+	X(PMEVTYPER21_EL0   , 3, 3, 14, 14, 5) \
+	X(PMEVTYPER22_EL0   , 3, 3, 14, 14, 6) \
+	X(PMEVTYPER23_EL0   , 3, 3, 14, 14, 7) \
+	X(PMEVTYPER24_EL0   , 3, 3, 14, 15, 0) \
+	X(PMEVTYPER25_EL0   , 3, 3, 14, 15, 1) \
+	X(PMEVTYPER26_EL0   , 3, 3, 14, 15, 2) \
+	X(PMEVTYPER27_EL0   , 3, 3, 14, 15, 3) \
+	X(PMEVTYPER28_EL0   , 3, 3, 14, 15, 4) \
+	X(PMEVTYPER29_EL0   , 3, 3, 14, 15, 5) \
+	X(PMEVTYPER30_EL0   , 3, 3, 14, 15, 6) \
+	X(PMCCFILTR_EL0     , 3, 3, 14, 15, 7)
+
+/* clang-format on */
+
+/**
+ * Returns true if the ESR register shows an access to a performance monitor
+ * register.
+ */
+bool perfmon_is_register_access(uintreg_t esr)
+{
+	uintreg_t op0 = GET_ISS_OP0(esr);
+	uintreg_t op1 = GET_ISS_OP1(esr);
+	uintreg_t crn = GET_ISS_CRN(esr);
+	uintreg_t crm = GET_ISS_CRM(esr);
+
+	/* From the Arm Architecture Reference Manual Table D12-2. */
+
+	/* For PMINTENCLR_EL1 and PMINTENSET_EL1*/
+	if (op0 == 3 && op1 == 0 && crn == 9 && crm == 14) {
+		return true;
+	}
+
+	/* For PMEVCNTRn_EL0, PMEVTYPERn_EL0, and PMCCFILTR_EL0. */
+	if (op0 == 3 && op1 == 3 && crn == 14 && crm >= 8 && crm <= 15) {
+		return true;
+	}
+
+	/* For all remaining performance monitor registers. */
+	return op0 == 3 && op1 == 3 && crn == 9 && crm >= 12 && crm <= 14;
+}
+
+/**
+ * Processes an access (msr, mrs) to a performance monitor register.
+ * Returns true if the access was allowed and performed, false otherwise.
+ */
+bool perfmon_process_access(struct vcpu *vcpu, spci_vm_id_t vm_id,
+			    uintreg_t esr)
+{
+	/*
+	 * For now, performance monitor registers are not supported by secondary
+	 * VMs. Disallow accesses to them.
+	 */
+	if (vm_id != HF_PRIMARY_VM_ID) {
+		return false;
+	}
+
+	uintreg_t sys_register = GET_ISS_SYSREG(esr);
+	uintreg_t rt_register = GET_ISS_RT(esr);
+	uintreg_t value;
+
+	/* +1 because Rt can access register XZR */
+	CHECK(rt_register < NUM_GP_REGS + 1);
+
+	if (ISS_IS_READ(esr)) {
+		switch (sys_register) {
+#define X(reg_name, op0, op1, crn, crm, op2)              \
+	case (GET_ISS_ENCODING(op0, op1, crn, crm, op2)): \
+		value = read_msr(reg_name);               \
+		break;
+			PERFMON_REGISTERS_READ
+			PERFMON_REGISTERS_READ_WRITE
+#undef X
+		default:
+			value = vcpu->regs.r[rt_register];
+			dlog("Unsupported performance monitor register read: "
+			     "op0=%d, op1=%d, crn=%d, crm=%d, op2=%d, rt=%d.\n",
+			     GET_ISS_OP0(esr), GET_ISS_OP1(esr),
+			     GET_ISS_CRN(esr), GET_ISS_CRM(esr),
+			     GET_ISS_OP2(esr), GET_ISS_RT(esr));
+			break;
+		}
+		if (rt_register != RT_REG_XZR) {
+			vcpu->regs.r[rt_register] = value;
+		}
+	} else {
+		if (rt_register != RT_REG_XZR) {
+			value = vcpu->regs.r[rt_register];
+		} else {
+			value = 0;
+		}
+		switch (sys_register) {
+#define X(reg_name, op0, op1, crn, crm, op2)              \
+	case (GET_ISS_ENCODING(op0, op1, crn, crm, op2)): \
+		write_msr(reg_name, value);               \
+		break;
+			PERFMON_REGISTERS_WRITE
+			PERFMON_REGISTERS_READ_WRITE
+#undef X
+		default:
+			dlog("Unsupported performance monitor register write: "
+			     "op0=%d, op1=%d, crn=%d, crm=%d, op2=%d, rt=%d.\n",
+			     GET_ISS_OP0(esr), GET_ISS_OP1(esr),
+			     GET_ISS_CRN(esr), GET_ISS_CRM(esr),
+			     GET_ISS_OP2(esr), GET_ISS_RT(esr));
+			break;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * Returns the value register PMCCFILTR_EL0 should have at initialization.
+ */
+uintreg_t perfmon_get_pmccfiltr_el0_init_value(spci_vm_id_t vm_id)
+{
+	if (vm_id != HF_PRIMARY_VM_ID) {
+		/* Disable cycle counting for secondary VMs.  */
+		return PMCCFILTR_EL0_P | PMCCFILTR_EL0_U;
+	}
+
+	return 0;
+}
diff --git a/src/arch/aarch64/hypervisor/perfmon.h b/src/arch/aarch64/hypervisor/perfmon.h
new file mode 100644
index 0000000..afeabd9
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/perfmon.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/arch/types.h"
+
+#include "hf/cpu.h"
+
+#include "vmapi/hf/spci.h"
+
+/**
+ * Set to disable cycle counting when event counting is prohibited.
+ */
+#define PMCR_EL0_DP 0x10
+
+/**
+ * Set to enable export of events where not prohibited.
+ */
+#define PMCR_EL0_X 0x8
+
+/**
+ * Set to enable event counting.
+ */
+#define PMCR_EL0_E 0x1
+
+/**
+ * Set to disable cycle counting in EL1.
+ */
+#define PMCCFILTR_EL0_P 0x80000000
+
+/**
+ * Set to disable cycle counting in EL0.
+ */
+#define PMCCFILTR_EL0_U 0x40000000
+
+/**
+ * Cycle counting in non-secure EL1 is enabled if NSK == P.
+ */
+#define PMCCFILTR_EL0_NSK 0x20000000
+
+/**
+ * Cycle counting in non-secure EL0 is enabled if NSU == U.
+ */
+#define PMCCFILTR_EL0_NSU 0x10000000
+
+/**
+ * Set to enable cycle counting in EL2.
+ */
+#define PMCCFILTR_EL0_NSH 0x8000000
+
+/**
+ * Cycle counting in EL3 is enabled if M == P.
+ */
+#define PMCCFILTR_EL0_M 0x4000000
+
+/**
+ * Cycle counting in Secutre EL2 is enabled if SH != NSH.
+ */
+#define PMCCFILTR_EL0_SH 0x1000000
+
+bool perfmon_is_register_access(uintreg_t esr_el2);
+
+bool perfmon_process_access(struct vcpu *vcpu, spci_vm_id_t vm_id,
+			    uintreg_t esr_el2);
+
+uintreg_t perfmon_get_pmccfiltr_el0_init_value(spci_vm_id_t vm_id);
diff --git a/src/arch/aarch64/hypervisor/plat_entry.S b/src/arch/aarch64/hypervisor/plat_entry.S
new file mode 100644
index 0000000..d7ecc87
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/plat_entry.S
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.section .init.image_entry, "ax"
+.global plat_entry
+.weak plat_entry
+plat_entry:
+	/* Do nothing. */
+	ret
diff --git a/src/arch/aarch64/hypervisor/psci_handler.c b/src/arch/aarch64/hypervisor/psci_handler.c
new file mode 100644
index 0000000..1d638e4
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/psci_handler.c
@@ -0,0 +1,413 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "psci_handler.h"
+
+#include <stdint.h>
+
+#include "hf/arch/types.h"
+
+#include "hf/api.h"
+#include "hf/cpu.h"
+#include "hf/dlog.h"
+#include "hf/panic.h"
+#include "hf/spci.h"
+#include "hf/vm.h"
+
+#include "psci.h"
+#include "smc.h"
+
+static uint32_t el3_psci_version;
+
+void cpu_entry(struct cpu *c);
+
+/* Performs arch specific boot time initialisation. */
+void arch_one_time_init(void)
+{
+	struct spci_value smc_res =
+		smc32(PSCI_VERSION, 0, 0, 0, 0, 0, 0, SMCCC_CALLER_HYPERVISOR);
+
+	el3_psci_version = smc_res.func;
+
+	/* Check there's nothing unexpected about PSCI. */
+	switch (el3_psci_version) {
+	case PSCI_VERSION_0_2:
+	case PSCI_VERSION_1_0:
+	case PSCI_VERSION_1_1:
+		/* Supported EL3 PSCI version. */
+		dlog("Found PSCI version: %#x\n", el3_psci_version);
+		break;
+
+	default:
+		/* Unsupported EL3 PSCI version. Log a warning but continue. */
+		dlog("Warning: unknown PSCI version: %#x\n", el3_psci_version);
+		el3_psci_version = 0;
+		break;
+	}
+}
+
+/**
+ * Handles PSCI requests received via HVC or SMC instructions from the primary
+ * VM.
+ *
+ * A minimal PSCI 1.1 interface is offered which can make use of the
+ * implementation of PSCI in EL3 by acting as an adapter.
+ *
+ * Returns true if the request was a PSCI one, false otherwise.
+ */
+bool psci_primary_vm_handler(struct vcpu *vcpu, uint32_t func, uintreg_t arg0,
+			     uintreg_t arg1, uintreg_t arg2, uintreg_t *ret)
+{
+	struct cpu *c;
+	struct spci_value smc_res;
+
+	/*
+	 * If there's a problem with the EL3 PSCI, block standard secure service
+	 * calls by marking them as unknown. Other calls will be allowed to pass
+	 * through.
+	 *
+	 * This blocks more calls than just PSCI so it may need to be made more
+	 * lenient in future.
+	 */
+	if (el3_psci_version == 0) {
+		*ret = SMCCC_ERROR_UNKNOWN;
+		return (func & SMCCC_SERVICE_CALL_MASK) ==
+		       SMCCC_STANDARD_SECURE_SERVICE_CALL;
+	}
+
+	switch (func & ~SMCCC_CONVENTION_MASK) {
+	case PSCI_VERSION:
+		*ret = PSCI_VERSION_1_1;
+		break;
+
+	case PSCI_FEATURES:
+		switch (arg0 & ~SMCCC_CONVENTION_MASK) {
+		case PSCI_CPU_SUSPEND:
+			if (el3_psci_version == PSCI_VERSION_0_2) {
+				/*
+				 * PSCI 0.2 doesn't support PSCI_FEATURES so
+				 * report PSCI 0.2 compatible features.
+				 */
+				*ret = 0;
+			} else {
+				/* PSCI 1.x only defines two feature bits. */
+				smc_res = smc32(func, arg0, 0, 0, 0, 0, 0,
+						SMCCC_CALLER_HYPERVISOR);
+				*ret = smc_res.func & 0x3;
+			}
+			break;
+
+		case PSCI_VERSION:
+		case PSCI_FEATURES:
+		case PSCI_SYSTEM_OFF:
+		case PSCI_SYSTEM_RESET:
+		case PSCI_AFFINITY_INFO:
+		case PSCI_CPU_OFF:
+		case PSCI_CPU_ON:
+			/* These are supported without special features. */
+			*ret = 0;
+			break;
+
+		default:
+			/* Everything else is unsupported. */
+			*ret = PSCI_ERROR_NOT_SUPPORTED;
+			break;
+		}
+		break;
+
+	case PSCI_SYSTEM_OFF:
+		smc32(PSCI_SYSTEM_OFF, 0, 0, 0, 0, 0, 0,
+		      SMCCC_CALLER_HYPERVISOR);
+		panic("System off failed");
+		break;
+
+	case PSCI_SYSTEM_RESET:
+		smc32(PSCI_SYSTEM_RESET, 0, 0, 0, 0, 0, 0,
+		      SMCCC_CALLER_HYPERVISOR);
+		panic("System reset failed");
+		break;
+
+	case PSCI_AFFINITY_INFO:
+		c = cpu_find(arg0);
+		if (!c) {
+			*ret = PSCI_ERROR_INVALID_PARAMETERS;
+			break;
+		}
+
+		if (arg1 != 0) {
+			*ret = PSCI_ERROR_NOT_SUPPORTED;
+			break;
+		}
+
+		sl_lock(&c->lock);
+		if (c->is_on) {
+			*ret = PSCI_RETURN_ON;
+		} else {
+			*ret = PSCI_RETURN_OFF;
+		}
+		sl_unlock(&c->lock);
+		break;
+
+	case PSCI_CPU_SUSPEND: {
+		/*
+		 * Update vCPU state to wake from the provided entry point but
+		 * if suspend returns, for example because it failed or was a
+		 * standby power state, the SMC will return and the updated
+		 * vCPU registers will be ignored.
+		 */
+		arch_regs_set_pc_arg(&vcpu->regs, ipa_init(arg1), arg2);
+		smc_res = smc64(PSCI_CPU_SUSPEND, arg0, (uintreg_t)&cpu_entry,
+				(uintreg_t)vcpu->cpu, 0, 0, 0,
+				SMCCC_CALLER_HYPERVISOR);
+		*ret = smc_res.func;
+		break;
+	}
+
+	case PSCI_CPU_OFF:
+		cpu_off(vcpu->cpu);
+		smc32(PSCI_CPU_OFF, 0, 0, 0, 0, 0, 0, SMCCC_CALLER_HYPERVISOR);
+		panic("CPU off failed");
+		break;
+
+	case PSCI_CPU_ON:
+		c = cpu_find(arg0);
+		if (!c) {
+			*ret = PSCI_ERROR_INVALID_PARAMETERS;
+			break;
+		}
+
+		if (cpu_on(c, ipa_init(arg1), arg2)) {
+			*ret = PSCI_ERROR_ALREADY_ON;
+			break;
+		}
+
+		/*
+		 * There's a race when turning a CPU on when it's in the
+		 * process of turning off. We need to loop here while it is
+		 * reported that the CPU is on (because it's about to turn
+		 * itself off).
+		 */
+		do {
+			smc_res = smc64(PSCI_CPU_ON, arg0,
+					(uintreg_t)&cpu_entry, (uintreg_t)c, 0,
+					0, 0, SMCCC_CALLER_HYPERVISOR);
+			*ret = smc_res.func;
+		} while (*ret == PSCI_ERROR_ALREADY_ON);
+
+		if (*ret != PSCI_RETURN_SUCCESS) {
+			cpu_off(c);
+		}
+		break;
+
+	case PSCI_MIGRATE:
+	case PSCI_MIGRATE_INFO_TYPE:
+	case PSCI_MIGRATE_INFO_UP_CPU:
+	case PSCI_CPU_FREEZE:
+	case PSCI_CPU_DEFAULT_SUSPEND:
+	case PSCI_NODE_HW_STATE:
+	case PSCI_SYSTEM_SUSPEND:
+	case PSCI_SET_SYSPEND_MODE:
+	case PSCI_STAT_RESIDENCY:
+	case PSCI_STAT_COUNT:
+	case PSCI_SYSTEM_RESET2:
+	case PSCI_MEM_PROTECT:
+	case PSCI_MEM_PROTECT_CHECK_RANGE:
+		/* Block all other known PSCI calls. */
+		*ret = PSCI_ERROR_NOT_SUPPORTED;
+		break;
+
+	default:
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * Convert a PSCI CPU / affinity ID for a secondary VM to the corresponding vCPU
+ * index.
+ */
+spci_vcpu_index_t vcpu_id_to_index(cpu_id_t vcpu_id)
+{
+	/* For now we use indices as IDs for the purposes of PSCI. */
+	return vcpu_id;
+}
+
+/**
+ * Handles PSCI requests received via HVC or SMC instructions from a secondary
+ * VM.
+ *
+ * A minimal PSCI 1.1 interface is offered which can start and stop vCPUs in
+ * collaboration with the scheduler in the primary VM.
+ *
+ * Returns true if the request was a PSCI one, false otherwise.
+ */
+bool psci_secondary_vm_handler(struct vcpu *vcpu, uint32_t func, uintreg_t arg0,
+			       uintreg_t arg1, uintreg_t arg2, uintreg_t *ret,
+			       struct vcpu **next)
+{
+	switch (func & ~SMCCC_CONVENTION_MASK) {
+	case PSCI_VERSION:
+		*ret = PSCI_VERSION_1_1;
+		break;
+
+	case PSCI_FEATURES:
+		switch (arg0 & ~SMCCC_CONVENTION_MASK) {
+		case PSCI_CPU_SUSPEND:
+			/*
+			 * Does not offer OS-initiated mode but does use
+			 * extended StateID Format.
+			 */
+			*ret = 0x2;
+			break;
+
+		case PSCI_VERSION:
+		case PSCI_FEATURES:
+		case PSCI_AFFINITY_INFO:
+		case PSCI_CPU_OFF:
+		case PSCI_CPU_ON:
+			/* These are supported without special features. */
+			*ret = 0;
+			break;
+
+		default:
+			/* Everything else is unsupported. */
+			*ret = PSCI_ERROR_NOT_SUPPORTED;
+			break;
+		}
+		break;
+
+	case PSCI_AFFINITY_INFO: {
+		cpu_id_t target_affinity = arg0;
+		uint32_t lowest_affinity_level = arg1;
+		struct vm *vm = vcpu->vm;
+		struct vcpu_locked target_vcpu;
+		spci_vcpu_index_t target_vcpu_index =
+			vcpu_id_to_index(target_affinity);
+
+		if (lowest_affinity_level != 0) {
+			/* Affinity levels greater than 0 not supported. */
+			*ret = PSCI_ERROR_INVALID_PARAMETERS;
+			break;
+		}
+
+		if (target_vcpu_index >= vm->vcpu_count) {
+			*ret = PSCI_ERROR_INVALID_PARAMETERS;
+			break;
+		}
+
+		target_vcpu = vcpu_lock(vm_get_vcpu(vm, target_vcpu_index));
+		*ret = vcpu_is_off(target_vcpu) ? PSCI_RETURN_OFF
+						: PSCI_RETURN_ON;
+		vcpu_unlock(&target_vcpu);
+		break;
+	}
+
+	case PSCI_CPU_SUSPEND: {
+		/*
+		 * Downgrade suspend request to WFI and return SUCCESS, as
+		 * allowed by the specification.
+		 */
+		*next = api_wait_for_interrupt(vcpu);
+		*ret = PSCI_RETURN_SUCCESS;
+		break;
+	}
+
+	case PSCI_CPU_OFF:
+		/*
+		 * Should never return to the caller, but in case it somehow
+		 * does.
+		 */
+		*ret = PSCI_ERROR_DENIED;
+		/* Tell the scheduler not to run the vCPU again. */
+		*next = api_vcpu_off(vcpu);
+		break;
+
+	case PSCI_CPU_ON: {
+		/* Parameter names as per PSCI specification. */
+		cpu_id_t target_cpu = arg0;
+		ipaddr_t entry_point_address = ipa_init(arg1);
+		uint64_t context_id = arg2;
+		spci_vcpu_index_t target_vcpu_index =
+			vcpu_id_to_index(target_cpu);
+		struct vm *vm = vcpu->vm;
+		struct vcpu *target_vcpu;
+
+		if (target_vcpu_index >= vm->vcpu_count) {
+			*ret = PSCI_ERROR_INVALID_PARAMETERS;
+			break;
+		}
+
+		target_vcpu = vm_get_vcpu(vm, target_vcpu_index);
+
+		if (vcpu_secondary_reset_and_start(
+			    target_vcpu, entry_point_address, context_id)) {
+			/*
+			 * Tell the scheduler that it can start running the new
+			 * vCPU now.
+			 */
+			*next = api_wake_up(vcpu, target_vcpu);
+			*ret = PSCI_RETURN_SUCCESS;
+		} else {
+			*ret = PSCI_ERROR_ALREADY_ON;
+		}
+
+		break;
+	}
+
+	case PSCI_SYSTEM_OFF:
+	case PSCI_SYSTEM_RESET:
+	case PSCI_MIGRATE:
+	case PSCI_MIGRATE_INFO_TYPE:
+	case PSCI_MIGRATE_INFO_UP_CPU:
+	case PSCI_CPU_FREEZE:
+	case PSCI_CPU_DEFAULT_SUSPEND:
+	case PSCI_NODE_HW_STATE:
+	case PSCI_SYSTEM_SUSPEND:
+	case PSCI_SET_SYSPEND_MODE:
+	case PSCI_STAT_RESIDENCY:
+	case PSCI_STAT_COUNT:
+	case PSCI_SYSTEM_RESET2:
+	case PSCI_MEM_PROTECT:
+	case PSCI_MEM_PROTECT_CHECK_RANGE:
+		/* Block all other known PSCI calls. */
+		*ret = PSCI_ERROR_NOT_SUPPORTED;
+		break;
+
+	default:
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * Handles PSCI requests received via HVC or SMC instructions from a VM.
+ * Requests from primary and secondary VMs are dealt with differently.
+ *
+ * Returns true if the request was a PSCI one, false otherwise.
+ */
+bool psci_handler(struct vcpu *vcpu, uint32_t func, uintreg_t arg0,
+		  uintreg_t arg1, uintreg_t arg2, uintreg_t *ret,
+		  struct vcpu **next)
+{
+	if (vcpu->vm->id == HF_PRIMARY_VM_ID) {
+		return psci_primary_vm_handler(vcpu, func, arg0, arg1, arg2,
+					       ret);
+	}
+	return psci_secondary_vm_handler(vcpu, func, arg0, arg1, arg2, ret,
+					 next);
+}
diff --git a/src/arch/aarch64/hypervisor/psci_handler.h b/src/arch/aarch64/hypervisor/psci_handler.h
new file mode 100644
index 0000000..479c1cd
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/psci_handler.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include "hf/arch/types.h"
+
+#include "hf/cpu.h"
+
+bool psci_handler(struct vcpu *vcpu, uint32_t func, uintreg_t arg0,
+		  uintreg_t arg1, uintreg_t arg2, uintreg_t *ret,
+		  struct vcpu **next);
diff --git a/src/arch/aarch64/hypervisor/sysregs.c b/src/arch/aarch64/hypervisor/sysregs.c
new file mode 100644
index 0000000..06385eb
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/sysregs.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "sysregs.h"
+
+#include "msr.h"
+
+/**
+ * Returns the value for HCR_EL2 for the particular VM.
+ * For now, the primary VM has one value and all secondary VMs share a value.
+ */
+uintreg_t get_hcr_el2_value(spci_vm_id_t vm_id)
+{
+	uintreg_t hcr_el2_value = 0;
+
+	/* Baseline values for all VMs. */
+
+	/*
+	 * Trap access to registers in ID group 3. These registers report on
+	 * the underlying support for CPU features. Because Hafnium restricts
+	 * certain features, e.g., RAS, it should emulate access to these
+	 * registers to report the correct set of features supported.
+	 */
+	hcr_el2_value |= HCR_EL2_TID3;
+
+	/* Execution state for EL1 is AArch64. */
+	hcr_el2_value |= HCR_EL2_RW;
+
+	/* Trap implementation registers and functionality. */
+	hcr_el2_value |= HCR_EL2_TACR | HCR_EL2_TIDCP;
+
+	/* Trap SMC instructions. */
+	hcr_el2_value |= HCR_EL2_TSC;
+
+	/*
+	 * Translation table access made as part of a stage 1 translation
+	 * table walk is subject to a stage 2 translation;
+	 */
+	hcr_el2_value |= HCR_EL2_PTW;
+
+	/* Enable stage 2 address translation;*/
+	hcr_el2_value |= HCR_EL2_VM;
+
+	/* Trap cache maintenance instructions that operate by Set/Way. */
+	hcr_el2_value |= HCR_EL2_TSW;
+
+	/* Do *not* trap PAuth. APK and API bits *disable* trapping when set. */
+	hcr_el2_value |= HCR_EL2_APK | HCR_EL2_API;
+
+	/* Baseline values for all secondary VMs. */
+	if (vm_id != HF_PRIMARY_VM_ID) {
+		/*
+		 * Set the minimum shareability domain to barrier instructions
+		 * as inner shareable.
+		 */
+		hcr_el2_value |= HCR_EL2_BSU_INNER_SHAREABLE;
+
+		/*
+		 * Broadcast instructions related to invalidating the TLB within
+		 * the Inner Shareable domain.
+		 */
+		hcr_el2_value |= HCR_EL2_FB;
+
+		/* Route physical SError/IRQ/FIQ interrupts to EL2. */
+		hcr_el2_value |= HCR_EL2_AMO | HCR_EL2_IMO | HCR_EL2_FMO;
+
+		/* Trap wait for event/interrupt instructions. */
+		hcr_el2_value |= HCR_EL2_TWE | HCR_EL2_TWI;
+	}
+
+	return hcr_el2_value;
+}
+
+/**
+ * Returns the default value for MDCR_EL2.
+ */
+uintreg_t get_mdcr_el2_value(void)
+{
+	uintreg_t mdcr_el2_value = read_msr(MDCR_EL2);
+	uintreg_t pmcr_el0 = read_msr(PMCR_EL0);
+
+	/* Baseline values for all VMs. */
+
+	/* Disable cycle and event counting at EL2. */
+	mdcr_el2_value |= MDCR_EL2_HCCD | MDCR_EL2_HPMD;
+
+	/* All available event counters accessible from all exception levels. */
+	mdcr_el2_value |= GET_PMCR_EL0_N(pmcr_el0) & MDCR_EL2_HPMN;
+
+	return mdcr_el2_value;
+}
+
+/**
+ * Returns the value for CPTR_EL2 for the CPU.
+ */
+uintreg_t get_cptr_el2_value(void)
+{
+	return CPTR_EL2_TTA;
+}
diff --git a/src/arch/aarch64/hypervisor/sysregs.h b/src/arch/aarch64/hypervisor/sysregs.h
new file mode 100644
index 0000000..250c9bf
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/sysregs.h
@@ -0,0 +1,451 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/arch/types.h"
+
+#include "hf/cpu.h"
+
+#include "vmapi/hf/spci.h"
+
+/**
+ * RT value that indicates an access to register XZR (always 0).
+ * See Arm Architecture Reference Manual Armv8-A, C1.2.5
+ */
+#define RT_REG_XZR (UINT64_C(31))
+
+/**
+ * Hypervisor (EL2) Cycle Count Disable.
+ */
+#define MDCR_EL2_HCCD (UINT64_C(0x1) << 23)
+
+/**
+ * Controls traps for Trace Filter when Self-hosted Trace is implemented.
+ */
+#define MDCR_EL2_TTRF (UINT64_C(0x1) << 19)
+
+/**
+ * Hypervisor (EL2) Event Count Disable.
+ */
+#define MDCR_EL2_HPMD (UINT64_C(0x1) << 17)
+
+/**
+ * Trap Performance Monitor Sampling.
+ * Traps access to Statistical Profiling control registers from EL1 when
+ * the Statistical Profiling Extension (SPE) is implemented.
+ */
+#define MDCR_EL2_TPMS (UINT64_C(0x1) << 14)
+
+/**
+ * Controls the owning translation regime and access to Profiling Buffer control
+ * registers from EL1. Depends on whether SPE is implemented.
+ */
+#define MDCR_EL2_E2PB (UINT64_C(0x3) << 12)
+
+/**
+ * Controls traps for Debug ROM.
+ */
+#define MDCR_EL2_TDRA (UINT64_C(0x1) << 11)
+
+/**
+ * Controls traps for debug OS-Related Register accesses when DoubleLock is
+ * implemented.
+ */
+#define MDCR_EL2_TDOSA (UINT64_C(0x1) << 10)
+
+/**
+ * Controls traps for remaining Debug Registers not trapped by TDRA and TDOSA.
+ */
+#define MDCR_EL2_TDA (UINT64_C(0x1) << 9)
+
+/**
+ * Controls traps for all debug exceptions (e.g., breakpoints).
+ */
+#define MDCR_EL2_TDE (UINT64_C(0x1) << 8)
+
+/**
+ * Controls traps for all performance monitor register accesses other than
+ * PMCR_EL0.
+ */
+#define MDCR_EL2_TPM (UINT64_C(0x1) << 6)
+
+/**
+ * Controls traps for performance monitor register PMCR_EL0.
+ */
+#define MDCR_EL2_TPMCR (UINT64_C(0x1) << 5)
+
+/**
+ * Defines the number of event counters that are accessible from various
+ * exception levels, if permitted. Dependant on whether PMUv3
+ * is implemented.
+ */
+#define MDCR_EL2_HPMN (UINT64_C(0x1f) << 0)
+
+/**
+ * System register are identified by op0, op2, op1, crn, crm. The ISS encoding
+ * includes also rt and direction. Exclude them, @see D13.2.37 (D13-2977).
+ */
+#define ISS_SYSREG_MASK                                                      \
+	(((UINT64_C(0x1) << 22) - UINT64_C(0x1)) & /* Select the ISS bits */ \
+	 ~(UINT64_C(0x1f) << 5) &		   /* exclude rt */          \
+	 ~UINT64_C(0x1) /* exclude direction */)
+
+#define GET_ISS_SYSREG(esr) (ISS_SYSREG_MASK & (esr))
+
+/**
+ * Op0 from the ISS encoding in the ESR.
+ */
+#define ISS_OP0_MASK UINT64_C(0x300000)
+#define ISS_OP0_SHIFT 20
+#define GET_ISS_OP0(esr) ((ISS_OP0_MASK & (esr)) >> ISS_OP0_SHIFT)
+
+/**
+ * Op1 from the ISS encoding in the ESR.
+ */
+#define ISS_OP1_MASK UINT64_C(0x1c000)
+#define ISS_OP1_SHIFT 14
+#define GET_ISS_OP1(esr) ((ISS_OP1_MASK & (esr)) >> ISS_OP1_SHIFT)
+
+/**
+ * Op2 from the ISS encoding in the ESR.
+ */
+#define ISS_OP2_MASK UINT64_C(0xe0000)
+#define ISS_OP2_SHIFT 17
+#define GET_ISS_OP2(esr) ((ISS_OP2_MASK & (esr)) >> ISS_OP2_SHIFT)
+
+/**
+ * CRn from the ISS encoding in the ESR.
+ */
+#define ISS_CRN_MASK UINT64_C(0x3c00)
+#define ISS_CRN_SHIFT 10
+#define GET_ISS_CRN(esr) ((ISS_CRN_MASK & (esr)) >> ISS_CRN_SHIFT)
+
+/**
+ * CRm from the ISS encoding in the ESR.
+ */
+#define ISS_CRM_MASK UINT64_C(0x1e)
+#define ISS_CRM_SHIFT 1
+#define GET_ISS_CRM(esr) ((ISS_CRM_MASK & (esr)) >> ISS_CRM_SHIFT)
+
+/**
+ * Rt, which identifies the general purpose register used for the operation.
+ */
+#define ISS_RT_MASK UINT64_C(0x3e0)
+#define ISS_RT_SHIFT 5
+#define GET_ISS_RT(esr) ((ISS_RT_MASK & (esr)) >> ISS_RT_SHIFT)
+
+/**
+ * Direction (i.e., read (1) or write (0), is the first bit in the ISS/ESR.
+ */
+#define ISS_DIRECTION_MASK UINT64_C(0x1)
+
+/**
+ * Gets the direction of the system register access, read (1) or write (0).
+ */
+#define GET_ISS_DIRECTION(esr) (ISS_DIRECTION_MASK & (esr))
+
+/**
+ * True if the ISS encoded in the esr indicates a read of the system register.
+ */
+#define ISS_IS_READ(esr) (ISS_DIRECTION_MASK & (esr))
+
+/**
+ * Returns the ISS encoding given the various instruction encoding parameters.
+ */
+#define GET_ISS_ENCODING(op0, op1, crn, crm, op2)          \
+	((op0) << ISS_OP0_SHIFT | (op2) << ISS_OP2_SHIFT | \
+	 (op1) << ISS_OP1_SHIFT | (crn) << ISS_CRN_SHIFT | \
+	 (crm) << ISS_CRM_SHIFT)
+
+#define PMCR_EL0_N_MASK UINT64_C(0xf800)
+#define PMCR_EL0_N_SHIFT 11
+#define GET_PMCR_EL0_N(pmcr) ((PMCR_EL0_N_MASK & (pmcr)) >> PMCR_EL0_N_SHIFT)
+
+/*
+ * Define various configurations bits for the Hypervisor Configuration Register,
+ * HCR_EL2. See Arm Architecture Reference Manual, D13.2.46.
+ */
+
+/**
+ * Trap ID group 5 (Armv8.5-MemTag related).
+ */
+#define HCR_EL2_TID5 (UINT64_C(0x1) << 58)
+
+/**
+ * Trap TLB maintenance instructions that operate on the Outer Shareable domain.
+ */
+#define HCR_EL2_TTLBOS (UINT64_C(0x1) << 55)
+
+/**
+ * Trap TLB maintenance instructions that operate on the Inner Shareable domain.
+ */
+#define HCR_EL2_TTLBIS (UINT64_C(0x1) << 54)
+
+/**
+ * Trap cache maintenance instructions that operate to the Point of Unification.
+ */
+#define HCR_EL2_TOCU (UINT64_C(0x1) << 52)
+
+/**
+ * Trap ICIALLUIS/IC IALLUIS cache maintenance instructions.
+ */
+#define HCR_EL2_TICAB (UINT64_C(0x1) << 52)
+
+/**
+ * Trap ID group 4.
+ */
+#define HCR_EL2_TID4 (UINT64_C(0x1) << 49)
+
+/**
+ * When set *disables* traps on Pointer Authentication related instruction
+ * execution.
+ */
+#define HCR_EL2_API (UINT64_C(0x1) << 41)
+
+/**
+ * When set *disables* traps on access to Pointer Authentication's "key"
+ * registers.
+ */
+#define HCR_EL2_APK (UINT64_C(0x1) << 40)
+
+/**
+ * Trap Error record accesses when RAS is implemented.
+ */
+#define HCR_EL2_TERR (UINT64_C(0x1) << 36)
+
+/**
+ * Trap LOR register accesses when LORegions is implemented.
+ */
+#define HCR_EL2_TLOR (UINT64_C(0x1) << 35)
+
+/**
+ * Stage 2 Instruction access cacheability disable.
+ * When set, forces all stage 2 translations for instruction accesses to normal
+ * memory to be non-cacheable.
+ */
+#define HCR_EL2_ID (UINT64_C(0x1) << 33)
+
+/**
+ * Stage 2 Data access cacheability disable.
+ * When set, forces all stage 2 translations for data accesses to normal memory
+ * to be non-cacheable.
+ */
+#define HCR_EL2_CD (UINT64_C(0x1) << 32)
+
+/**
+ * Execution state control for lower exception levels.
+ * When set, the execution state for EL1 is AArch64.
+ */
+#define HCR_EL2_RW (UINT64_C(0x1) << 31)
+
+/**
+ * Trap reads of Virtual Memory controls.
+ */
+#define HCR_EL2_TRVM (UINT64_C(0x1) << 30)
+
+/**
+ * Trap writes of Virtual Memory controls.
+ */
+#define HCR_EL2_TVM (UINT64_C(0x1) << 26)
+
+/**
+ * Trap TLB maintenance instructions.
+ */
+#define HCR_EL2_TTLB (UINT64_C(0x1) << 25)
+
+/**
+ * Trap cache maintenance instructions.
+ */
+#define HCR_EL2_TPU (UINT64_C(0x1) << 24)
+
+/**
+ * Trap data or unified cache maintenance instructions.
+ */
+#define HCR_EL2_TPCP (UINT64_C(0x1) << 23)
+
+/**
+ * Trap data or unified cache maintenance instructions that operate by Set/Way.
+ */
+#define HCR_EL2_TSW (UINT64_C(0x1) << 22)
+
+/**
+ * Trap Auxiliary Control Registers.
+ * When set, traps ACTLR_EL1 accesses to EL2.
+ */
+#define HCR_EL2_TACR (UINT64_C(0x1) << 21)
+
+/**
+ * Trap implementation defined functionality.
+ * When set, traps EL1 accesses to implementation defined encodings to EL2.
+ */
+#define HCR_EL2_TIDCP (UINT64_C(0x1) << 20)
+
+/**
+ * Trap SMC instructions.
+ * When set, traps EL1 execution of SMC instructions to EL2.
+ */
+#define HCR_EL2_TSC (UINT64_C(0x1) << 19)
+
+/**
+ * Trap ID group 3.
+ */
+#define HCR_EL2_TID3 (UINT64_C(0x1) << 18)
+
+/**
+ * Trap ID group 2.
+ */
+#define HCR_EL2_TID2 (UINT64_C(0x1) << 17)
+
+/**
+ * Trap ID group 1.
+ */
+#define HCR_EL2_TID1 (UINT64_C(0x1) << 16)
+
+/**
+ * Trap ID group 0.
+ */
+#define HCR_EL2_TID0 (UINT64_C(0x1) << 15)
+
+/**
+ * Traps EL0 and EL1 execution of Wait for Event (WFE) instructions to EL2.
+ */
+#define HCR_EL2_TWE (UINT64_C(0x1) << 14)
+
+/**
+ * Trap WFI instructions.
+ * When set, traps EL0 and EL1 execution of WFI instructions to EL2.
+ */
+#define HCR_EL2_TWI (UINT64_C(0x1) << 13)
+
+/**
+ * Barrier Shareability upgrade (2 bits).
+ * When set to 0b01, the minimum shareability domain to barrier instructions
+ * is inner shareable.
+ */
+#define HCR_EL2_BSU_INNER_SHAREABLE (UINT64_C(0x1) << 10)
+
+/**
+ * Force Broadcast.
+ * When set certain instructions related to invalidating the TLB are broadcast
+ * within the Inner Shareable domain.
+ */
+#define HCR_EL2_FB (UINT64_C(0x1) << 9)
+
+/**
+ * Virtual IRQ Interrupt.
+ * When set indicates that there is a virtual IRQ pending.
+ */
+#define HCR_EL2_VI (UINT64_C(0x1) << 7)
+
+/**
+ * Physical SError Routing.
+ * When set, physical SError interrupts are taken to EL2, unless routed to EL3.
+ */
+#define HCR_EL2_AMO (UINT64_C(0x1) << 5)
+
+/**
+ * Physical IRQ Routing.
+ * When set, physical IRQ interrupts are taken to EL2, unless routed to EL3.
+ */
+#define HCR_EL2_IMO (UINT64_C(0x1) << 4)
+
+/**
+ * Physical FIQ Routing.
+ * When set, physical FIQ interrupts are taken to EL2, unless routed to EL3.
+ */
+#define HCR_EL2_FMO (UINT64_C(0x1) << 3)
+
+/**
+ * Protected Table Walk.
+ * When set a translation table access made as part of a stage 1 translation
+ * table walk is subject to a stage 2 translation.  The memory access generates
+ * a stage 2 permission fault.
+ */
+#define HCR_EL2_PTW (UINT64_C(0x1) << 2)
+
+/**
+ * Set/Way Invalidation Override.
+ * Causes EL1 execution of the data cache invalidate by set/way instructions to
+ * perform a data cache clean and invalidate by set/way.
+ */
+#define HCR_EL2_SWIO (UINT64_C(0x1) << 1)
+
+/**
+ * Virtualization enable.
+ * When set EL1 and EL0 stage 2 address translation is enabled.
+ */
+#define HCR_EL2_VM (UINT64_C(0x1) << 0)
+
+/**
+ * Trap system register accesses to trace registers.
+ * Traps accesses to ETM registers using the register interface. Does not trap
+ * on accesses through the memory-mapped interface.
+ */
+#define CPTR_EL2_TTA (UINT64_C(0x1) << 28)
+
+/*
+ * Process State Bit definitions.
+ *
+ * These apply to the PSTATE, as well as registers that contain PSTATE fields,
+ * e.g., SPSR_EL1.
+ */
+
+/**
+ * Debug exception mask bit.
+ */
+#define PSR_D (UINT64_C(1) << 9)
+
+/**
+ * Asynchronous SError interrupt mask bit.
+ */
+#define PSR_A (UINT64_C(1) << 8)
+
+/**
+ * Asynchronous IRQ interrupt mask bit.
+ */
+#define PSR_I (UINT64_C(1) << 7)
+
+/**
+ * Asynchronous FIQ interrupt mask bit.
+ */
+#define PSR_F (UINT64_C(1) << 6)
+
+/**
+ * AArch32 State bit.
+ */
+#define PSR_ARCH_MODE_32 (UINT64_C(1) << 4)
+
+/**
+ * PE Mode bit mask.
+ */
+#define PSR_PE_MODE_MASK UINT64_C(0xf)
+
+/**
+ * PE Mode: EL0t.
+ */
+#define PSR_PE_MODE_EL0T UINT64_C(0x0)
+
+/**
+ * PE Mode: EL1h.
+ */
+#define PSR_PE_MODE_EL1H UINT64_C(0x5)
+
+uintreg_t get_hcr_el2_value(spci_vm_id_t vm_id);
+
+uintreg_t get_mdcr_el2_value(void);
+
+uintreg_t get_cptr_el2_value(void);
diff --git a/src/arch/aarch64/hypervisor/vm.c b/src/arch/aarch64/hypervisor/vm.c
new file mode 100644
index 0000000..619ee65
--- /dev/null
+++ b/src/arch/aarch64/hypervisor/vm.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm.h"
+
+#include "hypervisor/feature_id.h"
+
+void arch_vm_features_set(struct vm *vm)
+{
+	/* Features to trap for all VMs. */
+
+	/*
+	 * It is not safe to enable this yet, in part, because the feature's
+	 * registers are not context switched in Hafnium.
+	 */
+	vm->arch.trapped_features |= HF_FEATURE_LOR;
+
+	vm->arch.trapped_features |= HF_FEATURE_SPE;
+
+	vm->arch.trapped_features |= HF_FEATURE_TRACE;
+
+	vm->arch.trapped_features |= HF_FEATURE_DEBUG;
+
+	if (vm->id != HF_PRIMARY_VM_ID) {
+		/* Features to trap only for the secondary VMs. */
+
+		vm->arch.trapped_features |= HF_FEATURE_PERFMON;
+
+		/*
+		 * TODO(b/132395845): Access to RAS registers is not trapped at
+		 * the moment for the primary VM, only for the secondaries. RAS
+		 * register access isn't needed now, but it might be
+		 * required for debugging. When Hafnium introduces debug vs
+		 * release builds, trap accesses for primary VMs in release
+		 * builds, but do not trap them in debug builds.
+		 */
+		vm->arch.trapped_features |= HF_FEATURE_RAS;
+
+		/*
+		 * The PAuth mechanism holds state in the key registers. Only
+		 * the primary VM is allowed to use the PAuth functionality for
+		 * now. This prevents Hafnium from having to save/restore the
+		 * key register on a VM switch.
+		 */
+		vm->arch.trapped_features |= HF_FEATURE_PAUTH;
+	}
+}
diff --git a/src/arch/aarch64/inc/hf/arch/barriers.h b/src/arch/aarch64/inc/hf/arch/barriers.h
new file mode 100644
index 0000000..3d61434
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/barriers.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/** AArch64-specific API */
+
+/**
+ * Ensures explicit memory accesses before this point are completed before any
+ * later memory accesses are performed. The instruction argument specifies:
+ *   - the shareability domain over which the instruction must operate,
+ *   - the accesses for which the instruction operates.
+ */
+#define dmb(arg)                               \
+	do {                                   \
+		__asm__ volatile("dmb " #arg); \
+	} while (0)
+
+/**
+ * Ensures explicit memory access and management instructions have completed
+ * before continuing. The instruction argument specifies:
+ *   - the shareability domain over which the instruction must operate,
+ *   - the accesses for which the instruction operates.
+ */
+#define dsb(arg)                               \
+	do {                                   \
+		__asm__ volatile("dsb " #arg); \
+	} while (0)
+
+/**
+ * Flushes the instruction pipeline so that instructions are fetched from
+ * memory.
+ */
+#define isb()                            \
+	do {                             \
+		__asm__ volatile("isb"); \
+	} while (0)
+
+/** Platform-agnostic API */
+
+/**
+ * Ensures all explicit memory accesses before this point are completed before
+ * any later memory accesses are performed.
+ */
+#define memory_ordering_barrier() dmb(sy)
+
+/**
+ * Ensures all explicit memory access and management instructions have completed
+ * before continuing.
+ */
+#define data_sync_barrier() dsb(sy)
diff --git a/src/arch/aarch64/inc/hf/arch/spinlock.h b/src/arch/aarch64/inc/hf/arch/spinlock.h
new file mode 100644
index 0000000..3e3d1a0
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/spinlock.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/**
+ * Spinlock implementation using Armv8.0 LDXR/STXR pair and a WFE pause.
+ *
+ * Implementation using C11 atomics also generates a LDXR/STXR pair but no WFE.
+ * Without it we observe that Cortex A72 can easily livelock and not make
+ * forward progress.
+ *
+ * TODO(b/141087046): Forward progress is still not guaranteed as even with WFE
+ * we see that A72 can livelock for extremely tight loops. We should investigate
+ * the guarantees provided by atomic instructions introduced in Armv8.1 LSE.
+ */
+
+#include <stdint.h>
+
+#include "hf/arch/types.h"
+
+struct spinlock {
+	volatile uint32_t v;
+};
+
+#define SPINLOCK_INIT ((struct spinlock){.v = 0})
+
+static inline void sl_lock(struct spinlock *l)
+{
+	register uintreg_t tmp1;
+	register uintreg_t tmp2;
+
+	/*
+	 * Acquire the lock with a LDAXR/STXR pair (acquire semantics on the
+	 * load instruction). Pause using WFE if the lock is currently taken.
+	 * This is NOT guaranteed to make progress.
+	 */
+	__asm__ volatile(
+		"	mov	%w2, #1\n"
+		"	sevl\n" /* set event bit */
+		"1:	wfe\n"  /* wait for event, clear event bit */
+		"2:	ldaxr	%w1, [%0]\n"      /* load lock value */
+		"	cbnz	%w1, 1b\n"	/* if lock taken, goto WFE */
+		"	stxr	%w1, %w2, [%0]\n" /* try to take lock */
+		"	cbnz	%w1, 2b\n"	/* loop if unsuccessful */
+		: "+r"(l), "=&r"(tmp1), "=&r"(tmp2)
+		:
+		: "cc");
+}
+
+static inline void sl_unlock(struct spinlock *l)
+{
+	/*
+	 * Store zero to lock's value with release semantics. This triggers an
+	 * event which wakes up other threads waiting on a lock (no SEV needed).
+	 */
+	__asm__ volatile("stlr wzr, [%0]" : "+r"(l)::"cc");
+}
diff --git a/src/arch/aarch64/inc/hf/arch/types.h b/src/arch/aarch64/inc/hf/arch/types.h
new file mode 100644
index 0000000..667fc20
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/types.h
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/spci.h"
+#include "hf/static_assert.h"
+
+#define PAGE_BITS 12
+#define PAGE_LEVEL_BITS 9
+#define STACK_ALIGN 16
+#define FLOAT_REG_BYTES 16
+#define NUM_GP_REGS 31
+
+/** The type of a page table entry (PTE). */
+typedef uint64_t pte_t;
+
+/** Integer type large enough to hold a physical address. */
+typedef uintptr_t uintpaddr_t;
+
+/** Integer type large enough to hold a virtual address. */
+typedef uintptr_t uintvaddr_t;
+
+/** The integer type corresponding to the native register size. */
+typedef uint64_t uintreg_t;
+
+/** The ID of a physical or virtual CPU. */
+typedef uint64_t cpu_id_t;
+
+/** A bitset for AArch64 CPU features. */
+typedef uint64_t arch_features_t;
+
+/**
+ * The struct for storing a floating point register.
+ *
+ * 2 64-bit integers used to avoid need for FP support at this level.
+ */
+struct float_reg {
+	alignas(FLOAT_REG_BYTES) uint64_t low;
+	uint64_t high;
+};
+
+static_assert(sizeof(struct float_reg) == FLOAT_REG_BYTES,
+	      "Ensure float register type is 128 bits.");
+
+/** Arch-specific information about a VM. */
+struct arch_vm {
+	/**
+	 * The index of the last vCPU of this VM which ran on each pCPU. Each
+	 * element of this array should only be read or written by code running
+	 * on that CPU, which avoids contention and so no lock is needed to
+	 * access this field.
+	 */
+	spci_vcpu_index_t last_vcpu_on_cpu[MAX_CPUS];
+	arch_features_t trapped_features;
+
+	/*
+	 * Masks for feature registers trappable by HCR_EL2.TID3.
+	 */
+	struct {
+		uintreg_t id_aa64mmfr1_el1;
+		uintreg_t id_aa64pfr0_el1;
+		uintreg_t id_aa64dfr0_el1;
+		uintreg_t id_aa64isar1_el1;
+	} tid3_masks;
+};
+
+/** Type to represent the register state of a vCPU.  */
+struct arch_regs {
+	/* General purpose registers. */
+	uintreg_t r[NUM_GP_REGS];
+	uintreg_t pc;
+	uintreg_t spsr;
+
+	/*
+	 * System registers.
+	 * NOTE: Ordering is important. If adding to or reordering registers
+	 * below, make sure to update src/arch/aarch64/hypervisor/exceptions.S.
+	 */
+	struct {
+		uintreg_t vmpidr_el2;
+		uintreg_t csselr_el1;
+		uintreg_t sctlr_el1;
+		uintreg_t actlr_el1;
+		uintreg_t cpacr_el1;
+		uintreg_t ttbr0_el1;
+		uintreg_t ttbr1_el1;
+		uintreg_t tcr_el1;
+		uintreg_t esr_el1;
+		uintreg_t afsr0_el1;
+		uintreg_t afsr1_el1;
+		uintreg_t far_el1;
+		uintreg_t mair_el1;
+		uintreg_t vbar_el1;
+		uintreg_t contextidr_el1;
+		uintreg_t tpidr_el0;
+		uintreg_t tpidrro_el0;
+		uintreg_t tpidr_el1;
+		uintreg_t amair_el1;
+		uintreg_t cntkctl_el1;
+		uintreg_t sp_el0;
+		uintreg_t sp_el1;
+		uintreg_t elr_el1;
+		uintreg_t spsr_el1;
+		uintreg_t par_el1;
+		uintreg_t hcr_el2;
+		uintreg_t cnthctl_el2;
+		uintreg_t vttbr_el2;
+		uintreg_t mdcr_el2;
+		uintreg_t mdscr_el1;
+		uintreg_t pmccfiltr_el0;
+		uintreg_t pmcr_el0;
+		uintreg_t pmcntenset_el0;
+		uintreg_t pmintenset_el1;
+	} lazy;
+
+	/* Floating point registers. */
+	struct float_reg fp[32];
+	uintreg_t fpsr;
+	uintreg_t fpcr;
+
+#if GIC_VERSION == 3 || GIC_VERSION == 4
+	struct {
+		uintreg_t ich_hcr_el2;
+		uintreg_t icc_sre_el2;
+	} gic;
+#endif
+
+	/*
+	 * Peripheral registers, handled separately from other system registers.
+	 */
+	struct {
+		uintreg_t cntv_cval_el0;
+		uintreg_t cntv_ctl_el0;
+	} peripherals;
+};
diff --git a/src/arch/aarch64/inc/hf/arch/vm/events.h b/src/arch/aarch64/inc/hf/arch/vm/events.h
new file mode 100644
index 0000000..aff4d77
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/events.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+void event_wait(void);
+void event_send_local(void);
diff --git a/src/arch/aarch64/inc/hf/arch/vm/interrupts.h b/src/arch/aarch64/inc/hf/arch/vm/interrupts.h
new file mode 100644
index 0000000..33087b6
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/interrupts.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+
+void exception_setup(void (*irq)(void), bool (*exception)(void));
+void interrupt_wait(void);
diff --git a/src/arch/aarch64/inc/hf/arch/vm/interrupts_gicv3.h b/src/arch/aarch64/inc/hf/arch/vm/interrupts_gicv3.h
new file mode 100644
index 0000000..b98c304
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/interrupts_gicv3.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "hf/io.h"
+
+#if GIC_VERSION != 3 && GIC_VERSION != 4
+#error This header should only be included for GICv3 or v4.
+#endif
+
+/* Keep macro alignment */
+/* clang-format off */
+
+#define SGI_BASE (GICR_BASE + 0x10000)
+
+#define GICD_CTLR       IO32_C(GICD_BASE + 0x0000)
+#define GICD_ISENABLER  IO32_ARRAY_C(GICD_BASE + 0x0100, 32)
+#define GICD_ICENABLER  IO32_ARRAY_C(GICD_BASE + 0x0180, 32)
+#define GICD_ISPENDR    IO32_ARRAY_C(GICD_BASE + 0x0200, 32)
+#define GICD_ICPENDR    IO32_ARRAY_C(GICD_BASE + 0x0280, 32)
+#define GICD_ISACTIVER  IO32_ARRAY_C(GICD_BASE + 0x0300, 32)
+#define GICD_ICACTIVER  IO32_ARRAY_C(GICD_BASE + 0x0380, 32)
+#define GICD_IPRIORITYR IO8_ARRAY_C(GICD_BASE + 0x0400, 1020)
+#define GICD_ITARGETSR  IO8_ARRAY_C(GICD_BASE + 0x0800, 1020)
+#define GICD_ICFGR      IO32_ARRAY_C(GICD_BASE + 0x0c00, 64)
+#define GICR_WAKER      IO32_C(GICR_BASE + 0x0014)
+#define GICR_IGROUPR0   IO32_C(SGI_BASE + 0x0080)
+#define GICR_ISENABLER0 IO32_C(SGI_BASE + 0x0100)
+#define GICR_ICENABLER0 IO32_C(SGI_BASE + 0x0180)
+#define GICR_ISPENDR0   IO32_C(SGI_BASE + 0x0200)
+#define GICR_ICPENDR0   IO32_C(SGI_BASE + 0x0280)
+#define GICR_ISACTIVER0 IO32_C(SGI_BASE + 0x0300)
+#define GICR_ICFGR      IO32_ARRAY_C(SGI_BASE + 0x0c00, 32)
+
+/* clang-format on */
+
+void interrupt_gic_setup(void);
+void interrupt_enable(uint32_t intid, bool enable);
+void interrupt_enable_all(bool enable);
+void interrupt_set_priority_mask(uint8_t min_priority);
+void interrupt_set_priority(uint32_t intid, uint8_t priority);
+void interrupt_set_edge_triggered(uint32_t intid, bool edge_triggered);
+void interrupt_send_sgi(uint8_t intid, bool irm, uint8_t affinity3,
+			uint8_t affinity2, uint8_t affinity1,
+			uint16_t target_list);
+uint32_t interrupt_get_and_acknowledge(void);
+void interrupt_end(uint32_t intid);
diff --git a/src/arch/aarch64/inc/hf/arch/vm/mm.h b/src/arch/aarch64/inc/hf/arch/vm/mm.h
new file mode 100644
index 0000000..d212fa5
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/mm.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "hf/mm.h"
+
+bool arch_vm_mm_init(void);
+void arch_vm_mm_enable(paddr_t table);
diff --git a/src/arch/aarch64/inc/hf/arch/vm/power_mgmt.h b/src/arch/aarch64/inc/hf/arch/vm/power_mgmt.h
new file mode 100644
index 0000000..41ebe49
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/power_mgmt.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdnoreturn.h>
+
+#include "hf/arch/types.h"
+
+enum power_status {
+	POWER_STATUS_ON,
+	POWER_STATUS_OFF,
+	POWER_STATUS_ON_PENDING,
+};
+
+/**
+ * Holds temporary state used to set up the environment on which CPUs will
+ * start executing.
+ *
+ * vm_cpu_entry() depends on the layout of this struct.
+ */
+struct arch_cpu_start_state {
+	uintptr_t initial_sp;
+	void (*entry)(uintreg_t arg);
+	uintreg_t arg;
+};
+
+bool arch_cpu_start(uintptr_t id, struct arch_cpu_start_state *s);
+
+noreturn void arch_cpu_stop(void);
+enum power_status arch_cpu_status(cpu_id_t cpu_id);
+
+noreturn void arch_power_off(void);
diff --git a/src/arch/aarch64/inc/hf/arch/vm/registers.h b/src/arch/aarch64/inc/hf/arch/vm/registers.h
new file mode 100644
index 0000000..f6305af
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/registers.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+
+void fill_fp_registers(double value);
+bool check_fp_register(double value);
diff --git a/src/arch/aarch64/inc/hf/arch/vm/state.h b/src/arch/aarch64/inc/hf/arch/vm/state.h
new file mode 100644
index 0000000..8f2dbe2
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/state.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+void per_cpu_ptr_set(uintptr_t v);
+uintptr_t per_cpu_ptr_get(void);
diff --git a/src/arch/aarch64/inc/hf/arch/vm/timer.h b/src/arch/aarch64/inc/hf/arch/vm/timer.h
new file mode 100644
index 0000000..d1c07a3
--- /dev/null
+++ b/src/arch/aarch64/inc/hf/arch/vm/timer.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include "../msr.h"
+
+static inline void timer_set(uint32_t ticks)
+{
+	write_msr(CNTV_TVAL_EL0, ticks);
+}
+
+static inline void timer_start(void)
+{
+	write_msr(CNTV_CTL_EL0, 0x00000001);
+}
diff --git a/src/arch/aarch64/irq.c b/src/arch/aarch64/irq.c
new file mode 100644
index 0000000..4114d76
--- /dev/null
+++ b/src/arch/aarch64/irq.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+
+#include "msr.h"
+
+void arch_irq_disable(void)
+{
+	__asm__ volatile("msr DAIFSet, #0xf");
+}
+
+void arch_irq_enable(void)
+{
+	__asm__ volatile("msr DAIFClr, #0xf");
+}
diff --git a/src/arch/aarch64/mm.c b/src/arch/aarch64/mm.c
new file mode 100644
index 0000000..f856ac4
--- /dev/null
+++ b/src/arch/aarch64/mm.c
@@ -0,0 +1,700 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/mm.h"
+
+#include "hf/arch/barriers.h"
+#include "hf/arch/cpu.h"
+
+#include "hf/dlog.h"
+
+#include "msr.h"
+
+/* Keep macro alignment */
+/* clang-format off */
+
+#define NON_SHAREABLE   UINT64_C(0)
+#define OUTER_SHAREABLE UINT64_C(2)
+#define INNER_SHAREABLE UINT64_C(3)
+
+#define PTE_VALID        (UINT64_C(1) << 0)
+#define PTE_LEVEL0_BLOCK (UINT64_C(1) << 1)
+#define PTE_TABLE        (UINT64_C(1) << 1)
+
+#define STAGE1_XN          (UINT64_C(1) << 54)
+#define STAGE1_PXN         (UINT64_C(1) << 53)
+#define STAGE1_CONTIGUOUS  (UINT64_C(1) << 52)
+#define STAGE1_DBM         (UINT64_C(1) << 51)
+#define STAGE1_NG          (UINT64_C(1) << 11)
+#define STAGE1_AF          (UINT64_C(1) << 10)
+#define STAGE1_SH(x)       ((x) << 8)
+#define STAGE1_AP2         (UINT64_C(1) << 7)
+#define STAGE1_AP1         (UINT64_C(1) << 6)
+#define STAGE1_AP(x)       ((x) << 6)
+#define STAGE1_NS          (UINT64_C(1) << 5)
+#define STAGE1_ATTRINDX(x) ((x) << 2)
+
+#define STAGE1_READONLY  UINT64_C(2)
+#define STAGE1_READWRITE UINT64_C(0)
+
+#define STAGE1_DEVICEINDX UINT64_C(0)
+#define STAGE1_NORMALINDX UINT64_C(1)
+
+#define STAGE2_XN(x)      ((x) << 53)
+#define STAGE2_CONTIGUOUS (UINT64_C(1) << 52)
+#define STAGE2_DBM        (UINT64_C(1) << 51)
+#define STAGE2_AF         (UINT64_C(1) << 10)
+#define STAGE2_SH(x)      ((x) << 8)
+#define STAGE2_S2AP(x)    ((x) << 6)
+
+#define STAGE2_EXECUTE_ALL  UINT64_C(0)
+#define STAGE2_EXECUTE_EL0  UINT64_C(1)
+#define STAGE2_EXECUTE_NONE UINT64_C(2)
+#define STAGE2_EXECUTE_EL1  UINT64_C(3)
+#define STAGE2_EXECUTE_MASK UINT64_C(3)
+
+/* Table attributes only apply to stage 1 translations. */
+#define TABLE_NSTABLE  (UINT64_C(1) << 63)
+#define TABLE_APTABLE1 (UINT64_C(1) << 62)
+#define TABLE_APTABLE0 (UINT64_C(1) << 61)
+#define TABLE_XNTABLE  (UINT64_C(1) << 60)
+#define TABLE_PXNTABLE (UINT64_C(1) << 59)
+
+/* The following are stage-2 software defined attributes. */
+#define STAGE2_SW_OWNED     (UINT64_C(1) << 55)
+#define STAGE2_SW_EXCLUSIVE (UINT64_C(1) << 56)
+
+/* The following are stage-2 memory attributes for normal memory. */
+#define STAGE2_DEVICE_MEMORY UINT64_C(0)
+#define STAGE2_NONCACHEABLE  UINT64_C(1)
+#define STAGE2_WRITETHROUGH  UINT64_C(2)
+#define STAGE2_WRITEBACK     UINT64_C(3)
+
+/* The following are stage-2 memory attributes for device memory. */
+#define STAGE2_MEMATTR_DEVICE_nGnRnE UINT64_C(0)
+#define STAGE2_MEMATTR_DEVICE_nGnRE  UINT64_C(1)
+#define STAGE2_MEMATTR_DEVICE_nGRE   UINT64_C(2)
+#define STAGE2_MEMATTR_DEVICE_GRE    UINT64_C(3)
+
+/* The following construct and destruct stage-2 memory attributes. */
+#define STAGE2_MEMATTR(outer, inner) ((((outer) << 2) | (inner)) << 2)
+#define STAGE2_MEMATTR_TYPE_MASK UINT64_C(3 << 4)
+
+#define STAGE2_ACCESS_READ  UINT64_C(1)
+#define STAGE2_ACCESS_WRITE UINT64_C(2)
+
+#define CACHE_WORD_SIZE 4
+
+/**
+ * Threshold number of pages in TLB to invalidate after which we invalidate all
+ * TLB entries on a given level.
+ * Constant is the number of pointers per page table entry, also used by Linux.
+ */
+#define MAX_TLBI_OPS  MM_PTE_PER_PAGE
+
+/* clang-format on */
+
+#define tlbi(op)                               \
+	do {                                   \
+		__asm__ volatile("tlbi " #op); \
+	} while (0)
+#define tlbi_reg(op, reg)                                              \
+	do {                                                           \
+		__asm__ __volatile__("tlbi " #op ", %0" : : "r"(reg)); \
+	} while (0)
+
+/** Mask for the address bits of the pte. */
+#define PTE_ADDR_MASK \
+	(((UINT64_C(1) << 48) - 1) & ~((UINT64_C(1) << PAGE_BITS) - 1))
+
+/** Mask for the attribute bits of the pte. */
+#define PTE_ATTR_MASK (~(PTE_ADDR_MASK | (UINT64_C(1) << 1)))
+
+/**
+ * Configuration information for memory management. Order is important as this
+ * is read from assembly.
+ *
+ * It must only be written to from `arch_mm_one_time_init()` to avoid cache and
+ * synchronization problems.
+ */
+struct arch_mm_config {
+	uintreg_t ttbr0_el2;
+	uintreg_t vtcr_el2;
+	uintreg_t mair_el2;
+	uintreg_t tcr_el2;
+	uintreg_t sctlr_el2;
+} arch_mm_config;
+
+static uint8_t mm_s2_max_level;
+static uint8_t mm_s2_root_table_count;
+
+/**
+ * Returns the encoding of a page table entry that isn't present.
+ */
+pte_t arch_mm_absent_pte(uint8_t level)
+{
+	(void)level;
+	return 0;
+}
+
+/**
+ * Converts a physical address to a table PTE.
+ *
+ * The spec says that 'Table descriptors for stage 2 translations do not
+ * include any attribute field', so we don't take any attributes as arguments.
+ */
+pte_t arch_mm_table_pte(uint8_t level, paddr_t pa)
+{
+	/* This is the same for all levels on aarch64. */
+	(void)level;
+	return pa_addr(pa) | PTE_TABLE | PTE_VALID;
+}
+
+/**
+ * Converts a physical address to a block PTE.
+ *
+ * The level must allow block entries.
+ */
+pte_t arch_mm_block_pte(uint8_t level, paddr_t pa, uint64_t attrs)
+{
+	pte_t pte = pa_addr(pa) | attrs;
+
+	if (level == 0) {
+		/* A level 0 'block' is actually a page entry. */
+		pte |= PTE_LEVEL0_BLOCK;
+	}
+	return pte;
+}
+
+/**
+ * Specifies whether block mappings are acceptable at the given level.
+ *
+ * Level 0 must allow block entries.
+ */
+bool arch_mm_is_block_allowed(uint8_t level)
+{
+	return level <= 2;
+}
+
+/**
+ * Determines if the given pte is present, i.e., if it is valid or it is invalid
+ * but still holds state about the memory so needs to be present in the table.
+ */
+bool arch_mm_pte_is_present(pte_t pte, uint8_t level)
+{
+	return arch_mm_pte_is_valid(pte, level) || (pte & STAGE2_SW_OWNED) != 0;
+}
+
+/**
+ * Determines if the given pte is valid, i.e., if it points to another table,
+ * to a page, or a block of pages that can be accessed.
+ */
+bool arch_mm_pte_is_valid(pte_t pte, uint8_t level)
+{
+	(void)level;
+	return (pte & PTE_VALID) != 0;
+}
+
+/**
+ * Determines if the given pte references a block of pages.
+ */
+bool arch_mm_pte_is_block(pte_t pte, uint8_t level)
+{
+	/* We count pages at level 0 as blocks. */
+	return arch_mm_is_block_allowed(level) &&
+	       (level == 0 ? (pte & PTE_LEVEL0_BLOCK) != 0
+			   : arch_mm_pte_is_present(pte, level) &&
+				     !arch_mm_pte_is_table(pte, level));
+}
+
+/**
+ * Determines if the given pte references another table.
+ */
+bool arch_mm_pte_is_table(pte_t pte, uint8_t level)
+{
+	return level != 0 && arch_mm_pte_is_valid(pte, level) &&
+	       (pte & PTE_TABLE) != 0;
+}
+
+static uint64_t pte_addr(pte_t pte)
+{
+	return pte & PTE_ADDR_MASK;
+}
+
+/**
+ * Clears the given physical address, i.e., clears the bits of the address that
+ * are not used in the pte.
+ */
+paddr_t arch_mm_clear_pa(paddr_t pa)
+{
+	return pa_init(pte_addr(pa_addr(pa)));
+}
+
+/**
+ * Extracts the physical address of the block referred to by the given page
+ * table entry.
+ */
+paddr_t arch_mm_block_from_pte(pte_t pte, uint8_t level)
+{
+	(void)level;
+	return pa_init(pte_addr(pte));
+}
+
+/**
+ * Extracts the physical address of the page table referred to by the given page
+ * table entry.
+ */
+paddr_t arch_mm_table_from_pte(pte_t pte, uint8_t level)
+{
+	(void)level;
+	return pa_init(pte_addr(pte));
+}
+
+/**
+ * Extracts the architecture-specific attributes applies to the given page table
+ * entry.
+ */
+uint64_t arch_mm_pte_attrs(pte_t pte, uint8_t level)
+{
+	(void)level;
+	return pte & PTE_ATTR_MASK;
+}
+
+/**
+ * Invalidates stage-1 TLB entries referring to the given virtual address range.
+ */
+void arch_mm_invalidate_stage1_range(vaddr_t va_begin, vaddr_t va_end)
+{
+	uintvaddr_t begin = va_addr(va_begin);
+	uintvaddr_t end = va_addr(va_end);
+	uintvaddr_t it;
+
+	/* Sync with page table updates. */
+	dsb(ishst);
+
+	/*
+	 * Revisions prior to Armv8.4 do not support invalidating a range of
+	 * addresses, which means we have to loop over individual pages. If
+	 * there are too many, it is quicker to invalidate all TLB entries.
+	 */
+	if ((end - begin) > (MAX_TLBI_OPS * PAGE_SIZE)) {
+		if (VM_TOOLCHAIN == 1) {
+			tlbi(vmalle1is);
+		} else {
+			tlbi(alle2is);
+		}
+	} else {
+		begin >>= 12;
+		end >>= 12;
+		/* Invalidate stage-1 TLB, one page from the range at a time. */
+		for (it = begin; it < end;
+		     it += (UINT64_C(1) << (PAGE_BITS - 12))) {
+			if (VM_TOOLCHAIN == 1) {
+				tlbi_reg(vae1is, it);
+			} else {
+				tlbi_reg(vae2is, it);
+			}
+		}
+	}
+
+	/* Sync data accesses with TLB invalidation completion. */
+	dsb(ish);
+
+	/* Sync instruction fetches with TLB invalidation completion. */
+	isb();
+}
+
+/**
+ * Invalidates stage-2 TLB entries referring to the given intermediate physical
+ * address range.
+ */
+void arch_mm_invalidate_stage2_range(ipaddr_t va_begin, ipaddr_t va_end)
+{
+	uintpaddr_t begin = ipa_addr(va_begin);
+	uintpaddr_t end = ipa_addr(va_end);
+	uintpaddr_t it;
+
+	/* TODO: This only applies to the current VMID. */
+
+	/* Sync with page table updates. */
+	dsb(ishst);
+
+	/*
+	 * Revisions prior to Armv8.4 do not support invalidating a range of
+	 * addresses, which means we have to loop over individual pages. If
+	 * there are too many, it is quicker to invalidate all TLB entries.
+	 */
+	if ((end - begin) > (MAX_TLBI_OPS * PAGE_SIZE)) {
+		/*
+		 * Invalidate all stage-1 and stage-2 entries of the TLB for
+		 * the current VMID.
+		 */
+		tlbi(vmalls12e1is);
+	} else {
+		begin >>= 12;
+		end >>= 12;
+
+		/*
+		 * Invalidate stage-2 TLB, one page from the range at a time.
+		 * Note that this has no effect if the CPU has a TLB with
+		 * combined stage-1/stage-2 translation.
+		 */
+		for (it = begin; it < end;
+		     it += (UINT64_C(1) << (PAGE_BITS - 12))) {
+			tlbi_reg(ipas2e1is, it);
+		}
+
+		/*
+		 * Ensure completion of stage-2 invalidation in case a page
+		 * table walk on another CPU refilled the TLB with a complete
+		 * stage-1 + stage-2 walk based on the old stage-2 mapping.
+		 */
+		dsb(ish);
+
+		/*
+		 * Invalidate all stage-1 TLB entries. If the CPU has a combined
+		 * TLB for stage-1 and stage-2, this will invalidate stage-2 as
+		 * well.
+		 */
+		tlbi(vmalle1is);
+	}
+
+	/* Sync data accesses with TLB invalidation completion. */
+	dsb(ish);
+
+	/* Sync instruction fetches with TLB invalidation completion. */
+	isb();
+}
+
+/**
+ * Returns the smallest cache line size of all the caches for this core.
+ */
+static uint16_t arch_mm_dcache_line_size(void)
+{
+	return CACHE_WORD_SIZE *
+	       (UINT16_C(1) << ((read_msr(CTR_EL0) >> 16) & 0xf));
+}
+
+void arch_mm_flush_dcache(void *base, size_t size)
+{
+	/* Clean and invalidate each data cache line in the range. */
+	uint16_t line_size = arch_mm_dcache_line_size();
+	uintptr_t line_begin = (uintptr_t)base & ~(line_size - 1);
+	uintptr_t end = (uintptr_t)base + size;
+
+	while (line_begin < end) {
+		__asm__ volatile("dc civac, %0" : : "r"(line_begin));
+		line_begin += line_size;
+	}
+	dsb(sy);
+}
+
+uint64_t arch_mm_mode_to_stage1_attrs(uint32_t mode)
+{
+	uint64_t attrs = 0;
+
+	attrs |= STAGE1_AF | STAGE1_SH(OUTER_SHAREABLE);
+
+	/* Define the execute bits. */
+	if (!(mode & MM_MODE_X)) {
+		attrs |= STAGE1_XN;
+	}
+
+	/* Define the read/write bits. */
+	if (mode & MM_MODE_W) {
+		attrs |= STAGE1_AP(STAGE1_READWRITE);
+	} else {
+		attrs |= STAGE1_AP(STAGE1_READONLY);
+	}
+
+	/* Define the memory attribute bits. */
+	if (mode & MM_MODE_D) {
+		attrs |= STAGE1_ATTRINDX(STAGE1_DEVICEINDX);
+	} else {
+		attrs |= STAGE1_ATTRINDX(STAGE1_NORMALINDX);
+	}
+
+	/* Define the valid bit. */
+	if (!(mode & MM_MODE_INVALID)) {
+		attrs |= PTE_VALID;
+	}
+
+	return attrs;
+}
+
+uint64_t arch_mm_mode_to_stage2_attrs(uint32_t mode)
+{
+	uint64_t attrs = 0;
+	uint64_t access = 0;
+
+	/*
+	 * Non-shareable is the "neutral" share mode, i.e., the shareability
+	 * attribute of stage 1 will determine the actual attribute.
+	 */
+	attrs |= STAGE2_AF | STAGE2_SH(NON_SHAREABLE);
+
+	/* Define the read/write bits. */
+	if (mode & MM_MODE_R) {
+		access |= STAGE2_ACCESS_READ;
+	}
+
+	if (mode & MM_MODE_W) {
+		access |= STAGE2_ACCESS_WRITE;
+	}
+
+	attrs |= STAGE2_S2AP(access);
+
+	/* Define the execute bits. */
+	if (mode & MM_MODE_X) {
+		attrs |= STAGE2_XN(STAGE2_EXECUTE_ALL);
+	} else {
+		attrs |= STAGE2_XN(STAGE2_EXECUTE_NONE);
+	}
+
+	/*
+	 * Define the memory attribute bits, using the "neutral" values which
+	 * give the stage-1 attributes full control of the attributes.
+	 */
+	if (mode & MM_MODE_D) {
+		attrs |= STAGE2_MEMATTR(STAGE2_DEVICE_MEMORY,
+					STAGE2_MEMATTR_DEVICE_GRE);
+	} else {
+		attrs |= STAGE2_MEMATTR(STAGE2_WRITEBACK, STAGE2_WRITEBACK);
+	}
+
+	/* Define the ownership bit. */
+	if (!(mode & MM_MODE_UNOWNED)) {
+		attrs |= STAGE2_SW_OWNED;
+	}
+
+	/* Define the exclusivity bit. */
+	if (!(mode & MM_MODE_SHARED)) {
+		attrs |= STAGE2_SW_EXCLUSIVE;
+	}
+
+	/* Define the valid bit. */
+	if (!(mode & MM_MODE_INVALID)) {
+		attrs |= PTE_VALID;
+	}
+
+	return attrs;
+}
+
+uint32_t arch_mm_stage2_attrs_to_mode(uint64_t attrs)
+{
+	uint32_t mode = 0;
+
+	if (attrs & STAGE2_S2AP(STAGE2_ACCESS_READ)) {
+		mode |= MM_MODE_R;
+	}
+
+	if (attrs & STAGE2_S2AP(STAGE2_ACCESS_WRITE)) {
+		mode |= MM_MODE_W;
+	}
+
+	if ((attrs & STAGE2_XN(STAGE2_EXECUTE_MASK)) ==
+	    STAGE2_XN(STAGE2_EXECUTE_ALL)) {
+		mode |= MM_MODE_X;
+	}
+
+	if ((attrs & STAGE2_MEMATTR_TYPE_MASK) == STAGE2_DEVICE_MEMORY) {
+		mode |= MM_MODE_D;
+	}
+
+	if (!(attrs & STAGE2_SW_OWNED)) {
+		mode |= MM_MODE_UNOWNED;
+	}
+
+	if (!(attrs & STAGE2_SW_EXCLUSIVE)) {
+		mode |= MM_MODE_SHARED;
+	}
+
+	if (!(attrs & PTE_VALID)) {
+		mode |= MM_MODE_INVALID;
+	}
+
+	return mode;
+}
+
+uint8_t arch_mm_stage1_max_level(void)
+{
+	/*
+	 * For stage 1 we hard-code this to 2 for now so that we can
+	 * save one page table level at the expense of limiting the
+	 * physical memory to 512GB.
+	 */
+	return 2;
+}
+
+uint8_t arch_mm_stage2_max_level(void)
+{
+	return mm_s2_max_level;
+}
+
+uint8_t arch_mm_stage1_root_table_count(void)
+{
+	/* Stage 1 doesn't concatenate tables. */
+	return 1;
+}
+
+uint8_t arch_mm_stage2_root_table_count(void)
+{
+	return mm_s2_root_table_count;
+}
+
+/**
+ * Given the attrs from a table at some level and the attrs from all the blocks
+ * in that table, returns equivalent attrs to use for a block which will replace
+ * the entire table.
+ */
+uint64_t arch_mm_combine_table_entry_attrs(uint64_t table_attrs,
+					   uint64_t block_attrs)
+{
+	/*
+	 * Only stage 1 table descriptors have attributes, but the bits are res0
+	 * for stage 2 table descriptors so this code is safe for both.
+	 */
+	if (table_attrs & TABLE_NSTABLE) {
+		block_attrs |= STAGE1_NS;
+	}
+	if (table_attrs & TABLE_APTABLE1) {
+		block_attrs |= STAGE1_AP2;
+	}
+	if (table_attrs & TABLE_APTABLE0) {
+		block_attrs &= ~STAGE1_AP1;
+	}
+	if (table_attrs & TABLE_XNTABLE) {
+		block_attrs |= STAGE1_XN;
+	}
+	if (table_attrs & TABLE_PXNTABLE) {
+		block_attrs |= STAGE1_PXN;
+	}
+	return block_attrs;
+}
+
+/**
+ * This is called early in initialization without MMU or caches enabled.
+ */
+bool arch_mm_init(paddr_t table)
+{
+	static const int pa_bits_table[16] = {32, 36, 40, 42, 44, 48};
+	uint64_t features = read_msr(id_aa64mmfr0_el1);
+	int pa_bits = pa_bits_table[features & 0xf];
+	int extend_bits;
+	int sl0;
+
+	/* Check that 4KB granules are supported. */
+	if ((features >> 28) & 0xf) {
+		dlog("4KB granules are not supported\n");
+		return false;
+	}
+
+	/* Check the physical address range. */
+	if (!pa_bits) {
+		dlog("Unsupported value of id_aa64mmfr0_el1.PARange: %x\n",
+		     features & 0xf);
+		return false;
+	}
+
+	dlog("Supported bits in physical address: %d\n", pa_bits);
+
+	/*
+	 * Determine sl0, starting level of the page table, based on the number
+	 * of bits. The value is chosen to give the shallowest tree by making
+	 * use of concatenated translation tables.
+	 *
+	 *  - 0 => start at level 1
+	 *  - 1 => start at level 2
+	 *  - 2 => start at level 3
+	 */
+	if (pa_bits >= 44) {
+		sl0 = 2;
+		mm_s2_max_level = 3;
+	} else if (pa_bits >= 35) {
+		sl0 = 1;
+		mm_s2_max_level = 2;
+	} else {
+		sl0 = 0;
+		mm_s2_max_level = 1;
+	}
+
+	/*
+	 * Since the shallowest possible tree is used, the maximum number of
+	 * concatenated tables must be used. This means if no more than 4 bits
+	 * are used from the next level, they are instead used to index into the
+	 * concatenated tables.
+	 */
+	extend_bits = ((pa_bits - PAGE_BITS) % PAGE_LEVEL_BITS);
+	if (extend_bits > 4) {
+		extend_bits = 0;
+	}
+	mm_s2_root_table_count = 1 << extend_bits;
+
+	dlog("Stage 2 has %d page table levels with %d pages at the root.\n",
+	     mm_s2_max_level + 1, mm_s2_root_table_count);
+
+	arch_mm_config = (struct arch_mm_config){
+		.ttbr0_el2 = pa_addr(table),
+
+		.vtcr_el2 =
+			(1U << 31) |		   /* RES1. */
+			((features & 0xf) << 16) | /* PS, matching features. */
+			(0 << 14) |		   /* TG0: 4 KB granule. */
+			(3 << 12) |		   /* SH0: inner shareable. */
+			(1 << 10) |  /* ORGN0: normal, cacheable ... */
+			(1 << 8) |   /* IRGN0: normal, cacheable ... */
+			(sl0 << 6) | /* SL0. */
+			((64 - pa_bits) << 0) | /* T0SZ: dependent on PS. */
+			0,
+
+		/*
+		 * 0    -> Device-nGnRnE memory
+		 * 0xff -> Normal memory, Inner/Outer Write-Back Non-transient,
+		 *         Write-Alloc, Read-Alloc.
+		 */
+		.mair_el2 = (0 << (8 * STAGE1_DEVICEINDX)) |
+			    (0xff << (8 * STAGE1_NORMALINDX)),
+
+		/*
+		 * Configure tcr_el2.
+		 */
+		.tcr_el2 =
+			(1 << 20) |		   /* TBI, top byte ignored. */
+			((features & 0xf) << 16) | /* PS. */
+			(0 << 14) |		   /* TG0, granule size, 4KB. */
+			(3 << 12) |		   /* SH0, inner shareable. */
+			(1 << 10) | /* ORGN0, normal mem, WB RA WA Cacheable. */
+			(1 << 8) |  /* IRGN0, normal mem, WB RA WA Cacheable. */
+			(25 << 0) | /* T0SZ, input address is 2^39 bytes. */
+			0,
+
+		.sctlr_el2 = (1 << 0) |  /* M, enable stage 1 EL2 MMU. */
+			     (1 << 1) |  /* A, enable alignment check faults. */
+			     (1 << 2) |  /* C, data cache enable. */
+			     (1 << 3) |  /* SA, enable stack alignment check. */
+			     (3 << 4) |  /* RES1 bits. */
+			     (1 << 11) | /* RES1 bit. */
+			     (1 << 12) | /* I, instruction cache enable. */
+			     (1 << 16) | /* RES1 bit. */
+			     (1 << 18) | /* RES1 bit. */
+			     (1 << 19) | /* WXN bit, writable execute never. */
+			     (3 << 22) | /* RES1 bits. */
+			     (3 << 28) | /* RES1 bits. */
+			     0,
+	};
+
+	return true;
+}
diff --git a/src/arch/aarch64/msr.h b/src/arch/aarch64/msr.h
new file mode 100644
index 0000000..1b96538
--- /dev/null
+++ b/src/arch/aarch64/msr.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stddef.h>
+
+#include "hf/arch/cpu.h"
+
+/**
+ * Macros to stringify a parameter, and to allow the results of a macro to be
+ * stringified in turn.
+ */
+#define str_(s) #s
+#define str(s) str_(s)
+
+/**
+ * Reads a system register, supported by the current assembler, and returns the
+ * result.
+ */
+#define read_msr(name)                                              \
+	__extension__({                                             \
+		uintreg_t __v;                                      \
+		__asm__ volatile("mrs %0, " str(name) : "=r"(__v)); \
+		__v;                                                \
+	})
+
+/**
+ * Writes the value to the system register, supported by the current assembler.
+ */
+#define write_msr(name, value)                                \
+	__extension__({                                       \
+		__asm__ volatile("msr " str(name) ", %x0"     \
+				 :                            \
+				 : "rZ"((uintreg_t)(value))); \
+	})
+
+/*
+ * Encodings for registers supported after Armv8.0.
+ * We aim to build one binary that supports a variety of platforms, therefore,
+ * use encodings in Arm Architecture Reference Manual Armv8-a, D13.2 for
+ * registers supported after Armv8.0.
+ */
+
+/*
+ * Registers supported from Armv8.1 onwards.
+ */
+
+/*
+ * Registers for feature Armv8.1-LOR (Limited Ordering Regions).
+ */
+
+/**
+ * Encoding for the LORegion Control register (LORC_EL1).
+ * This register enables and disables LORegions (Armv8.1).
+ */
+#define MSR_LORC_EL1 S3_0_C10_C4_3
diff --git a/src/arch/aarch64/pl011/BUILD.gn b/src/arch/aarch64/pl011/BUILD.gn
new file mode 100644
index 0000000..7d97e33
--- /dev/null
+++ b/src/arch/aarch64/pl011/BUILD.gn
@@ -0,0 +1,29 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("args.gni")
+
+# aarch64 PL011 implementation of putchar for debugging.
+source_set("pl011") {
+  sources = [
+    "pl011.c",
+  ]
+  deps = [
+    "//src/arch/aarch64:arch",
+  ]
+
+  assert(defined(pl011_base_address),
+         "\"pl011_base_address\" must be defined for ${target_name}.")
+  defines = [ "PL011_BASE=${pl011_base_address}" ]
+}
diff --git a/src/arch/aarch64/pl011/args.gni b/src/arch/aarch64/pl011/args.gni
new file mode 100644
index 0000000..f5b1e99
--- /dev/null
+++ b/src/arch/aarch64/pl011/args.gni
@@ -0,0 +1,17 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare_args() {
+  pl011_base_address = ""
+}
diff --git a/src/arch/aarch64/pl011/pl011.c b/src/arch/aarch64/pl011/pl011.c
new file mode 100644
index 0000000..f98f36e
--- /dev/null
+++ b/src/arch/aarch64/pl011/pl011.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/io.h"
+#include "hf/mm.h"
+#include "hf/mpool.h"
+#include "hf/plat/console.h"
+
+/* UART Data Register. */
+#define UARTDR IO32_C(PL011_BASE + 0x0)
+
+/* UART Flag Register. */
+#define UARTFR IO32_C(PL011_BASE + 0x018)
+
+/* UART Flag Register bit: transmit fifo is full. */
+#define UARTFR_TXFF (1 << 5)
+
+/* UART Flag Register bit: UART is busy. */
+#define UARTFR_BUSY (1 << 3)
+
+void plat_console_init(void)
+{
+	/* No hardware initialisation required. */
+}
+
+void plat_console_mm_init(struct mm_stage1_locked stage1_locked,
+			  struct mpool *ppool)
+{
+	/* Map page for UART. */
+	mm_identity_map(stage1_locked, pa_init(PL011_BASE),
+			pa_add(pa_init(PL011_BASE), PAGE_SIZE),
+			MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool);
+}
+
+void plat_console_putchar(char c)
+{
+	/* Print a carriage-return as well. */
+	if (c == '\n') {
+		plat_console_putchar('\r');
+	}
+
+	/* Wait until there is room in the tx buffer. */
+	while (io_read32(UARTFR) & UARTFR_TXFF) {
+		/* do nothing */
+	}
+
+	/* Write the character out, force memory access ordering. */
+	memory_ordering_barrier();
+	io_write32(UARTDR, c);
+	memory_ordering_barrier();
+
+	/* Wait until the UART is no longer busy. */
+	while (io_read32_mb(UARTFR) & UARTFR_BUSY) {
+		/* do nothing */
+	}
+}
diff --git a/src/arch/aarch64/psci.h b/src/arch/aarch64/psci.h
new file mode 100644
index 0000000..355c618
--- /dev/null
+++ b/src/arch/aarch64/psci.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "smc.h"
+
+/* clang-format off */
+
+/* The following are PSCI version codes. */
+#define PSCI_VERSION_0_2 0x00000002
+#define PSCI_VERSION_1_0 0x00010000
+#define PSCI_VERSION_1_1 0x00010001
+
+/* The following are function identifiers for PSCI. */
+#define PSCI_VERSION                 0x84000000
+#define PSCI_CPU_SUSPEND             0x84000001
+#define PSCI_CPU_OFF                 0x84000002
+#define PSCI_CPU_ON                  0x84000003
+#define PSCI_AFFINITY_INFO           0x84000004
+#define PSCI_MIGRATE                 0x84000005
+#define PSCI_MIGRATE_INFO_TYPE       0x84000006
+#define PSCI_MIGRATE_INFO_UP_CPU     0x84000007
+#define PSCI_SYSTEM_OFF              0x84000008
+#define PSCI_SYSTEM_RESET            0x84000009
+#define PSCI_FEATURES                0x8400000a
+#define PSCI_CPU_FREEZE              0x8400000b
+#define PSCI_CPU_DEFAULT_SUSPEND     0x8400000c
+#define PSCI_NODE_HW_STATE           0x8400000d
+#define PSCI_SYSTEM_SUSPEND          0x8400000e
+#define PSCI_SET_SYSPEND_MODE        0x8400000f
+#define PSCI_STAT_RESIDENCY          0x84000010
+#define PSCI_STAT_COUNT              0x84000011
+#define PSCI_SYSTEM_RESET2           0x84000012
+#define PSCI_MEM_PROTECT             0x84000013
+#define PSCI_MEM_PROTECT_CHECK_RANGE 0x84000014
+
+/* The following are return codes for PSCI. */
+#define PSCI_RETURN_ON_PENDING         2
+#define PSCI_RETURN_OFF                1
+#define PSCI_RETURN_ON                 0
+#define PSCI_RETURN_SUCCESS            0
+#define PSCI_ERROR_NOT_SUPPORTED       SMCCC_ERROR_UNKNOWN
+#define PSCI_ERROR_INVALID_PARAMETERS  (-2)
+#define PSCI_ERROR_DENIED              (-3)
+#define PSCI_ERROR_ALREADY_ON          (-4)
+#define PSCI_ERROR_ON_PENDING          (-5)
+#define PSCI_ERROR_INTERNAL_FAILURE    (-6)
+#define PSCI_ERROR_NOT_PRESENT         (-7)
+#define PSCI_ERROR_DISABLE             (-8)
+#define PSCI_ERROR_INVALID_ADDRESS     (-9)
+
+/* clang-format on */
diff --git a/src/arch/aarch64/smc.c b/src/arch/aarch64/smc.c
new file mode 100644
index 0000000..e5de0bd
--- /dev/null
+++ b/src/arch/aarch64/smc.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "smc.h"
+
+#include <stdint.h>
+
+#include "vmapi/hf/spci.h"
+
+static struct spci_value smc_internal(uint32_t func, uint64_t arg0,
+				      uint64_t arg1, uint64_t arg2,
+				      uint64_t arg3, uint64_t arg4,
+				      uint64_t arg5, uint32_t caller_id)
+{
+	register uint64_t r0 __asm__("x0") = func;
+	register uint64_t r1 __asm__("x1") = arg0;
+	register uint64_t r2 __asm__("x2") = arg1;
+	register uint64_t r3 __asm__("x3") = arg2;
+	register uint64_t r4 __asm__("x4") = arg3;
+	register uint64_t r5 __asm__("x5") = arg4;
+	register uint64_t r6 __asm__("x6") = arg5;
+	register uint64_t r7 __asm__("x7") = caller_id;
+
+	__asm__ volatile(
+		"smc #0"
+		: /* Output registers, also used as inputs ('+' constraint). */
+		"+r"(r0), "+r"(r1), "+r"(r2), "+r"(r3), "+r"(r4), "+r"(r5),
+		"+r"(r6), "+r"(r7));
+
+	return (struct spci_value){.func = r0,
+				   .arg1 = r1,
+				   .arg2 = r2,
+				   .arg3 = r3,
+				   .arg4 = r4,
+				   .arg5 = r5,
+				   .arg6 = r6,
+				   .arg7 = r7};
+}
+
+struct spci_value smc32(uint32_t func, uint32_t arg0, uint32_t arg1,
+			uint32_t arg2, uint32_t arg3, uint32_t arg4,
+			uint32_t arg5, uint32_t caller_id)
+{
+	return smc_internal(func | SMCCC_32_BIT, arg0, arg1, arg2, arg3, arg4,
+			    arg5, caller_id);
+}
+
+struct spci_value smc64(uint32_t func, uint64_t arg0, uint64_t arg1,
+			uint64_t arg2, uint64_t arg3, uint64_t arg4,
+			uint64_t arg5, uint32_t caller_id)
+{
+	return smc_internal(func | SMCCC_64_BIT, arg0, arg1, arg2, arg3, arg4,
+			    arg5, caller_id);
+}
+
+struct spci_value smc_forward(uint32_t func, uint64_t arg0, uint64_t arg1,
+			      uint64_t arg2, uint64_t arg3, uint64_t arg4,
+			      uint64_t arg5, uint32_t caller_id)
+{
+	return smc_internal(func, arg0, arg1, arg2, arg3, arg4, arg5,
+			    caller_id);
+}
diff --git a/src/arch/aarch64/smc.h b/src/arch/aarch64/smc.h
new file mode 100644
index 0000000..ad8ce5b
--- /dev/null
+++ b/src/arch/aarch64/smc.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include "vmapi/hf/spci.h"
+
+/* clang-format off */
+
+#define SMCCC_CALL_TYPE_MASK  0x80000000
+#define SMCCC_YIELDING_CALL   0x00000000
+#define SMCCC_FAST_CALL       0x80000000
+
+#define SMCCC_CONVENTION_MASK 0x40000000
+#define SMCCC_32_BIT          0x00000000
+#define SMCCC_64_BIT          0x40000000
+
+#define SMCCC_SERVICE_CALL_MASK                0x3f000000
+#define SMCCC_ARM_ARCHITECTURE_CALL            0x00000000
+#define SMCCC_CPU_SERVICE_CALL                 0x01000000
+#define SMCCC_SIP_SERVICE_CALL                 0x02000000
+#define SMCCC_OEM_SERVICE_CALL                 0x03000000
+#define SMCCC_STANDARD_SECURE_SERVICE_CALL     0x04000000
+#define SMCCC_STANDARD_HYPERVISOR_SERVICE_CALL 0x05000000
+#define SMCCC_VENDOR_HYPERVISOR_SERVICE_CALL   0x06000000
+
+#define SMCCC_CALLER_HYPERVISOR   0x0
+/*
+ * TODO: Trusted application call: 0x30000000 - 0x31000000
+ * TODO: Trusted OS call: 0x32000000 - 0x3f000000
+ */
+
+#define SMCCC_ERROR_UNKNOWN  (-1)
+
+/* clang-format on */
+
+struct spci_value smc32(uint32_t func, uint32_t arg0, uint32_t arg1,
+			uint32_t arg2, uint32_t arg3, uint32_t arg4,
+			uint32_t arg5, uint32_t caller_id);
+
+struct spci_value smc64(uint32_t func, uint64_t arg0, uint64_t arg1,
+			uint64_t arg2, uint64_t arg3, uint64_t arg4,
+			uint64_t arg5, uint32_t caller_id);
+
+struct spci_value smc_forward(uint32_t func, uint64_t arg0, uint64_t arg1,
+			      uint64_t arg2, uint64_t arg3, uint64_t arg4,
+			      uint64_t arg5, uint32_t caller_id);
diff --git a/src/arch/aarch64/smc/BUILD.gn b/src/arch/aarch64/smc/BUILD.gn
new file mode 100644
index 0000000..00004f3
--- /dev/null
+++ b/src/arch/aarch64/smc/BUILD.gn
@@ -0,0 +1,19 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source_set("absent") {
+  sources = [
+    "absent.c",
+  ]
+}
diff --git a/src/arch/aarch64/smc/absent.c b/src/arch/aarch64/smc/absent.c
new file mode 100644
index 0000000..6ea9b8d
--- /dev/null
+++ b/src/arch/aarch64/smc/absent.c
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/plat/smc.h"
+
+void plat_smc_post_forward(struct spci_value args, struct spci_value *ret)
+{
+	(void)args;
+	(void)ret;
+}
diff --git a/src/arch/aarch64/stack_protector.c b/src/arch/aarch64/stack_protector.c
new file mode 100644
index 0000000..489bb50
--- /dev/null
+++ b/src/arch/aarch64/stack_protector.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+#include <stdnoreturn.h>
+
+#include "hf/panic.h"
+
+/**
+ * This is the value that is used as the stack canary. It is written to the top
+ * of the stack when entering a function and compared against the stack when
+ * exiting a function. If there is a mismatch, a failure is triggered.
+ *
+ * As the value must be the same at the beginning and end of the function, this
+ * is a global variable and there are multiple CPUs executing concurrently, this
+ * value cannot change after being initialized.
+ *
+ * TODO: initialize to a random value at boot.
+ */
+uint64_t __attribute__((used)) __stack_chk_guard = 0x72afaf72bad0feed;
+
+/**
+ * Called when the stack canary is invalid. The stack can no longer be trusted
+ * so this function must not return.
+ */
+noreturn void __stack_chk_fail(void)
+{
+	panic("stack corruption");
+}
diff --git a/src/arch/aarch64/std.c b/src/arch/aarch64/std.c
new file mode 100644
index 0000000..c7f109d
--- /dev/null
+++ b/src/arch/aarch64/std.c
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/std.h"
+
+void *memset(void *s, int c, size_t n)
+{
+	char *p = (char *)s;
+
+	while (n--) {
+		*p++ = c;
+	}
+
+	return s;
+}
+
+void *memcpy(void *dst, const void *src, size_t n)
+{
+	char *x = dst;
+	const char *y = src;
+
+	while (n--) {
+		*x = *y;
+		x++;
+		y++;
+	}
+
+	return dst;
+}
+
+void *memmove(void *dst, const void *src, size_t n)
+{
+	char *x;
+	const char *y;
+
+	if (dst < src) {
+		/*
+		 * Clang analyzer doesn't like us calling unsafe memory
+		 * functions, so make it ignore this while still knowing that
+		 * the function returns.
+		 */
+#ifdef __clang_analyzer__
+		return dst;
+#else
+		return memcpy(dst, src, n);
+#endif
+	}
+
+	x = (char *)dst + n - 1;
+	y = (const char *)src + n - 1;
+
+	while (n--) {
+		*x = *y;
+		x--;
+		y--;
+	}
+
+	return dst;
+}
+
+int memcmp(const void *a, const void *b, size_t n)
+{
+	const char *x = a;
+	const char *y = b;
+
+	while (n--) {
+		if (*x != *y) {
+			return *x - *y;
+		}
+		x++;
+		y++;
+	}
+
+	return 0;
+}
+
+int strcmp(const char *a, const char *b)
+{
+	const char *x = a;
+	const char *y = b;
+
+	while (*x != 0 && *y != 0) {
+		if (*x != *y) {
+			return *x - *y;
+		}
+		x++;
+		y++;
+	}
+
+	return *x - *y;
+}
diff --git a/src/arch/aarch64/timer.c b/src/arch/aarch64/timer.c
new file mode 100644
index 0000000..3b76e26
--- /dev/null
+++ b/src/arch/aarch64/timer.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/timer.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hf/arch/cpu.h"
+
+#include "hf/addr.h"
+
+#include "msr.h"
+
+#define CNTV_CTL_EL0_ENABLE (1u << 0)
+#define CNTV_CTL_EL0_IMASK (1u << 1)
+#define CNTV_CTL_EL0_ISTATUS (1u << 2)
+
+#define NANOS_PER_UNIT 1000000000
+
+/**
+ * Sets the bit to mask virtual timer interrupts.
+ */
+void arch_timer_mask(struct arch_regs *regs)
+{
+	regs->peripherals.cntv_ctl_el0 |= CNTV_CTL_EL0_IMASK;
+}
+
+/**
+ * Checks whether the virtual timer is enabled and its interrupt not masked.
+ */
+bool arch_timer_enabled(struct arch_regs *regs)
+{
+	uintreg_t cntv_ctl_el0 = regs->peripherals.cntv_ctl_el0;
+
+	return (cntv_ctl_el0 & CNTV_CTL_EL0_ENABLE) &&
+	       !(cntv_ctl_el0 & CNTV_CTL_EL0_IMASK);
+}
+
+/**
+ * Converts a number of timer ticks to the equivalent number of nanoseconds.
+ */
+static uint64_t ticks_to_ns(uint64_t ticks)
+{
+	return (ticks * NANOS_PER_UNIT) / read_msr(cntfrq_el0);
+}
+
+/**
+ * Returns the number of ticks remaining on the virtual timer as stored in
+ * the given `arch_regs`, or 0 if it has already expired. This is undefined if
+ * the timer is not enabled.
+ */
+static uint64_t arch_timer_remaining_ticks(struct arch_regs *regs)
+{
+	/*
+	 * Calculate the value from the saved CompareValue (cntv_cval_el0) and
+	 * the virtual count value.
+	 */
+	uintreg_t cntv_cval_el0 = regs->peripherals.cntv_cval_el0;
+	uintreg_t cntvct_el0 = read_msr(cntvct_el0);
+
+	if (cntv_cval_el0 >= cntvct_el0) {
+		return cntv_cval_el0 - cntvct_el0;
+	}
+
+	return 0;
+}
+
+/**
+ * Returns the number of nanoseconds remaining on the virtual timer as stored in
+ * the given `arch_regs`, or 0 if it has already expired. This is undefined if
+ * the timer is not enabled.
+ */
+uint64_t arch_timer_remaining_ns(struct arch_regs *regs)
+{
+	return ticks_to_ns(arch_timer_remaining_ticks(regs));
+}
+
+/**
+ * Returns whether the timer is ready to fire: i.e. it is enabled, not masked,
+ * and the condition is met.
+ */
+bool arch_timer_pending(struct arch_regs *regs)
+{
+	if (!arch_timer_enabled(regs)) {
+		return false;
+	}
+
+	if (regs->peripherals.cntv_ctl_el0 & CNTV_CTL_EL0_ISTATUS) {
+		return true;
+	}
+
+	if (arch_timer_remaining_ticks(regs) == 0) {
+		/*
+		 * This can happen even if the (stored) ISTATUS bit is not set,
+		 * because time has passed between when the registers were
+		 * stored and now.
+		 */
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * Checks whether the virtual timer is enabled and its interrupt not masked, for
+ * the currently active vCPU.
+ */
+bool arch_timer_enabled_current(void)
+{
+	uintreg_t cntv_ctl_el0 = read_msr(cntv_ctl_el0);
+
+	return (cntv_ctl_el0 & CNTV_CTL_EL0_ENABLE) &&
+	       !(cntv_ctl_el0 & CNTV_CTL_EL0_IMASK);
+}
+
+/**
+ * Disables the virtual timer for the currently active vCPU.
+ */
+void arch_timer_disable_current(void)
+{
+	write_msr(cntv_ctl_el0, 0x0);
+}
+
+/**
+ * Returns the number of ticks remaining on the virtual timer of the currently
+ * active vCPU, or 0 if it has already expired. This is undefined if the timer
+ * is not enabled.
+ */
+static uint64_t arch_timer_remaining_ticks_current(void)
+{
+	uintreg_t cntv_cval_el0 = read_msr(cntv_cval_el0);
+	uintreg_t cntvct_el0 = read_msr(cntvct_el0);
+
+	if (cntv_cval_el0 >= cntvct_el0) {
+		return cntv_cval_el0 - cntvct_el0;
+	}
+
+	return 0;
+}
+
+/**
+ * Returns the number of nanoseconds remaining on the virtual timer of the
+ * currently active vCPU, or 0 if it has already expired. This is undefined if
+ * the timer is not enabled.
+ */
+uint64_t arch_timer_remaining_ns_current(void)
+{
+	return ticks_to_ns(arch_timer_remaining_ticks_current());
+}
diff --git a/src/arch/fake/BUILD.gn b/src/arch/fake/BUILD.gn
new file mode 100644
index 0000000..a0c7219
--- /dev/null
+++ b/src/arch/fake/BUILD.gn
@@ -0,0 +1,37 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source_set("fake") {
+  sources = [
+    "mm.c",
+    "timer.c",
+  ]
+}
+
+# Empty implementation of platform boot flow.
+# Fake arch targets should not depend on the boot flow functions. Will fail to
+# compile if they do.
+source_set("boot_flow") {
+}
+
+# Fake implementation of putchar logs to the console.
+source_set("console") {
+  sources = [
+    "console.c",
+  ]
+}
+
+# Empty, as the functions are provided by libc already.
+source_set("std") {
+}
diff --git a/src/arch/fake/console.c b/src/arch/fake/console.c
new file mode 100644
index 0000000..036cced
--- /dev/null
+++ b/src/arch/fake/console.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/plat/console.h"
+
+#include <stdio.h>
+
+#include "hf/mm.h"
+#include "hf/mpool.h"
+
+void plat_console_init(void)
+{
+}
+
+void plat_console_mm_init(struct mm_stage1_locked stage1_locked,
+			  struct mpool *ppool)
+{
+}
+
+void plat_console_putchar(char c)
+{
+	putchar(c);
+}
diff --git a/src/arch/fake/hftest/BUILD.gn b/src/arch/fake/hftest/BUILD.gn
new file mode 100644
index 0000000..c69041f
--- /dev/null
+++ b/src/arch/fake/hftest/BUILD.gn
@@ -0,0 +1,23 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These components are only used by tests running until Linux VMs.
+
+# Shutdown the system.
+source_set("power_mgmt") {
+  testonly = true
+  sources = [
+    "power_mgmt.c",
+  ]
+}
diff --git a/src/arch/fake/hftest/power_mgmt.c b/src/arch/fake/hftest/power_mgmt.c
new file mode 100644
index 0000000..a171036
--- /dev/null
+++ b/src/arch/fake/hftest/power_mgmt.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include <sys/reboot.h>
+
+noreturn void arch_power_off(void)
+{
+	reboot(RB_POWER_OFF);
+	for (;;) {
+		/* This should never be reached. */
+	}
+}
diff --git a/src/arch/fake/hypervisor/BUILD.gn b/src/arch/fake/hypervisor/BUILD.gn
new file mode 100644
index 0000000..b6435ed
--- /dev/null
+++ b/src/arch/fake/hypervisor/BUILD.gn
@@ -0,0 +1,22 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source_set("hypervisor") {
+  sources = [
+    "cpu.c",
+  ]
+  deps = [
+    "//src/arch/fake",
+  ]
+}
diff --git a/src/arch/fake/hypervisor/cpu.c b/src/arch/fake/hypervisor/cpu.c
new file mode 100644
index 0000000..3fc09f9
--- /dev/null
+++ b/src/arch/fake/hypervisor/cpu.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/cpu.h"
+
+#include "hf/cpu.h"
+#include "hf/spci.h"
+
+void arch_irq_disable(void)
+{
+	/* TODO */
+}
+
+void arch_irq_enable(void)
+{
+	/* TODO */
+}
+
+void arch_regs_reset(struct vcpu *vcpu)
+{
+	/* TODO */
+	(void)vcpu;
+}
+
+void arch_regs_set_pc_arg(struct arch_regs *r, ipaddr_t pc, uintreg_t arg)
+{
+	(void)pc;
+	r->arg[0] = arg;
+}
+
+void arch_regs_set_retval(struct arch_regs *r, struct spci_value v)
+{
+	r->arg[0] = v.func;
+	r->arg[1] = v.arg1;
+	r->arg[2] = v.arg2;
+	r->arg[3] = v.arg3;
+	r->arg[4] = v.arg4;
+	r->arg[5] = v.arg5;
+	r->arg[6] = v.arg6;
+	r->arg[7] = v.arg7;
+}
diff --git a/src/arch/fake/inc/hf/arch/barriers.h b/src/arch/fake/inc/hf/arch/barriers.h
new file mode 100644
index 0000000..46f35a9
--- /dev/null
+++ b/src/arch/fake/inc/hf/arch/barriers.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdatomic.h>
+
+/** Platform-agnostic API */
+
+/**
+ * Ensures all explicit memory accesses before this point are completed before
+ * any later memory accesses are performed.
+ */
+#define memory_ordering_barrier() atomic_thread_fence(memory_order_seq_cst)
+
+/**
+ * Ensures all explicit memory access and management instructions have completed
+ * before continuing.
+ *
+ * FIXME: this is just a memory barrier but, without MMIO or registers to modify
+ * operation in the fake architecture, this is likely enough. If there's a way
+ * to have a true synchronization then we should update it.
+ */
+#define data_sync_barrier() atomic_thread_fence(memory_order_seq_cst)
diff --git a/src/arch/fake/inc/hf/arch/spinlock.h b/src/arch/fake/inc/hf/arch/spinlock.h
new file mode 100644
index 0000000..db3b45c
--- /dev/null
+++ b/src/arch/fake/inc/hf/arch/spinlock.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/**
+ * Generic implementation of a spinlock using C11 atomics.
+ * Does not work very well under contention.
+ */
+
+#include <stdatomic.h>
+
+struct spinlock {
+	atomic_flag v;
+};
+
+#define SPINLOCK_INIT ((struct spinlock){.v = ATOMIC_FLAG_INIT})
+
+static inline void sl_lock(struct spinlock *l)
+{
+	while (atomic_flag_test_and_set_explicit(&l->v, memory_order_acquire)) {
+		/* do nothing */
+	}
+}
+
+static inline void sl_unlock(struct spinlock *l)
+{
+	atomic_flag_clear_explicit(&l->v, memory_order_release);
+}
diff --git a/src/arch/fake/inc/hf/arch/types.h b/src/arch/fake/inc/hf/arch/types.h
new file mode 100644
index 0000000..4cb5be4
--- /dev/null
+++ b/src/arch/fake/inc/hf/arch/types.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#define PAGE_BITS 12
+#define PAGE_LEVEL_BITS 9
+#define STACK_ALIGN 64
+
+/** The type of a page table entry (PTE). */
+typedef uint64_t pte_t;
+
+/** Integer type large enough to hold a physical address. */
+typedef uintptr_t uintpaddr_t;
+
+/** Integer type large enough to hold a virtual address. */
+typedef uintptr_t uintvaddr_t;
+
+/** The integer corresponding to the native register size. */
+typedef uint64_t uintreg_t;
+
+/** The ID of a physical or virtual CPU. */
+typedef uint32_t cpu_id_t;
+
+/** Arch-specifc information about a VM. */
+struct arch_vm {
+	/* This field is only here because empty structs aren't allowed. */
+	void *dummy;
+};
+
+/** Type to represent the register state of a VM.  */
+struct arch_regs {
+	uintreg_t arg[8];
+	cpu_id_t vcpu_id;
+	bool virtual_interrupt;
+};
diff --git a/src/arch/fake/inc/hf/arch/vm/power_mgmt.h b/src/arch/fake/inc/hf/arch/vm/power_mgmt.h
new file mode 100644
index 0000000..607705e
--- /dev/null
+++ b/src/arch/fake/inc/hf/arch/vm/power_mgmt.h
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdnoreturn.h>
+
+noreturn void arch_power_off(void);
diff --git a/src/arch/fake/mm.c b/src/arch/fake/mm.c
new file mode 100644
index 0000000..d0066fb
--- /dev/null
+++ b/src/arch/fake/mm.c
@@ -0,0 +1,170 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/mm.h"
+
+#include "hf/mm.h"
+
+/*
+ * The fake architecture uses the mode flags to represent the attributes applied
+ * to memory. The flags are shifted to avoid equality of modes and attributes.
+ */
+#define PTE_ATTR_MODE_SHIFT 48
+#define PTE_ATTR_MODE_MASK                                              \
+	((uint64_t)(MM_MODE_R | MM_MODE_W | MM_MODE_X | MM_MODE_D |     \
+		    MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED) \
+	 << PTE_ATTR_MODE_SHIFT)
+
+/* The bit to distinguish a table from a block is the highest of the page bits.
+ */
+#define PTE_TABLE (UINT64_C(1) << (PAGE_BITS - 1))
+
+/* Mask for the address part of an entry. */
+#define PTE_ADDR_MASK (~(PTE_ATTR_MODE_MASK | (UINT64_C(1) << PAGE_BITS) - 1))
+
+/* Offset the bits of each level so they can't be misued. */
+#define PTE_LEVEL_SHIFT(lvl) ((lvl)*2)
+
+pte_t arch_mm_absent_pte(uint8_t level)
+{
+	return ((uint64_t)(MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED)
+		<< PTE_ATTR_MODE_SHIFT) >>
+	       PTE_LEVEL_SHIFT(level);
+}
+
+pte_t arch_mm_table_pte(uint8_t level, paddr_t pa)
+{
+	return (pa_addr(pa) | PTE_TABLE) >> PTE_LEVEL_SHIFT(level);
+}
+
+pte_t arch_mm_block_pte(uint8_t level, paddr_t pa, uint64_t attrs)
+{
+	return (pa_addr(pa) | attrs) >> PTE_LEVEL_SHIFT(level);
+}
+
+bool arch_mm_is_block_allowed(uint8_t level)
+{
+	(void)level;
+	return true;
+}
+
+bool arch_mm_pte_is_present(pte_t pte, uint8_t level)
+{
+	return arch_mm_pte_is_valid(pte, level) ||
+	       !(((pte << PTE_LEVEL_SHIFT(level)) >> PTE_ATTR_MODE_SHIFT) &
+		 MM_MODE_UNOWNED);
+}
+
+bool arch_mm_pte_is_valid(pte_t pte, uint8_t level)
+{
+	return !(((pte << PTE_LEVEL_SHIFT(level)) >> PTE_ATTR_MODE_SHIFT) &
+		 MM_MODE_INVALID);
+}
+
+bool arch_mm_pte_is_block(pte_t pte, uint8_t level)
+{
+	return arch_mm_pte_is_present(pte, level) &&
+	       !arch_mm_pte_is_table(pte, level);
+}
+
+bool arch_mm_pte_is_table(pte_t pte, uint8_t level)
+{
+	return (pte << PTE_LEVEL_SHIFT(level)) & PTE_TABLE;
+}
+
+paddr_t arch_mm_clear_pa(paddr_t pa)
+{
+	return pa_init(pa_addr(pa) & PTE_ADDR_MASK);
+}
+
+paddr_t arch_mm_block_from_pte(pte_t pte, uint8_t level)
+{
+	return pa_init((pte << PTE_LEVEL_SHIFT(level)) & PTE_ADDR_MASK);
+}
+
+paddr_t arch_mm_table_from_pte(pte_t pte, uint8_t level)
+{
+	return pa_init((pte << PTE_LEVEL_SHIFT(level)) & PTE_ADDR_MASK);
+}
+
+uint64_t arch_mm_pte_attrs(pte_t pte, uint8_t level)
+{
+	return (pte << PTE_LEVEL_SHIFT(level)) & PTE_ATTR_MODE_MASK;
+}
+
+uint64_t arch_mm_combine_table_entry_attrs(uint64_t table_attrs,
+					   uint64_t block_attrs)
+{
+	return table_attrs | block_attrs;
+}
+
+void arch_mm_invalidate_stage1_range(vaddr_t va_begin, vaddr_t va_end)
+{
+	/* There's no modelling of the stage-1 TLB. */
+}
+
+void arch_mm_invalidate_stage2_range(ipaddr_t va_begin, ipaddr_t va_end)
+{
+	/* There's no modelling of the stage-2 TLB. */
+}
+
+void arch_mm_flush_dcache(void *base, size_t size)
+{
+	/* There's no modelling of the cache. */
+}
+
+uint8_t arch_mm_stage1_max_level(void)
+{
+	return 2;
+}
+
+uint8_t arch_mm_stage2_max_level(void)
+{
+	return 2;
+}
+
+uint8_t arch_mm_stage1_root_table_count(void)
+{
+	return 1;
+}
+
+uint8_t arch_mm_stage2_root_table_count(void)
+{
+	/* Stage-2 has many concatenated page tables. */
+	return 4;
+}
+
+uint64_t arch_mm_mode_to_stage1_attrs(uint32_t mode)
+{
+	return ((uint64_t)mode << PTE_ATTR_MODE_SHIFT) & PTE_ATTR_MODE_MASK;
+}
+
+uint64_t arch_mm_mode_to_stage2_attrs(uint32_t mode)
+{
+	return ((uint64_t)mode << PTE_ATTR_MODE_SHIFT) & PTE_ATTR_MODE_MASK;
+}
+
+uint32_t arch_mm_stage2_attrs_to_mode(uint64_t attrs)
+{
+	return attrs >> PTE_ATTR_MODE_SHIFT;
+}
+
+bool arch_mm_init(paddr_t table)
+{
+	/* No initialization required. */
+	(void)table;
+	return true;
+}
diff --git a/src/arch/fake/timer.c b/src/arch/fake/timer.c
new file mode 100644
index 0000000..1c0d449
--- /dev/null
+++ b/src/arch/fake/timer.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/timer.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "hf/arch/types.h"
+
+bool arch_timer_pending(struct arch_regs *regs)
+{
+	/* TODO */
+	(void)regs;
+	return false;
+}
+
+void arch_timer_mask(struct arch_regs *regs)
+{
+	/* TODO */
+	(void)regs;
+}
+
+bool arch_timer_enabled(struct arch_regs *regs)
+{
+	/* TODO */
+	(void)regs;
+	return false;
+}
+
+uint64_t arch_timer_remaining_ns(struct arch_regs *regs)
+{
+	/* TODO */
+	(void)regs;
+	return 0;
+}
+
+bool arch_timer_enabled_current(void)
+{
+	/* TODO */
+	return false;
+}
+
+void arch_timer_disable_current(void)
+{
+	/* TODO */
+}
+
+uint64_t arch_timer_remaining_ns_current(void)
+{
+	/* TODO */
+	return 0;
+}
diff --git a/src/boot_flow/BUILD.gn b/src/boot_flow/BUILD.gn
new file mode 100644
index 0000000..9de339d
--- /dev/null
+++ b/src/boot_flow/BUILD.gn
@@ -0,0 +1,41 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/platform.gni")
+
+source_set("common") {
+  sources = [
+    "common.c",
+  ]
+}
+
+source_set("android") {
+  sources = [
+    "android.c",
+  ]
+  deps = [
+    ":common",
+    "//src/arch/${plat_arch}/boot_flow:android",
+  ]
+}
+
+source_set("linux") {
+  sources = [
+    "linux.c",
+  ]
+  deps = [
+    ":common",
+    "//src/arch/${plat_arch}/boot_flow:linux",
+  ]
+}
diff --git a/src/boot_flow/android.c b/src/boot_flow/android.c
new file mode 100644
index 0000000..0221674
--- /dev/null
+++ b/src/boot_flow/android.c
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/layout.h"
+#include "hf/plat/boot_flow.h"
+
+/**
+ * FDT was compiled into Hafnium. Return physical address of the `.plat.fdt`
+ * section of Hafnium image.
+ */
+paddr_t plat_boot_flow_get_fdt_addr(void)
+{
+	return layout_fdt_begin();
+}
+
+/**
+ * Android boot flow does not use kernel arguments. Pass zero.
+ */
+uintreg_t plat_boot_flow_get_kernel_arg(void)
+{
+	return 0;
+}
+
+/**
+ * Initrd was compiled into Hafnium. Return range of the '.plat.initrd' section.
+ */
+bool plat_boot_flow_get_initrd_range(const struct fdt_node *fdt_root,
+				     paddr_t *begin, paddr_t *end)
+{
+	(void)fdt_root;
+
+	*begin = layout_initrd_begin();
+	*end = layout_initrd_end();
+	return true;
+}
+
+/**
+ * Android boot flow does not change based on the updates.
+ */
+bool plat_boot_flow_update(struct mm_stage1_locked stage1_locked,
+			   const struct manifest *manifest,
+			   struct boot_params_update *p, struct memiter *cpio,
+			   struct mpool *ppool)
+{
+	(void)stage1_locked;
+	(void)manifest;
+	(void)p;
+	(void)cpio;
+	(void)ppool;
+
+	return true;
+}
diff --git a/src/boot_flow/common.c b/src/boot_flow/common.c
new file mode 100644
index 0000000..834f746
--- /dev/null
+++ b/src/boot_flow/common.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/boot_flow.h"
+#include "hf/dlog.h"
+#include "hf/fdt_handler.h"
+#include "hf/plat/boot_flow.h"
+
+/**
+ * Extract the boot parameters from the FDT and the boot-flow driver.
+ */
+static bool boot_params_init(struct boot_params *p,
+			     const struct fdt_node *fdt_root)
+{
+	p->mem_ranges_count = 0;
+	p->kernel_arg = plat_boot_flow_get_kernel_arg();
+
+	return plat_boot_flow_get_initrd_range(fdt_root, &p->initrd_begin,
+					       &p->initrd_end) &&
+	       fdt_find_cpus(fdt_root, p->cpu_ids, &p->cpu_count) &&
+	       fdt_find_memory_ranges(fdt_root, p);
+}
+
+/**
+ * Parses information from FDT needed to initialize Hafnium.
+ * FDT is mapped at the beginning and unmapped before exiting the function.
+ */
+bool boot_flow_init(const struct fdt_node *fdt_root, struct manifest *manifest,
+		    struct boot_params *boot_params)
+{
+	enum manifest_return_code manifest_ret;
+
+	/* Get the memory map from the FDT. */
+
+	manifest_ret = manifest_init(manifest, fdt_root);
+	if (manifest_ret != MANIFEST_SUCCESS) {
+		dlog("Could not parse manifest: %s.\n",
+		     manifest_strerror(manifest_ret));
+		return false;
+	}
+
+	if (!boot_params_init(boot_params, fdt_root)) {
+		dlog("Could not parse boot params.\n");
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * Takes action on any updates that were generated.
+ */
+bool boot_flow_update(struct mm_stage1_locked stage1_locked,
+		      const struct manifest *manifest,
+		      struct boot_params_update *p, struct memiter *cpio,
+		      struct mpool *ppool)
+{
+	return plat_boot_flow_update(stage1_locked, manifest, p, cpio, ppool);
+}
diff --git a/src/boot_flow/linux.c b/src/boot_flow/linux.c
new file mode 100644
index 0000000..e1e257c
--- /dev/null
+++ b/src/boot_flow/linux.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/check.h"
+#include "hf/cpio.h"
+#include "hf/dlog.h"
+#include "hf/fdt_handler.h"
+#include "hf/plat/boot_flow.h"
+#include "hf/std.h"
+
+/* Set by arch-specific boot-time hook. */
+uintreg_t plat_boot_flow_fdt_addr;
+
+/**
+ * Returns the physical address of board FDT. This was passed to Hafnium in the
+ * first kernel arg by the boot loader.
+ */
+paddr_t plat_boot_flow_get_fdt_addr(void)
+{
+	return pa_init((uintpaddr_t)plat_boot_flow_fdt_addr);
+}
+
+/**
+ * When handing over to the primary, give it the same FDT address that was given
+ * to Hafnium. The FDT may have been modified during Hafnium init.
+ */
+uintreg_t plat_boot_flow_get_kernel_arg(void)
+{
+	return plat_boot_flow_fdt_addr;
+}
+
+/**
+ * Load initrd range from the board FDT.
+ */
+bool plat_boot_flow_get_initrd_range(const struct fdt_node *fdt_root,
+				     paddr_t *begin, paddr_t *end)
+{
+	return fdt_find_initrd(fdt_root, begin, end);
+}
+
+bool plat_boot_flow_update(struct mm_stage1_locked stage1_locked,
+			   const struct manifest *manifest,
+			   struct boot_params_update *update,
+			   struct memiter *cpio, struct mpool *ppool)
+{
+	struct memiter primary_initrd;
+	const struct string *filename =
+		&manifest->vm[HF_PRIMARY_VM_INDEX].primary.ramdisk_filename;
+
+	if (string_is_empty(filename)) {
+		memiter_init(&primary_initrd, NULL, 0);
+	} else if (!cpio_get_file(cpio, filename, &primary_initrd)) {
+		dlog("Unable to find primary initrd \"%s\".\n",
+		     string_data(filename));
+		return false;
+	}
+
+	update->initrd_begin = pa_from_va(va_from_ptr(primary_initrd.next));
+	update->initrd_end = pa_from_va(va_from_ptr(primary_initrd.limit));
+
+	return fdt_patch(stage1_locked, plat_boot_flow_get_fdt_addr(), update,
+			 ppool);
+}
diff --git a/src/cpio.c b/src/cpio.c
new file mode 100644
index 0000000..58626ec
--- /dev/null
+++ b/src/cpio.c
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/cpio.h"
+
+#include <stdint.h>
+
+#include "hf/std.h"
+
+#pragma pack(push, 1)
+struct cpio_header {
+	uint16_t magic;
+	uint16_t dev;
+	uint16_t ino;
+	uint16_t mode;
+	uint16_t uid;
+	uint16_t gid;
+	uint16_t nlink;
+	uint16_t rdev;
+	uint16_t mtime[2];
+	uint16_t namesize;
+	uint16_t filesize[2];
+};
+#pragma pack(pop)
+
+/**
+ * Retrieves the next file stored in the cpio archive stored in the cpio, and
+ * advances the iterator such that another call to this function would return
+ * the following file.
+ */
+static bool cpio_next(struct memiter *iter, const char **name,
+		      const void **contents, size_t *size)
+{
+	size_t len;
+	struct memiter lit = *iter;
+	const struct cpio_header *h = (const struct cpio_header *)lit.next;
+
+	if (!memiter_advance(&lit, sizeof(struct cpio_header))) {
+		return false;
+	}
+
+	*name = lit.next;
+
+	/* TODO: Check magic. */
+
+	len = (h->namesize + 1) & ~1;
+	if (!memiter_advance(&lit, len)) {
+		return false;
+	}
+
+	*contents = lit.next;
+
+	len = (size_t)h->filesize[0] << 16 | h->filesize[1];
+	if (!memiter_advance(&lit, (len + 1) & ~1)) {
+		return false;
+	}
+
+	/* TODO: Check that string is null-terminated. */
+
+	/* Stop enumerating files when we hit the end marker. */
+	if (!strcmp(*name, "TRAILER!!!")) {
+		return false;
+	}
+
+	*size = len;
+	*iter = lit;
+
+	return true;
+}
+
+/**
+ * Looks for a file in the given cpio archive. The file, if found, is returned
+ * in the "it" argument.
+ */
+bool cpio_get_file(const struct memiter *cpio, const struct string *name,
+		   struct memiter *it)
+{
+	const char *fname;
+	const void *fcontents;
+	size_t fsize;
+	struct memiter iter = *cpio;
+
+	while (cpio_next(&iter, &fname, &fcontents, &fsize)) {
+		if (!strcmp(fname, string_data(name))) {
+			memiter_init(it, fcontents, fsize);
+			return true;
+		}
+	}
+
+	return false;
+}
diff --git a/src/cpu.c b/src/cpu.c
new file mode 100644
index 0000000..f8beed6
--- /dev/null
+++ b/src/cpu.c
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/cpu.h"
+
+#include <stdalign.h>
+
+#include "hf/api.h"
+#include "hf/check.h"
+#include "hf/dlog.h"
+
+#include "vmapi/hf/call.h"
+
+#define STACK_SIZE PAGE_SIZE
+
+/**
+ * The stacks to be used by the CPUs.
+ *
+ * Align to page boundaries to ensure that cache lines are not shared between a
+ * CPU's stack and data that can be accessed from other CPUs. If this did
+ * happen, there may be coherency problems when the stack is being used before
+ * caching is enabled.
+ */
+alignas(PAGE_SIZE) static char callstacks[MAX_CPUS][STACK_SIZE];
+
+/* NOLINTNEXTLINE(misc-redundant-expression) */
+static_assert((STACK_SIZE % PAGE_SIZE) == 0, "Keep each stack page aligned.");
+static_assert((PAGE_SIZE % STACK_ALIGN) == 0,
+	      "Page alignment is too weak for the stack.");
+
+/**
+ * Internal buffer used to store SPCI messages from a VM Tx. Its usage prevents
+ * TOCTOU issues while Hafnium performs actions on information that would
+ * otherwise be re-writable by the VM.
+ *
+ * Each buffer is owned by a single CPU. The buffer can only be used for
+ * spci_msg_send. The information stored in the buffer is only valid during the
+ * spci_msg_send request is performed.
+ */
+alignas(PAGE_SIZE) static uint8_t cpu_message_buffer[MAX_CPUS][PAGE_SIZE];
+
+uint8_t *cpu_get_buffer(cpu_id_t cpu_id)
+{
+	CHECK(cpu_id < MAX_CPUS);
+
+	return cpu_message_buffer[cpu_id];
+}
+
+uint32_t cpu_get_buffer_size(cpu_id_t cpu_id)
+{
+	CHECK(cpu_id < MAX_CPUS);
+
+	return sizeof(cpu_message_buffer[cpu_id]);
+}
+
+/* State of all supported CPUs. The stack of the first one is initialized. */
+struct cpu cpus[MAX_CPUS] = {
+	{
+		.is_on = 1,
+		.stack_bottom = &callstacks[0][STACK_SIZE],
+	},
+};
+
+static uint32_t cpu_count = 1;
+
+void cpu_module_init(const cpu_id_t *cpu_ids, size_t count)
+{
+	uint32_t i;
+	uint32_t j;
+	cpu_id_t boot_cpu_id = cpus[0].id;
+	bool found_boot_cpu = false;
+
+	cpu_count = count;
+
+	/*
+	 * Initialize CPUs with the IDs from the configuration passed in. The
+	 * CPUs after the boot CPU are initialized in reverse order. The boot
+	 * CPU is initialized when it is found or in place of the last CPU if it
+	 * is not found.
+	 */
+	j = cpu_count;
+	for (i = 0; i < cpu_count; ++i) {
+		struct cpu *c;
+		cpu_id_t id = cpu_ids[i];
+
+		if (found_boot_cpu || id != boot_cpu_id) {
+			--j;
+			c = &cpus[j];
+			c->stack_bottom = &callstacks[j][STACK_SIZE];
+		} else {
+			found_boot_cpu = true;
+			c = &cpus[0];
+			CHECK(c->stack_bottom == &callstacks[0][STACK_SIZE]);
+		}
+
+		sl_init(&c->lock);
+		c->id = id;
+	}
+
+	if (!found_boot_cpu) {
+		/* Boot CPU was initialized but with wrong ID. */
+		dlog("Boot CPU's ID not found in config.\n");
+		cpus[0].id = boot_cpu_id;
+	}
+}
+
+size_t cpu_index(struct cpu *c)
+{
+	return c - cpus;
+}
+
+/**
+ * Turns CPU on and returns the previous state.
+ */
+bool cpu_on(struct cpu *c, ipaddr_t entry, uintreg_t arg)
+{
+	bool prev;
+
+	sl_lock(&c->lock);
+	prev = c->is_on;
+	c->is_on = true;
+	sl_unlock(&c->lock);
+
+	if (!prev) {
+		struct vm *vm = vm_find(HF_PRIMARY_VM_ID);
+		struct vcpu *vcpu = vm_get_vcpu(vm, cpu_index(c));
+		struct vcpu_locked vcpu_locked;
+
+		vcpu_locked = vcpu_lock(vcpu);
+		vcpu_on(vcpu_locked, entry, arg);
+		vcpu_unlock(&vcpu_locked);
+	}
+
+	return prev;
+}
+
+/**
+ * Prepares the CPU for turning itself off.
+ */
+void cpu_off(struct cpu *c)
+{
+	sl_lock(&c->lock);
+	c->is_on = false;
+	sl_unlock(&c->lock);
+}
+
+/**
+ * Searches for a CPU based on its ID.
+ */
+struct cpu *cpu_find(cpu_id_t id)
+{
+	size_t i;
+
+	for (i = 0; i < cpu_count; i++) {
+		if (cpus[i].id == id) {
+			return &cpus[i];
+		}
+	}
+
+	return NULL;
+}
diff --git a/src/dlog.c b/src/dlog.c
new file mode 100644
index 0000000..1bda568
--- /dev/null
+++ b/src/dlog.c
@@ -0,0 +1,372 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/dlog.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "hf/plat/console.h"
+#include "hf/spci.h"
+#include "hf/spinlock.h"
+#include "hf/std.h"
+
+/* Keep macro alignment */
+/* clang-format off */
+
+#define FLAG_SPACE 0x01
+#define FLAG_ZERO  0x02
+#define FLAG_MINUS 0x04
+#define FLAG_PLUS  0x08
+#define FLAG_ALT   0x10
+#define FLAG_UPPER 0x20
+#define FLAG_NEG   0x40
+
+#define DLOG_MAX_STRING_LENGTH 64
+
+/* clang-format on */
+
+static bool dlog_lock_enabled = false;
+static struct spinlock sl = SPINLOCK_INIT;
+
+/*
+ * These global variables for the log buffer are not static because a test needs
+ * to access them directly.
+ */
+size_t dlog_buffer_offset;
+char dlog_buffer[DLOG_BUFFER_SIZE];
+
+/**
+ * Takes the lock, if it is enabled.
+ */
+static void lock(void)
+{
+	if (dlog_lock_enabled) {
+		sl_lock(&sl);
+	}
+}
+
+/**
+ * Releases the lock, if it is enabled.
+ */
+static void unlock(void)
+{
+	if (dlog_lock_enabled) {
+		sl_unlock(&sl);
+	}
+}
+
+/**
+ * Enables the lock protecting the serial device.
+ */
+void dlog_enable_lock(void)
+{
+	dlog_lock_enabled = true;
+}
+
+static void dlog_putchar(char c)
+{
+	dlog_buffer[dlog_buffer_offset] = c;
+	dlog_buffer_offset = (dlog_buffer_offset + 1) % DLOG_BUFFER_SIZE;
+	plat_console_putchar(c);
+}
+
+/**
+ * Prints a raw string to the debug log and returns its length.
+ */
+static size_t print_raw_string(const char *str)
+{
+	const char *c = str;
+
+	while (*c != '\0') {
+		dlog_putchar(*c++);
+	}
+
+	return c - str;
+}
+
+/**
+ * Prints a formatted string to the debug log. The format includes a minimum
+ * width, the fill character, and flags (whether to align to left or right).
+ *
+ * str is the full string, while suffix is a pointer within str that indicates
+ * where the suffix begins. This is used when printing right-aligned numbers
+ * with a zero fill; for example, -10 with width 4 should be padded to -010,
+ * so suffix would point to index one of the "-10" string .
+ */
+static void print_string(const char *str, const char *suffix, size_t width,
+			 int flags, char fill)
+{
+	size_t len = suffix - str;
+
+	/* Print the string up to the beginning of the suffix. */
+	while (str != suffix) {
+		dlog_putchar(*str++);
+	}
+
+	if (flags & FLAG_MINUS) {
+		/* Left-aligned. Print suffix, then print padding if needed. */
+		len += print_raw_string(suffix);
+		while (len < width) {
+			dlog_putchar(' ');
+			len++;
+		}
+		return;
+	}
+
+	/* Fill until we reach the desired length. */
+	len += strnlen_s(suffix, DLOG_MAX_STRING_LENGTH);
+	while (len < width) {
+		dlog_putchar(fill);
+		len++;
+	}
+
+	/* Now print the rest of the string. */
+	print_raw_string(suffix);
+}
+
+/**
+ * Prints a number to the debug log. The caller specifies the base, its minimum
+ * width and printf-style flags.
+ */
+static void print_num(size_t v, size_t base, size_t width, int flags)
+{
+	static const char *digits_lower = "0123456789abcdefx";
+	static const char *digits_upper = "0123456789ABCDEFX";
+	const char *d = (flags & FLAG_UPPER) ? digits_upper : digits_lower;
+	char buf[DLOG_MAX_STRING_LENGTH];
+	char *ptr = &buf[sizeof(buf) - 1];
+	char *num;
+	*ptr = '\0';
+	do {
+		--ptr;
+		*ptr = d[v % base];
+		v /= base;
+	} while (v);
+
+	/* Num stores where the actual number begins. */
+	num = ptr;
+
+	/* Add prefix if requested. */
+	if (flags & FLAG_ALT) {
+		switch (base) {
+		case 16:
+			ptr -= 2;
+			ptr[0] = '0';
+			ptr[1] = d[16];
+			break;
+
+		case 8:
+			ptr--;
+			*ptr = '0';
+			break;
+		}
+	}
+
+	/* Add sign if requested. */
+	if (flags & FLAG_NEG) {
+		*--ptr = '-';
+	} else if (flags & FLAG_PLUS) {
+		*--ptr = '+';
+	} else if (flags & FLAG_SPACE) {
+		*--ptr = ' ';
+	}
+	if (flags & FLAG_ZERO) {
+		print_string(ptr, num, width, flags, '0');
+	} else {
+		print_string(ptr, ptr, width, flags, ' ');
+	}
+}
+
+/**
+ * Parses the optional flags field of a printf-style format. It returns the spot
+ * on the string where a non-flag character was found.
+ */
+static const char *parse_flags(const char *p, int *flags)
+{
+	for (;;) {
+		switch (*p) {
+		case ' ':
+			*flags |= FLAG_SPACE;
+			break;
+
+		case '0':
+			*flags |= FLAG_ZERO;
+			break;
+
+		case '-':
+			*flags |= FLAG_MINUS;
+			break;
+
+		case '+':
+			*flags |= FLAG_PLUS;
+
+		case '#':
+			*flags |= FLAG_ALT;
+			break;
+
+		default:
+			return p;
+		}
+		p++;
+	}
+}
+
+/**
+ * Send the contents of the given VM's log buffer to the log, preceded by the VM
+ * ID and followed by a newline.
+ */
+void dlog_flush_vm_buffer(spci_vm_id_t id, char buffer[], size_t length)
+{
+	lock();
+
+	print_raw_string("VM ");
+	print_num(id, 10, 0, 0);
+	print_raw_string(": ");
+
+	for (size_t i = 0; i < length; ++i) {
+		dlog_putchar(buffer[i]);
+		buffer[i] = '\0';
+	}
+	dlog_putchar('\n');
+
+	unlock();
+}
+
+/**
+ * Same as "dlog", except that arguments are passed as a va_list
+ */
+void vdlog(const char *fmt, va_list args)
+{
+	const char *p;
+	size_t w;
+	int flags;
+	char buf[2];
+
+	lock();
+
+	for (p = fmt; *p; p++) {
+		switch (*p) {
+		default:
+			dlog_putchar(*p);
+			break;
+
+		case '%':
+			/* Read optional flags. */
+			flags = 0;
+			p = parse_flags(p + 1, &flags) - 1;
+
+			/* Read the minimum width, if one is specified. */
+			w = 0;
+			while (p[1] >= '0' && p[1] <= '9') {
+				w = (w * 10) + (p[1] - '0');
+				p++;
+			}
+
+			/* Read minimum width from arguments. */
+			if (w == 0 && p[1] == '*') {
+				int v = va_arg(args, int);
+
+				if (v >= 0) {
+					w = v;
+				} else {
+					w = -v;
+					flags |= FLAG_MINUS;
+				}
+				p++;
+			}
+
+			/* Handle the format specifier. */
+			switch (p[1]) {
+			case 's': {
+				char *str = va_arg(args, char *);
+
+				print_string(str, str, w, flags, ' ');
+				p++;
+			} break;
+
+			case 'd':
+			case 'i': {
+				int v = va_arg(args, int);
+
+				if (v < 0) {
+					flags |= FLAG_NEG;
+					v = -v;
+				}
+
+				print_num((size_t)v, 10, w, flags);
+				p++;
+			} break;
+
+			case 'X':
+				flags |= FLAG_UPPER;
+				print_num(va_arg(args, size_t), 16, w, flags);
+				p++;
+				break;
+
+			case 'p':
+				print_num(va_arg(args, size_t), 16,
+					  sizeof(size_t) * 2, FLAG_ZERO);
+				p++;
+				break;
+
+			case 'x':
+				print_num(va_arg(args, size_t), 16, w, flags);
+				p++;
+				break;
+
+			case 'u':
+				print_num(va_arg(args, size_t), 10, w, flags);
+				p++;
+				break;
+
+			case 'o':
+				print_num(va_arg(args, size_t), 8, w, flags);
+				p++;
+				break;
+
+			case 'c':
+				buf[1] = 0;
+				buf[0] = va_arg(args, int);
+				print_string(buf, buf, w, flags, ' ');
+				p++;
+				break;
+
+			case '%':
+				break;
+
+			default:
+				dlog_putchar('%');
+			}
+
+			break;
+		}
+	}
+
+	unlock();
+}
+
+/**
+ * Prints the given format string to the debug log.
+ */
+void dlog(const char *fmt, ...)
+{
+	va_list args;
+
+	va_start(args, fmt);
+	vdlog(fmt, args);
+	va_end(args);
+}
diff --git a/src/fdt.c b/src/fdt.c
new file mode 100644
index 0000000..d12faf2
--- /dev/null
+++ b/src/fdt.c
@@ -0,0 +1,467 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/fdt.h"
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+struct fdt_header {
+	uint32_t magic;
+	uint32_t totalsize;
+	uint32_t off_dt_struct;
+	uint32_t off_dt_strings;
+	uint32_t off_mem_rsvmap;
+	uint32_t version;
+	uint32_t last_comp_version;
+	uint32_t boot_cpuid_phys;
+	uint32_t size_dt_strings;
+	uint32_t size_dt_struct;
+};
+
+struct fdt_reserve_entry {
+	uint64_t address;
+	uint64_t size;
+};
+
+enum fdt_token {
+	FDT_BEGIN_NODE = 1,
+	FDT_END_NODE = 2,
+	FDT_PROP = 3,
+	FDT_NOP = 4,
+	FDT_END = 9,
+};
+
+struct fdt_tokenizer {
+	const char *cur;
+	const char *end;
+	const char *strs;
+};
+
+#define FDT_VERSION 17
+#define FDT_MAGIC 0xd00dfeed
+
+#define FDT_TOKEN_ALIGNMENT sizeof(uint32_t)
+
+static void fdt_tokenizer_init(struct fdt_tokenizer *t, const char *strs,
+			       const char *begin, const char *end)
+{
+	t->strs = strs;
+	t->cur = begin;
+	t->end = end;
+}
+
+static void fdt_tokenizer_align(struct fdt_tokenizer *t)
+{
+	t->cur = (char *)align_up(t->cur, FDT_TOKEN_ALIGNMENT);
+}
+
+static bool fdt_tokenizer_uint32(struct fdt_tokenizer *t, uint32_t *res)
+{
+	const char *next = t->cur + sizeof(*res);
+
+	if (next > t->end) {
+		return false;
+	}
+
+	*res = be32toh(*(uint32_t *)t->cur);
+	t->cur = next;
+
+	return true;
+}
+
+static bool fdt_tokenizer_token(struct fdt_tokenizer *t, uint32_t *res)
+{
+	uint32_t v;
+
+	while (fdt_tokenizer_uint32(t, &v)) {
+		if (v != FDT_NOP) {
+			*res = v;
+			return true;
+		}
+	}
+	return false;
+}
+
+static bool fdt_tokenizer_bytes(struct fdt_tokenizer *t, const char **res,
+				size_t size)
+{
+	const char *next = t->cur + size;
+
+	if (next > t->end) {
+		return false;
+	}
+
+	*res = t->cur;
+	t->cur = next;
+	fdt_tokenizer_align(t);
+
+	return true;
+}
+
+static bool fdt_tokenizer_str(struct fdt_tokenizer *t, const char **res)
+{
+	const char *p;
+
+	for (p = t->cur; p < t->end; p++) {
+		if (!*p) {
+			/* Found the end of the string. */
+			*res = t->cur;
+			t->cur = p + 1;
+			fdt_tokenizer_align(t);
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool fdt_root_node(struct fdt_node *node, const struct fdt_header *hdr)
+{
+	uint32_t max_ver;
+	uint32_t min_ver;
+	uint32_t begin = be32toh(hdr->off_dt_struct);
+	uint32_t size = be32toh(hdr->size_dt_struct);
+
+	memset_s(node, sizeof(*node), 0, sizeof(*node));
+
+	/* Check the magic number before anything else. */
+	if (hdr->magic != be32toh(FDT_MAGIC)) {
+		return false;
+	}
+
+	/* Check the version. */
+	max_ver = be32toh(hdr->version);
+	min_ver = be32toh(hdr->last_comp_version);
+	if (FDT_VERSION < min_ver || FDT_VERSION > max_ver) {
+		return false;
+	}
+
+	/* TODO: Verify that it is all within the fdt. */
+	node->begin = (const char *)hdr + begin;
+	node->end = node->begin + size;
+
+	/* TODO: Verify strings as well. */
+	node->strs = (char *)hdr + be32toh(hdr->off_dt_strings);
+
+	return true;
+}
+
+static bool fdt_next_property(struct fdt_tokenizer *t, const char **name,
+			      const char **buf, uint32_t *size)
+{
+	uint32_t token;
+	uint32_t nameoff;
+
+	if (!fdt_tokenizer_token(t, &token)) {
+		return false;
+	}
+
+	if (token != FDT_PROP) {
+		/* Rewind so that caller will get the same token. */
+		t->cur -= sizeof(uint32_t);
+		return false;
+	}
+
+	if (!fdt_tokenizer_uint32(t, size) ||
+	    !fdt_tokenizer_uint32(t, &nameoff) ||
+	    !fdt_tokenizer_bytes(t, buf, *size)) {
+		/*
+		 * Move cursor to the end so that caller won't get any new
+		 * tokens.
+		 */
+		t->cur = t->end;
+		return false;
+	}
+
+	/* TODO: Need to verify the strings. */
+	*name = t->strs + nameoff;
+
+	return true;
+}
+
+static bool fdt_next_subnode(struct fdt_tokenizer *t, const char **name)
+{
+	uint32_t token;
+
+	if (!fdt_tokenizer_token(t, &token)) {
+		return false;
+	}
+
+	if (token != FDT_BEGIN_NODE) {
+		/* Rewind so that caller will get the same token. */
+		t->cur -= sizeof(uint32_t);
+		return false;
+	}
+
+	if (!fdt_tokenizer_str(t, name)) {
+		/*
+		 * Move cursor to the end so that caller won't get any new
+		 * tokens.
+		 */
+		t->cur = t->end;
+		return false;
+	}
+
+	return true;
+}
+
+static void fdt_skip_properties(struct fdt_tokenizer *t)
+{
+	const char *name;
+	const char *buf;
+	uint32_t size;
+
+	while (fdt_next_property(t, &name, &buf, &size)) {
+		/* do nothing */
+	}
+}
+
+static bool fdt_skip_node(struct fdt_tokenizer *t)
+{
+	const char *name;
+	uint32_t token;
+	size_t pending = 1;
+
+	fdt_skip_properties(t);
+
+	do {
+		while (fdt_next_subnode(t, &name)) {
+			fdt_skip_properties(t);
+			pending++;
+		}
+
+		if (!fdt_tokenizer_token(t, &token)) {
+			return false;
+		}
+
+		if (token != FDT_END_NODE) {
+			t->cur = t->end;
+			return false;
+		}
+
+		pending--;
+	} while (pending);
+
+	return true;
+}
+
+bool fdt_read_property(const struct fdt_node *node, const char *name,
+		       const char **buf, uint32_t *size)
+{
+	struct fdt_tokenizer t;
+	const char *prop_name;
+
+	fdt_tokenizer_init(&t, node->strs, node->begin, node->end);
+
+	while (fdt_next_property(&t, &prop_name, buf, size)) {
+		if (!strcmp(prop_name, name)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+/**
+ * Helper method for parsing 32/64-bit uints from FDT data.
+ */
+bool fdt_parse_number(const char *data, uint32_t size, uint64_t *value)
+{
+	union {
+		volatile uint64_t v;
+		char a[8];
+	} t;
+
+	/* FDT values should be aligned to 32-bit boundary. */
+	CHECK(is_aligned(data, FDT_TOKEN_ALIGNMENT));
+
+	switch (size) {
+	case sizeof(uint32_t):
+		/*
+		 * Assert that `data` is already sufficiently aligned to
+		 * dereference as uint32_t. We cannot use static_assert()
+		 * because alignof() is not an expression under ISO C11.
+		 */
+		CHECK(alignof(uint32_t) <= FDT_TOKEN_ALIGNMENT);
+		*value = be32toh(*(uint32_t *)data);
+		return true;
+	case sizeof(uint64_t):
+		/*
+		 * Armv8 requires `data` to be realigned to 64-bit boundary
+		 * to dereference as uint64_t. May not be needed on other
+		 * architectures.
+		 */
+		memcpy_s(t.a, sizeof(t.a), data, sizeof(uint64_t));
+		*value = be64toh(t.v);
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool fdt_first_child(struct fdt_node *node, const char **child_name)
+{
+	struct fdt_tokenizer t;
+
+	fdt_tokenizer_init(&t, node->strs, node->begin, node->end);
+
+	fdt_skip_properties(&t);
+
+	if (!fdt_next_subnode(&t, child_name)) {
+		return false;
+	}
+
+	node->begin = t.cur;
+
+	return true;
+}
+
+bool fdt_next_sibling(struct fdt_node *node, const char **sibling_name)
+{
+	struct fdt_tokenizer t;
+
+	fdt_tokenizer_init(&t, node->strs, node->begin, node->end);
+
+	if (!fdt_skip_node(&t)) {
+		return false;
+	}
+
+	if (!fdt_next_subnode(&t, sibling_name)) {
+		return false;
+	}
+
+	node->begin = t.cur;
+
+	return true;
+}
+
+bool fdt_find_child(struct fdt_node *node, const char *child)
+{
+	struct fdt_tokenizer t;
+	const char *name;
+
+	fdt_tokenizer_init(&t, node->strs, node->begin, node->end);
+
+	fdt_skip_properties(&t);
+
+	while (fdt_next_subnode(&t, &name)) {
+		if (!strcmp(name, child)) {
+			node->begin = t.cur;
+			return true;
+		}
+
+		fdt_skip_node(&t);
+	}
+
+	return false;
+}
+
+void fdt_dump(const struct fdt_header *hdr)
+{
+	uint32_t token;
+	size_t depth = 0;
+	const char *name;
+	struct fdt_tokenizer t;
+	struct fdt_node node;
+
+	/* Traverse the whole thing. */
+	if (!fdt_root_node(&node, hdr)) {
+		dlog("FDT failed validation.\n");
+		return;
+	}
+
+	fdt_tokenizer_init(&t, node.strs, node.begin, node.end);
+
+	do {
+		while (fdt_next_subnode(&t, &name)) {
+			const char *buf;
+			uint32_t size;
+
+			dlog("%*sNew node: \"%s\"\n", 2 * depth, "", name);
+			depth++;
+			while (fdt_next_property(&t, &name, &buf, &size)) {
+				uint32_t i;
+
+				dlog("%*sproperty: \"%s\" (", 2 * depth, "",
+				     name);
+				for (i = 0; i < size; i++) {
+					dlog("%s%02x", i == 0 ? "" : " ",
+					     buf[i]);
+				}
+				dlog(")\n");
+			}
+		}
+
+		if (!fdt_tokenizer_token(&t, &token)) {
+			return;
+		}
+
+		if (token != FDT_END_NODE) {
+			return;
+		}
+
+		depth--;
+	} while (depth);
+
+	dlog("fdt: off_mem_rsvmap=%u\n", be32toh(hdr->off_mem_rsvmap));
+	{
+		struct fdt_reserve_entry *e =
+			(struct fdt_reserve_entry
+				 *)((uintptr_t)hdr +
+				    be32toh(hdr->off_mem_rsvmap));
+		while (e->address || e->size) {
+			dlog("Entry: %p (%#x bytes)\n", be64toh(e->address),
+			     be64toh(e->size));
+			e++;
+		}
+	}
+}
+
+void fdt_add_mem_reservation(struct fdt_header *hdr, uint64_t addr,
+			     uint64_t len)
+{
+	/* TODO: Clean this up. */
+	uint8_t *begin = (uint8_t *)hdr + be32toh(hdr->off_mem_rsvmap);
+	struct fdt_reserve_entry *e = (struct fdt_reserve_entry *)begin;
+	size_t old_size =
+		be32toh(hdr->totalsize) - be32toh(hdr->off_mem_rsvmap);
+
+	hdr->totalsize = htobe32(be32toh(hdr->totalsize) +
+				 sizeof(struct fdt_reserve_entry));
+	hdr->off_dt_struct = htobe32(be32toh(hdr->off_dt_struct) +
+				     sizeof(struct fdt_reserve_entry));
+	hdr->off_dt_strings = htobe32(be32toh(hdr->off_dt_strings) +
+				      sizeof(struct fdt_reserve_entry));
+	memmove_s(begin + sizeof(struct fdt_reserve_entry), old_size, begin,
+		  old_size);
+	e->address = htobe64(addr);
+	e->size = htobe64(len);
+}
+
+size_t fdt_header_size(void)
+{
+	return sizeof(struct fdt_header);
+}
+
+uint32_t fdt_total_size(struct fdt_header *hdr)
+{
+	return be32toh(hdr->totalsize);
+}
diff --git a/src/fdt_handler.c b/src/fdt_handler.c
new file mode 100644
index 0000000..057a2ca
--- /dev/null
+++ b/src/fdt_handler.c
@@ -0,0 +1,380 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/fdt_handler.h"
+
+#include "hf/boot_params.h"
+#include "hf/check.h"
+#include "hf/cpu.h"
+#include "hf/dlog.h"
+#include "hf/fdt.h"
+#include "hf/layout.h"
+#include "hf/mm.h"
+#include "hf/std.h"
+
+static bool fdt_read_number(const struct fdt_node *node, const char *name,
+			    uint64_t *value)
+{
+	const char *data;
+	uint32_t size;
+
+	if (!fdt_read_property(node, name, &data, &size)) {
+		return false;
+	}
+
+	switch (size) {
+	case sizeof(uint32_t):
+	case sizeof(uint64_t):
+		CHECK(fdt_parse_number(data, size, value));
+		break;
+
+	default:
+		return false;
+	}
+
+	return true;
+}
+
+static bool fdt_write_number(struct fdt_node *node, const char *name,
+			     uint64_t value)
+{
+	const char *data;
+	uint32_t size;
+	union {
+		volatile uint64_t v;
+		char a[8];
+	} t;
+
+	if (!fdt_read_property(node, name, &data, &size)) {
+		return false;
+	}
+
+	switch (size) {
+	case sizeof(uint32_t):
+		*(uint32_t *)data = be32toh(value);
+		break;
+
+	case sizeof(uint64_t):
+		t.v = be64toh(value);
+		memcpy_s((void *)data, size, t.a, sizeof(uint64_t));
+		break;
+
+	default:
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * Finds the memory region where initrd is stored.
+ */
+bool fdt_find_initrd(const struct fdt_node *root, paddr_t *begin, paddr_t *end)
+{
+	struct fdt_node n = *root;
+	uint64_t initrd_begin;
+	uint64_t initrd_end;
+
+	if (!fdt_find_child(&n, "chosen")) {
+		dlog("Unable to find 'chosen'\n");
+		return false;
+	}
+
+	if (!fdt_read_number(&n, "linux,initrd-start", &initrd_begin)) {
+		dlog("Unable to read linux,initrd-start\n");
+		return false;
+	}
+
+	if (!fdt_read_number(&n, "linux,initrd-end", &initrd_end)) {
+		dlog("Unable to read linux,initrd-end\n");
+		return false;
+	}
+
+	*begin = pa_init(initrd_begin);
+	*end = pa_init(initrd_end);
+
+	return true;
+}
+
+bool fdt_find_cpus(const struct fdt_node *root, cpu_id_t *cpu_ids,
+		   size_t *cpu_count)
+{
+	struct fdt_node n = *root;
+	const char *name;
+	uint64_t address_size;
+
+	*cpu_count = 0;
+
+	if (!fdt_find_child(&n, "cpus")) {
+		dlog("Unable to find 'cpus'\n");
+		return false;
+	}
+
+	if (fdt_read_number(&n, "#address-cells", &address_size)) {
+		address_size *= sizeof(uint32_t);
+	} else {
+		address_size = sizeof(uint32_t);
+	}
+
+	if (!fdt_first_child(&n, &name)) {
+		return false;
+	}
+
+	do {
+		const char *data;
+		uint32_t size;
+
+		if (!fdt_read_property(&n, "device_type", &data, &size) ||
+		    size != sizeof("cpu") ||
+		    memcmp(data, "cpu", sizeof("cpu")) != 0 ||
+		    !fdt_read_property(&n, "reg", &data, &size)) {
+			continue;
+		}
+
+		/* Get all entries for this CPU. */
+		while (size >= address_size) {
+			uint64_t value;
+
+			if (*cpu_count >= MAX_CPUS) {
+				dlog("Found more than %d CPUs\n", MAX_CPUS);
+				return false;
+			}
+
+			if (!fdt_parse_number(data, address_size, &value)) {
+				dlog("Could not parse CPU id\n");
+				return false;
+			}
+			cpu_ids[(*cpu_count)++] = value;
+
+			size -= address_size;
+			data += address_size;
+		}
+	} while (fdt_next_sibling(&n, &name));
+
+	return true;
+}
+
+bool fdt_find_memory_ranges(const struct fdt_node *root, struct boot_params *p)
+{
+	struct fdt_node n = *root;
+	const char *name;
+	uint64_t address_size;
+	uint64_t size_size;
+	uint64_t entry_size;
+	size_t mem_range_index = 0;
+
+	/* Get the sizes of memory range addresses and sizes. */
+	if (fdt_read_number(&n, "#address-cells", &address_size)) {
+		address_size *= sizeof(uint32_t);
+	} else {
+		address_size = sizeof(uint32_t);
+	}
+
+	if (fdt_read_number(&n, "#size-cells", &size_size)) {
+		size_size *= sizeof(uint32_t);
+	} else {
+		size_size = sizeof(uint32_t);
+	}
+
+	entry_size = address_size + size_size;
+
+	/* Look for nodes with the device_type set to "memory". */
+	if (!fdt_first_child(&n, &name)) {
+		return false;
+	}
+
+	do {
+		const char *data;
+		uint32_t size;
+
+		if (!fdt_read_property(&n, "device_type", &data, &size) ||
+		    size != sizeof("memory") ||
+		    memcmp(data, "memory", sizeof("memory")) != 0 ||
+		    !fdt_read_property(&n, "reg", &data, &size)) {
+			continue;
+		}
+
+		/* Traverse all memory ranges within this node. */
+		while (size >= entry_size) {
+			uintpaddr_t addr;
+			size_t len;
+
+			CHECK(fdt_parse_number(data, address_size, &addr));
+			CHECK(fdt_parse_number(data + address_size, size_size,
+					       &len));
+
+			if (mem_range_index < MAX_MEM_RANGES) {
+				p->mem_ranges[mem_range_index].begin =
+					pa_init(addr);
+				p->mem_ranges[mem_range_index].end =
+					pa_init(addr + len);
+				++mem_range_index;
+			} else {
+				dlog("Found memory range %u in FDT but only "
+				     "%u supported, ignoring additional range "
+				     "of size %u.\n",
+				     mem_range_index, MAX_MEM_RANGES, len);
+			}
+
+			size -= entry_size;
+			data += entry_size;
+		}
+	} while (fdt_next_sibling(&n, &name));
+	p->mem_ranges_count = mem_range_index;
+
+	/* TODO: Check for "reserved-memory" nodes. */
+
+	return true;
+}
+
+struct fdt_header *fdt_map(struct mm_stage1_locked stage1_locked,
+			   paddr_t fdt_addr, struct fdt_node *n,
+			   struct mpool *ppool)
+{
+	struct fdt_header *fdt;
+
+	/* Map the fdt header in. */
+	fdt = mm_identity_map(stage1_locked, fdt_addr,
+			      pa_add(fdt_addr, fdt_header_size()), MM_MODE_R,
+			      ppool);
+	if (!fdt) {
+		dlog("Unable to map FDT header.\n");
+		return NULL;
+	}
+
+	if (!fdt_root_node(n, fdt)) {
+		dlog("FDT failed validation.\n");
+		goto fail;
+	}
+
+	/* Map the rest of the fdt in. */
+	fdt = mm_identity_map(stage1_locked, fdt_addr,
+			      pa_add(fdt_addr, fdt_total_size(fdt)), MM_MODE_R,
+			      ppool);
+	if (!fdt) {
+		dlog("Unable to map full FDT.\n");
+		goto fail;
+	}
+
+	return fdt;
+
+fail:
+	mm_unmap(stage1_locked, fdt_addr, pa_add(fdt_addr, fdt_header_size()),
+		 ppool);
+	return NULL;
+}
+
+bool fdt_unmap(struct mm_stage1_locked stage1_locked, struct fdt_header *fdt,
+	       struct mpool *ppool)
+{
+	paddr_t fdt_addr = pa_from_va(va_from_ptr(fdt));
+
+	return mm_unmap(stage1_locked, fdt_addr,
+			pa_add(fdt_addr, fdt_total_size(fdt)), ppool);
+}
+
+bool fdt_patch(struct mm_stage1_locked stage1_locked, paddr_t fdt_addr,
+	       struct boot_params_update *p, struct mpool *ppool)
+{
+	struct fdt_header *fdt;
+	struct fdt_node n;
+	bool ret = false;
+	size_t i;
+
+	/* Map the fdt header in. */
+	fdt = mm_identity_map(stage1_locked, fdt_addr,
+			      pa_add(fdt_addr, fdt_header_size()), MM_MODE_R,
+			      ppool);
+	if (!fdt) {
+		dlog("Unable to map FDT header.\n");
+		return false;
+	}
+
+	if (!fdt_root_node(&n, fdt)) {
+		dlog("FDT failed validation.\n");
+		goto err_unmap_fdt_header;
+	}
+
+	/* Map the fdt (+ a page) in r/w mode in preparation for updating it. */
+	fdt = mm_identity_map(stage1_locked, fdt_addr,
+			      pa_add(fdt_addr, fdt_total_size(fdt) + PAGE_SIZE),
+			      MM_MODE_R | MM_MODE_W, ppool);
+	if (!fdt) {
+		dlog("Unable to map FDT in r/w mode.\n");
+		goto err_unmap_fdt_header;
+	}
+
+	if (!fdt_find_child(&n, "")) {
+		dlog("Unable to find FDT root node.\n");
+		goto out_unmap_fdt;
+	}
+
+	if (!fdt_find_child(&n, "chosen")) {
+		dlog("Unable to find 'chosen'\n");
+		goto out_unmap_fdt;
+	}
+
+	/* Patch FDT to point to new ramdisk. */
+	if (!fdt_write_number(&n, "linux,initrd-start",
+			      pa_addr(p->initrd_begin))) {
+		dlog("Unable to write linux,initrd-start\n");
+		goto out_unmap_fdt;
+	}
+
+	if (!fdt_write_number(&n, "linux,initrd-end", pa_addr(p->initrd_end))) {
+		dlog("Unable to write linux,initrd-end\n");
+		goto out_unmap_fdt;
+	}
+
+	/*
+	 * Patch FDT to reserve hypervisor memory so the primary VM doesn't try
+	 * to use it.
+	 */
+	fdt_add_mem_reservation(
+		fdt, pa_addr(layout_text_begin()),
+		pa_difference(layout_text_begin(), layout_text_end()));
+	fdt_add_mem_reservation(
+		fdt, pa_addr(layout_rodata_begin()),
+		pa_difference(layout_rodata_begin(), layout_rodata_end()));
+	fdt_add_mem_reservation(
+		fdt, pa_addr(layout_data_begin()),
+		pa_difference(layout_data_begin(), layout_data_end()));
+
+	/* Patch FDT to reserve memory for secondary VMs. */
+	for (i = 0; i < p->reserved_ranges_count; ++i) {
+		fdt_add_mem_reservation(
+			fdt, pa_addr(p->reserved_ranges[i].begin),
+			pa_addr(p->reserved_ranges[i].end) -
+				pa_addr(p->reserved_ranges[i].begin));
+	}
+
+	ret = true;
+
+out_unmap_fdt:
+	/* Unmap FDT. */
+	if (!mm_unmap(stage1_locked, fdt_addr,
+		      pa_add(fdt_addr, fdt_total_size(fdt) + PAGE_SIZE),
+		      ppool)) {
+		dlog("Unable to unmap writable FDT.\n");
+		return false;
+	}
+	return ret;
+
+err_unmap_fdt_header:
+	mm_unmap(stage1_locked, fdt_addr, pa_add(fdt_addr, fdt_header_size()),
+		 ppool);
+	return false;
+}
diff --git a/src/fdt_handler_test.cc b/src/fdt_handler_test.cc
new file mode 100644
index 0000000..acd355e
--- /dev/null
+++ b/src/fdt_handler_test.cc
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gmock/gmock.h>
+
+extern "C" {
+#include "hf/boot_params.h"
+#include "hf/fdt_handler.h"
+#include "hf/mpool.h"
+}
+
+#include <memory>
+
+namespace
+{
+using ::testing::Eq;
+using ::testing::NotNull;
+
+constexpr size_t TEST_HEAP_SIZE = PAGE_SIZE * 10;
+
+/*
+ * /dts-v1/;
+ *
+ * / {
+ *       #address-cells = <2>;
+ *       #size-cells = <2>;
+ *
+ *       memory@0 {
+ *           device_type = "memory";
+ *           reg = <0x00000000 0x00000000 0x00000000 0x20000000
+ *                  0x00000000 0x30000000 0x00000000 0x00010000>;
+ *       };
+ *       memory@1 {
+ *           device_type = "memory";
+ *           reg = <0x00000000 0x30020000 0x00000000 0x00010000>;
+ *       };
+ *
+ *       chosen {
+ *           linux,initrd-start = <0x00000000>;
+ *           linux,initrd-end = <0x00000000>;
+ *       };
+ * };
+ *
+ * $ dtc --boot-cpu 0 --in-format dts --out-format dtb --out-version 17 test.dts
+ * | xxd -i
+ */
+
+constexpr uint8_t test_dtb[] = {
+	0xd0, 0x0d, 0xfe, 0xed, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, 0x00, 0x38,
+	0x00, 0x00, 0x01, 0x30, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x11,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f,
+	0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
+	0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x02,
+	0x00, 0x00, 0x00, 0x01, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x40, 0x30,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x07,
+	0x00, 0x00, 0x00, 0x1b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x27,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
+	0x00, 0x00, 0x00, 0x01, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x40, 0x31,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x07,
+	0x00, 0x00, 0x00, 0x1b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x27,
+	0x00, 0x00, 0x00, 0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01,
+	0x63, 0x68, 0x6f, 0x73, 0x65, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+	0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x3e,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02,
+	0x00, 0x00, 0x00, 0x09, 0x23, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73,
+	0x2d, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x00, 0x23, 0x73, 0x69, 0x7a, 0x65,
+	0x2d, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x00, 0x64, 0x65, 0x76, 0x69, 0x63,
+	0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x00, 0x72, 0x65, 0x67, 0x00, 0x6c,
+	0x69, 0x6e, 0x75, 0x78, 0x2c, 0x69, 0x6e, 0x69, 0x74, 0x72, 0x64, 0x2d,
+	0x73, 0x74, 0x61, 0x72, 0x74, 0x00, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x2c,
+	0x69, 0x6e, 0x69, 0x74, 0x72, 0x64, 0x2d, 0x65, 0x6e, 0x64, 0x00};
+
+TEST(fdt, find_memory_ranges)
+{
+	struct mpool ppool;
+	std::unique_ptr<uint8_t[]> test_heap(new uint8_t[TEST_HEAP_SIZE]);
+
+	mpool_init(&ppool, sizeof(struct mm_page_table));
+	mpool_add_chunk(&ppool, test_heap.get(), TEST_HEAP_SIZE);
+	mm_init(&ppool);
+
+	struct fdt_header *fdt;
+	struct fdt_node n;
+	struct boot_params params = {};
+
+	struct mm_stage1_locked mm_stage1_locked = mm_lock_stage1();
+	fdt = fdt_map(mm_stage1_locked, pa_init((uintpaddr_t)&test_dtb), &n,
+		      &ppool);
+	ASSERT_THAT(fdt, NotNull());
+	ASSERT_TRUE(fdt_find_child(&n, ""));
+	fdt_find_memory_ranges(&n, &params);
+	ASSERT_TRUE(fdt_unmap(mm_stage1_locked, fdt, &ppool));
+	mm_unlock_stage1(&mm_stage1_locked);
+
+	EXPECT_THAT(params.mem_ranges_count, Eq(3));
+	EXPECT_THAT(pa_addr(params.mem_ranges[0].begin), Eq(0x00000000));
+	EXPECT_THAT(pa_addr(params.mem_ranges[0].end), Eq(0x20000000));
+	EXPECT_THAT(pa_addr(params.mem_ranges[1].begin), Eq(0x30000000));
+	EXPECT_THAT(pa_addr(params.mem_ranges[1].end), Eq(0x30010000));
+	EXPECT_THAT(pa_addr(params.mem_ranges[2].begin), Eq(0x30020000));
+	EXPECT_THAT(pa_addr(params.mem_ranges[2].end), Eq(0x30030000));
+}
+
+} /* namespace */
diff --git a/src/fdt_test.cc b/src/fdt_test.cc
new file mode 100644
index 0000000..0ee534d
--- /dev/null
+++ b/src/fdt_test.cc
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+extern "C" {
+
+#include "hf/fdt.h"
+}
+
+#include <gmock/gmock.h>
+
+namespace
+{
+using ::testing::Eq;
+
+/*
+ * /dts-v1/;
+ *
+ * / {
+ *       model = "SomeModel";
+ *       compatible = "Nothing";
+ *       #address-cells = <2>;
+ *       #size-cells = <2>;
+ *
+ *       memory@0 {
+ *           device_type = "memory";
+ *           reg = <0x00000000 0x00000000 0x00000000 0x20000000>;
+ *       };
+ *
+ *       cpus {
+ *           #address-cells = <1>;
+ *           #size-cells = <0>;
+ *       };
+ *
+ * };
+ *
+ * $ dtc --boot-cpu 0 --in-format dts --out-format dtb --out-version 17 test.dts
+ * | xxd -i
+ */
+
+const uint8_t test_dtb[] = {
+	0xd0, 0x0d, 0xfe, 0xed, 0x00, 0x00, 0x01, 0x44, 0x00, 0x00, 0x00, 0x38,
+	0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x11,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c,
+	0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x0a,
+	0x00, 0x00, 0x00, 0x00, 0x53, 0x6f, 0x6d, 0x65, 0x4d, 0x6f, 0x64, 0x65,
+	0x6c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x08,
+	0x00, 0x00, 0x00, 0x06, 0x4e, 0x6f, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x00,
+	0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x11,
+	0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04,
+	0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01,
+	0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x40, 0x30, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x2c,
+	0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x63, 0x70, 0x75, 0x73,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04,
+	0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03,
+	0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x09,
+	0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x00, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x74,
+	0x69, 0x62, 0x6c, 0x65, 0x00, 0x23, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73,
+	0x73, 0x2d, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x00, 0x23, 0x73, 0x69, 0x7a,
+	0x65, 0x2d, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x00, 0x64, 0x65, 0x76, 0x69,
+	0x63, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x00, 0x72, 0x65, 0x67, 0x00};
+
+TEST(fdt, total_size)
+{
+	EXPECT_THAT(fdt_total_size((struct fdt_header *)&test_dtb[0]),
+		    Eq(sizeof(test_dtb)));
+}
+
+} /* namespace */
diff --git a/src/init.c b/src/init.c
new file mode 100644
index 0000000..897039f
--- /dev/null
+++ b/src/init.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/init.h"
+
+#include <stdalign.h>
+#include <stddef.h>
+
+#include "hf/api.h"
+#include "hf/boot_flow.h"
+#include "hf/boot_params.h"
+#include "hf/cpio.h"
+#include "hf/cpu.h"
+#include "hf/dlog.h"
+#include "hf/fdt_handler.h"
+#include "hf/load.h"
+#include "hf/mm.h"
+#include "hf/mpool.h"
+#include "hf/panic.h"
+#include "hf/plat/boot_flow.h"
+#include "hf/plat/console.h"
+#include "hf/plat/iommu.h"
+#include "hf/std.h"
+#include "hf/vm.h"
+
+#include "vmapi/hf/call.h"
+
+alignas(alignof(
+	struct mm_page_table)) char ptable_buf[sizeof(struct mm_page_table) *
+					       HEAP_PAGES];
+
+static struct mpool ppool;
+
+/**
+ * Performs one-time initialisation of memory management for the hypervisor.
+ *
+ * This is the only C code entry point called with MMU and caching disabled. The
+ * page table returned is used to set up the MMU and caches for all subsequent
+ * code.
+ */
+void one_time_init_mm(void)
+{
+	/* Make sure the console is initialised before calling dlog. */
+	plat_console_init();
+
+	dlog("Initialising hafnium\n");
+
+	mpool_init(&ppool, sizeof(struct mm_page_table));
+	mpool_add_chunk(&ppool, ptable_buf, sizeof(ptable_buf));
+
+	if (!mm_init(&ppool)) {
+		panic("mm_init failed");
+	}
+}
+
+/**
+ * Performs one-time initialisation of the hypervisor.
+ */
+void one_time_init(void)
+{
+	struct fdt_header *fdt;
+	struct fdt_node fdt_root;
+	struct manifest manifest;
+	struct boot_params params;
+	struct boot_params_update update;
+	struct memiter cpio;
+	void *initrd;
+	size_t i;
+	struct mm_stage1_locked mm_stage1_locked;
+
+	arch_one_time_init();
+
+	/* Enable locks now that mm is initialised. */
+	dlog_enable_lock();
+	mpool_enable_locks();
+
+	mm_stage1_locked = mm_lock_stage1();
+
+	fdt = fdt_map(mm_stage1_locked, plat_boot_flow_get_fdt_addr(),
+		      &fdt_root, &ppool);
+	if (fdt == NULL) {
+		panic("Unable to map FDT.\n");
+	}
+
+	if (!fdt_find_child(&fdt_root, "")) {
+		panic("Unable to find FDT root node.\n");
+	}
+
+	if (!boot_flow_init(&fdt_root, &manifest, &params)) {
+		panic("Could not parse data from FDT.");
+	}
+
+	if (!plat_iommu_init(&fdt_root, mm_stage1_locked, &ppool)) {
+		panic("Could not initialize IOMMUs.");
+	}
+
+	if (!fdt_unmap(mm_stage1_locked, fdt, &ppool)) {
+		panic("Unable to unmap FDT.\n");
+	}
+
+	cpu_module_init(params.cpu_ids, params.cpu_count);
+
+	for (i = 0; i < params.mem_ranges_count; ++i) {
+		dlog("Memory range:  %#x - %#x\n",
+		     pa_addr(params.mem_ranges[i].begin),
+		     pa_addr(params.mem_ranges[i].end) - 1);
+	}
+
+	dlog("Ramdisk range: %#x - %#x\n", pa_addr(params.initrd_begin),
+	     pa_addr(params.initrd_end) - 1);
+
+	/* Map initrd in, and initialise cpio parser. */
+	initrd = mm_identity_map(mm_stage1_locked, params.initrd_begin,
+				 params.initrd_end, MM_MODE_R, &ppool);
+	if (!initrd) {
+		panic("Unable to map initrd.");
+	}
+
+	memiter_init(&cpio, initrd,
+		     pa_difference(params.initrd_begin, params.initrd_end));
+
+	/* Load all VMs. */
+	update.reserved_ranges_count = 0;
+	if (!load_vms(mm_stage1_locked, &manifest, &cpio, &params, &update,
+		      &ppool)) {
+		panic("Unable to load VMs.");
+	}
+
+	if (!boot_flow_update(mm_stage1_locked, &manifest, &update, &cpio,
+			      &ppool)) {
+		panic("Unable to update boot flow.");
+	}
+
+	mm_defrag(mm_stage1_locked, &ppool);
+	mm_unlock_stage1(&mm_stage1_locked);
+
+	/* Initialise the API page pool. ppool will be empty from now on. */
+	api_init(&ppool);
+
+	/* Enable TLB invalidation for VM page table updates. */
+	mm_vm_enable_invalidation();
+
+	dlog("Hafnium initialisation completed\n");
+}
diff --git a/src/iommu/BUILD.gn b/src/iommu/BUILD.gn
new file mode 100644
index 0000000..00004f3
--- /dev/null
+++ b/src/iommu/BUILD.gn
@@ -0,0 +1,19 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source_set("absent") {
+  sources = [
+    "absent.c",
+  ]
+}
diff --git a/src/iommu/absent.c b/src/iommu/absent.c
new file mode 100644
index 0000000..c89e9bb
--- /dev/null
+++ b/src/iommu/absent.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/plat/iommu.h"
+
+bool plat_iommu_init(const struct fdt_node *fdt_root,
+		     struct mm_stage1_locked stage1_locked, struct mpool *ppool)
+{
+	(void)fdt_root;
+	(void)stage1_locked;
+	(void)ppool;
+
+	return true;
+}
+
+void plat_iommu_identity_map(struct vm_locked vm_locked, paddr_t begin,
+			     paddr_t end, uint32_t mode)
+{
+	(void)vm_locked;
+	(void)begin;
+	(void)end;
+	(void)mode;
+}
diff --git a/src/layout.c b/src/layout.c
new file mode 100644
index 0000000..edb583d
--- /dev/null
+++ b/src/layout.c
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/layout.h"
+
+#include "hf/std.h"
+
+/**
+ * Get the address the .text section begins at.
+ */
+paddr_t layout_text_begin(void)
+{
+	extern uint8_t text_begin[];
+
+	return pa_init((uintpaddr_t)text_begin);
+}
+
+/**
+ * Get the address the .text section ends at.
+ */
+paddr_t layout_text_end(void)
+{
+	extern uint8_t text_end[];
+
+	return pa_init((uintpaddr_t)text_end);
+}
+
+/**
+ * Get the address the .rodata section begins at.
+ */
+paddr_t layout_rodata_begin(void)
+{
+	extern uint8_t rodata_begin[];
+
+	return pa_init((uintpaddr_t)rodata_begin);
+}
+
+/**
+ * Get the address the .rodata section ends at.
+ */
+paddr_t layout_rodata_end(void)
+{
+	extern uint8_t rodata_end[];
+
+	return pa_init((uintpaddr_t)rodata_end);
+}
+
+/**
+ * Get the address the .data section begins at.
+ */
+paddr_t layout_data_begin(void)
+{
+	extern uint8_t data_begin[];
+
+	return pa_init((uintpaddr_t)data_begin);
+}
+
+/**
+ * Get the address the .data section ends at.
+ */
+paddr_t layout_data_end(void)
+{
+	extern uint8_t data_end[];
+
+	return pa_init((uintpaddr_t)data_end);
+}
+
+/**
+ * Get the address the .initrd section begins at.
+ */
+paddr_t layout_initrd_begin(void)
+{
+	extern uint8_t initrd_begin[];
+
+	return pa_init((uintpaddr_t)initrd_begin);
+}
+
+/**
+ * Get the address the .initrd section ends at.
+ */
+paddr_t layout_initrd_end(void)
+{
+	extern uint8_t initrd_end[];
+
+	return pa_init((uintpaddr_t)initrd_end);
+}
+
+/**
+ * Get the address the .fdt section begins at.
+ */
+paddr_t layout_fdt_begin(void)
+{
+	extern uint8_t fdt_begin[];
+
+	return pa_init((uintpaddr_t)fdt_begin);
+}
+
+/**
+ * Get the address the .fdt section ends at.
+ */
+paddr_t layout_fdt_end(void)
+{
+	extern uint8_t fdt_end[];
+
+	return pa_init((uintpaddr_t)fdt_end);
+}
+
+/**
+ * Get the address the loaded image ends at.
+ */
+paddr_t layout_image_end(void)
+{
+	extern uint8_t image_end[];
+
+	return pa_init((uintpaddr_t)image_end);
+}
+
+/**
+ * Get the address to load the primary VM at.
+ *
+ * This is placed just after the image.
+ */
+paddr_t layout_primary_begin(void)
+{
+	paddr_t image_end = layout_image_end();
+
+	/*
+	 * Linux usually expects to be loaded at offset 0x80000 into a 2MB
+	 * aligned address.
+	 * TODO: This is a hack, and isn't always correct. We should really read
+	 * the alignment from the header of the binary, or have a bootloader
+	 * within the VM do so.
+	 */
+	return pa_init(align_up(pa_addr(image_end), 0x200000) + 0x80000);
+}
diff --git a/src/layout_fake.c b/src/layout_fake.c
new file mode 100644
index 0000000..542f92a
--- /dev/null
+++ b/src/layout_fake.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/layout.h"
+
+paddr_t layout_text_begin(void)
+{
+	return pa_init(0);
+}
+
+paddr_t layout_text_end(void)
+{
+	return pa_init(100);
+}
+
+paddr_t layout_rodata_begin(void)
+{
+	return pa_init(200);
+}
+
+paddr_t layout_rodata_end(void)
+{
+	return pa_init(300);
+}
+
+paddr_t layout_data_begin(void)
+{
+	return pa_init(400);
+}
+
+paddr_t layout_data_end(void)
+{
+	return pa_init(500);
+}
+
+paddr_t layout_image_end(void)
+{
+	return pa_init(600);
+}
+
+paddr_t layout_primary_begin(void)
+{
+	return pa_init(0x80000);
+}
diff --git a/src/load.c b/src/load.c
new file mode 100644
index 0000000..d69b36d
--- /dev/null
+++ b/src/load.c
@@ -0,0 +1,420 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/load.h"
+
+#include <stdbool.h>
+
+#include "hf/arch/vm.h"
+
+#include "hf/api.h"
+#include "hf/boot_params.h"
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/layout.h"
+#include "hf/memiter.h"
+#include "hf/mm.h"
+#include "hf/plat/console.h"
+#include "hf/static_assert.h"
+#include "hf/std.h"
+#include "hf/vm.h"
+
+#include "vmapi/hf/call.h"
+
+/**
+ * Copies data to an unmapped location by mapping it for write, copying the
+ * data, then unmapping it.
+ *
+ * The data is written so that it is available to all cores with the cache
+ * disabled. When switching to the partitions, the caching is initially disabled
+ * so the data must be available without the cache.
+ */
+static bool copy_to_unmapped(struct mm_stage1_locked stage1_locked, paddr_t to,
+			     struct memiter *from_it, struct mpool *ppool)
+{
+	const void *from = memiter_base(from_it);
+	size_t size = memiter_size(from_it);
+	paddr_t to_end = pa_add(to, size);
+	void *ptr;
+
+	ptr = mm_identity_map(stage1_locked, to, to_end, MM_MODE_W, ppool);
+	if (!ptr) {
+		return false;
+	}
+
+	memcpy_s(ptr, size, from, size);
+	arch_mm_flush_dcache(ptr, size);
+
+	CHECK(mm_unmap(stage1_locked, to, to_end, ppool));
+
+	return true;
+}
+
+static bool load_kernel(struct mm_stage1_locked stage1_locked, paddr_t begin,
+			paddr_t end, const struct manifest_vm *manifest_vm,
+			const struct memiter *cpio, struct mpool *ppool)
+{
+	struct memiter kernel;
+
+	if (string_is_empty(&manifest_vm->kernel_filename)) {
+		/* This signals the kernel has been preloaded. */
+		return true;
+	}
+
+	if (!cpio_get_file(cpio, &manifest_vm->kernel_filename, &kernel)) {
+		dlog("Could not find kernel file \"%s\".\n",
+		     string_data(&manifest_vm->kernel_filename));
+		return false;
+	}
+
+	if (pa_difference(begin, end) < memiter_size(&kernel)) {
+		dlog("Kernel is larger than available memory.\n");
+		return false;
+	}
+
+	if (!copy_to_unmapped(stage1_locked, begin, &kernel, ppool)) {
+		dlog("Unable to copy kernel.\n");
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * Performs VM loading activities that are common between the primary and
+ * secondaries.
+ */
+static bool load_common(const struct manifest_vm *manifest_vm, struct vm *vm)
+{
+	vm->smc_whitelist = manifest_vm->smc_whitelist;
+
+	/* Initialize architecture-specific features.  */
+	arch_vm_features_set(vm);
+
+	return true;
+}
+
+/**
+ * Loads the primary VM.
+ */
+static bool load_primary(struct mm_stage1_locked stage1_locked,
+			 const struct manifest_vm *manifest_vm,
+			 const struct memiter *cpio,
+			 const struct boot_params *params, struct mpool *ppool)
+{
+	paddr_t primary_begin = layout_primary_begin();
+	struct vm *vm;
+	struct vm_locked vm_locked;
+	struct vcpu_locked vcpu_locked;
+	size_t i;
+	bool ret;
+
+	/*
+	 * TODO: This bound is currently meaningless but will be addressed when
+	 * the manifest specifies the load address.
+	 */
+	paddr_t primary_end = pa_add(primary_begin, 0x8000000);
+
+	if (!load_kernel(stage1_locked, primary_begin, primary_end, manifest_vm,
+			 cpio, ppool)) {
+		dlog("Unable to load primary kernel.");
+		return false;
+	}
+
+	if (!vm_init(MAX_CPUS, ppool, &vm)) {
+		dlog("Unable to initialise primary vm\n");
+		return false;
+	}
+
+	if (vm->id != HF_PRIMARY_VM_ID) {
+		dlog("Primary vm was not given correct id\n");
+		return false;
+	}
+
+	vm_locked = vm_lock(vm);
+
+	if (!load_common(manifest_vm, vm)) {
+		ret = false;
+		goto out;
+	}
+
+	/*
+	 * Map 1TB of address space as device memory to, most likely, make all
+	 * devices available to the primary VM.
+	 *
+	 * TODO: We should do a whitelist rather than a blacklist.
+	 */
+	if (!vm_identity_map(vm_locked, pa_init(0),
+			     pa_init(UINT64_C(1024) * 1024 * 1024 * 1024),
+			     MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool, NULL)) {
+		dlog("Unable to initialise address space for primary vm\n");
+		ret = false;
+		goto out;
+	}
+
+	/* Map normal memory as such to permit caching, execution, etc. */
+	for (i = 0; i < params->mem_ranges_count; ++i) {
+		if (!vm_identity_map(vm_locked, params->mem_ranges[i].begin,
+				     params->mem_ranges[i].end,
+				     MM_MODE_R | MM_MODE_W | MM_MODE_X, ppool,
+				     NULL)) {
+			dlog("Unable to initialise memory for primary vm\n");
+			ret = false;
+			goto out;
+		}
+	}
+
+	if (!vm_unmap_hypervisor(vm_locked, ppool)) {
+		dlog("Unable to unmap hypervisor from primary vm\n");
+		ret = false;
+		goto out;
+	}
+
+	vcpu_locked = vcpu_lock(vm_get_vcpu(vm, 0));
+	vcpu_on(vcpu_locked, ipa_from_pa(primary_begin), params->kernel_arg);
+	vcpu_unlock(&vcpu_locked);
+	ret = true;
+
+out:
+	vm_unlock(&vm_locked);
+
+	return ret;
+}
+
+/*
+ * Loads a secondary VM.
+ */
+static bool load_secondary(struct mm_stage1_locked stage1_locked,
+			   paddr_t mem_begin, paddr_t mem_end,
+			   const struct manifest_vm *manifest_vm,
+			   const struct memiter *cpio, struct mpool *ppool)
+{
+	struct vm *vm;
+	struct vm_locked vm_locked;
+	struct vcpu *vcpu;
+	ipaddr_t secondary_entry;
+	bool ret;
+
+	if (!load_kernel(stage1_locked, mem_begin, mem_end, manifest_vm, cpio,
+			 ppool)) {
+		dlog("Unable to load kernel.\n");
+		return false;
+	}
+
+	if (!vm_init(manifest_vm->secondary.vcpu_count, ppool, &vm)) {
+		dlog("Unable to initialise VM.\n");
+		return false;
+	}
+
+	if (!load_common(manifest_vm, vm)) {
+		return false;
+	}
+
+	vm_locked = vm_lock(vm);
+
+	/* Grant the VM access to the memory. */
+	if (!vm_identity_map(vm_locked, mem_begin, mem_end,
+			     MM_MODE_R | MM_MODE_W | MM_MODE_X, ppool,
+			     &secondary_entry)) {
+		dlog("Unable to initialise memory.\n");
+		ret = false;
+		goto out;
+	}
+
+	dlog("Loaded with %u vCPUs, entry at %#x.\n",
+	     manifest_vm->secondary.vcpu_count, pa_addr(mem_begin));
+
+	vcpu = vm_get_vcpu(vm, 0);
+	vcpu_secondary_reset_and_start(vcpu, secondary_entry,
+				       pa_difference(mem_begin, mem_end));
+	ret = true;
+
+out:
+	vm_unlock(&vm_locked);
+
+	return ret;
+}
+
+/**
+ * Try to find a memory range of the given size within the given ranges, and
+ * remove it from them. Return true on success, or false if no large enough
+ * contiguous range is found.
+ */
+static bool carve_out_mem_range(struct mem_range *mem_ranges,
+				size_t mem_ranges_count, uint64_t size_to_find,
+				paddr_t *found_begin, paddr_t *found_end)
+{
+	size_t i;
+
+	/*
+	 * TODO(b/116191358): Consider being cleverer about how we pack VMs
+	 * together, with a non-greedy algorithm.
+	 */
+	for (i = 0; i < mem_ranges_count; ++i) {
+		if (size_to_find <=
+		    pa_difference(mem_ranges[i].begin, mem_ranges[i].end)) {
+			/*
+			 * This range is big enough, take some of it from the
+			 * end and reduce its size accordingly.
+			 */
+			*found_end = mem_ranges[i].end;
+			*found_begin = pa_init(pa_addr(mem_ranges[i].end) -
+					       size_to_find);
+			mem_ranges[i].end = *found_begin;
+			return true;
+		}
+	}
+	return false;
+}
+
+/**
+ * Given arrays of memory ranges before and after memory was removed for
+ * secondary VMs, add the difference to the reserved ranges of the given update.
+ * Return true on success, or false if there would be more than MAX_MEM_RANGES
+ * reserved ranges after adding the new ones.
+ * `before` and `after` must be arrays of exactly `mem_ranges_count` elements.
+ */
+static bool update_reserved_ranges(struct boot_params_update *update,
+				   const struct mem_range *before,
+				   const struct mem_range *after,
+				   size_t mem_ranges_count)
+{
+	size_t i;
+
+	for (i = 0; i < mem_ranges_count; ++i) {
+		if (pa_addr(after[i].begin) > pa_addr(before[i].begin)) {
+			if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
+				dlog("Too many reserved ranges after loading "
+				     "secondary VMs.\n");
+				return false;
+			}
+			update->reserved_ranges[update->reserved_ranges_count]
+				.begin = before[i].begin;
+			update->reserved_ranges[update->reserved_ranges_count]
+				.end = after[i].begin;
+			update->reserved_ranges_count++;
+		}
+		if (pa_addr(after[i].end) < pa_addr(before[i].end)) {
+			if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
+				dlog("Too many reserved ranges after loading "
+				     "secondary VMs.\n");
+				return false;
+			}
+			update->reserved_ranges[update->reserved_ranges_count]
+				.begin = after[i].end;
+			update->reserved_ranges[update->reserved_ranges_count]
+				.end = before[i].end;
+			update->reserved_ranges_count++;
+		}
+	}
+
+	return true;
+}
+
+/*
+ * Loads alls VMs from the manifest.
+ */
+bool load_vms(struct mm_stage1_locked stage1_locked,
+	      const struct manifest *manifest, const struct memiter *cpio,
+	      const struct boot_params *params,
+	      struct boot_params_update *update, struct mpool *ppool)
+{
+	struct vm *primary;
+	struct mem_range mem_ranges_available[MAX_MEM_RANGES];
+	struct vm_locked primary_vm_locked;
+	size_t i;
+	bool success = true;
+
+	if (!load_primary(stage1_locked, &manifest->vm[HF_PRIMARY_VM_INDEX],
+			  cpio, params, ppool)) {
+		dlog("Unable to load primary VM.\n");
+		return false;
+	}
+
+	static_assert(
+		sizeof(mem_ranges_available) == sizeof(params->mem_ranges),
+		"mem_range arrays must be the same size for memcpy.");
+	static_assert(sizeof(mem_ranges_available) < 500,
+		      "This will use too much stack, either make "
+		      "MAX_MEM_RANGES smaller or change this.");
+	memcpy_s(mem_ranges_available, sizeof(mem_ranges_available),
+		 params->mem_ranges, sizeof(params->mem_ranges));
+
+	/* Round the last addresses down to the page size. */
+	for (i = 0; i < params->mem_ranges_count; ++i) {
+		mem_ranges_available[i].end = pa_init(align_down(
+			pa_addr(mem_ranges_available[i].end), PAGE_SIZE));
+	}
+
+	primary = vm_find(HF_PRIMARY_VM_ID);
+	primary_vm_locked = vm_lock(primary);
+
+	for (i = 0; i < manifest->vm_count; ++i) {
+		const struct manifest_vm *manifest_vm = &manifest->vm[i];
+		spci_vm_id_t vm_id = HF_VM_ID_OFFSET + i;
+		uint64_t mem_size;
+		paddr_t secondary_mem_begin;
+		paddr_t secondary_mem_end;
+
+		if (vm_id == HF_PRIMARY_VM_ID) {
+			continue;
+		}
+
+		dlog("Loading VM%d: %s.\n", (int)vm_id,
+		     manifest_vm->debug_name);
+
+		mem_size = align_up(manifest_vm->secondary.mem_size, PAGE_SIZE);
+		if (!carve_out_mem_range(mem_ranges_available,
+					 params->mem_ranges_count, mem_size,
+					 &secondary_mem_begin,
+					 &secondary_mem_end)) {
+			dlog("Not enough memory (%u bytes).\n", mem_size);
+			continue;
+		}
+
+		if (!load_secondary(stage1_locked, secondary_mem_begin,
+				    secondary_mem_end, manifest_vm, cpio,
+				    ppool)) {
+			dlog("Unable to load VM.\n");
+			continue;
+		}
+
+		/* Deny the primary VM access to this memory. */
+		if (!vm_unmap(primary_vm_locked, secondary_mem_begin,
+			      secondary_mem_end, ppool)) {
+			dlog("Unable to unmap secondary VM from primary VM.\n");
+			success = false;
+			break;
+		}
+	}
+
+	vm_unlock(&primary_vm_locked);
+
+	if (!success) {
+		return false;
+	}
+
+	/*
+	 * Add newly reserved areas to update params by looking at the
+	 * difference between the available ranges from the original params and
+	 * the updated mem_ranges_available. We assume that the number and order
+	 * of available ranges is the same, i.e. we don't remove any ranges
+	 * above only make them smaller.
+	 */
+	return update_reserved_ranges(update, params->mem_ranges,
+				      mem_ranges_available,
+				      params->mem_ranges_count);
+}
diff --git a/src/main.c b/src/main.c
new file mode 100644
index 0000000..b12032e
--- /dev/null
+++ b/src/main.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/cpu.h"
+#include "hf/vm.h"
+
+/**
+ * The entry point of CPUs when they are turned on. It is supposed to initialise
+ * all state and return the first vCPU to run.
+ */
+struct vcpu *cpu_main(struct cpu *c)
+{
+	struct vcpu *vcpu;
+
+	vcpu = vm_get_vcpu(vm_find(HF_PRIMARY_VM_ID), cpu_index(c));
+	vcpu->cpu = c;
+
+	arch_cpu_init();
+
+	/* Reset the registers to give a clean start for the primary's vCPU. */
+	arch_regs_reset(vcpu);
+
+	return vcpu;
+}
diff --git a/src/manifest.c b/src/manifest.c
new file mode 100644
index 0000000..c3b5566
--- /dev/null
+++ b/src/manifest.c
@@ -0,0 +1,405 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/manifest.h"
+
+#include "hf/addr.h"
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/fdt.h"
+#include "hf/static_assert.h"
+#include "hf/std.h"
+
+#define TRY(expr)                                            \
+	do {                                                 \
+		enum manifest_return_code ret_code = (expr); \
+		if (ret_code != MANIFEST_SUCCESS) {          \
+			return ret_code;                     \
+		}                                            \
+	} while (0)
+
+#define VM_NAME_BUF_SIZE (2 + 5 + 1) /* "vm" + number + null terminator */
+static_assert(MAX_VMS <= 99999, "Insufficient VM_NAME_BUF_SIZE");
+
+/**
+ * Generates a string with the two letters "vm" followed by an integer.
+ * Assumes `buf` is of size VM_NAME_BUF_SIZE.
+ */
+static const char *generate_vm_node_name(char *buf, spci_vm_id_t vm_id)
+{
+	static const char *digits = "0123456789";
+	char *ptr = buf + VM_NAME_BUF_SIZE;
+
+	*(--ptr) = '\0';
+	do {
+		*(--ptr) = digits[vm_id % 10];
+		vm_id /= 10;
+	} while (vm_id);
+	*(--ptr) = 'm';
+	*(--ptr) = 'v';
+
+	return ptr;
+}
+
+/**
+ * Read a boolean property: true if present; false if not. If present, the value
+ * of the property must be empty else it is considered malformed.
+ */
+static enum manifest_return_code read_bool(const struct fdt_node *node,
+					   const char *property, bool *out)
+{
+	const char *data;
+	uint32_t size;
+	bool present = fdt_read_property(node, property, &data, &size);
+
+	if (present && size != 0) {
+		return MANIFEST_ERROR_MALFORMED_BOOLEAN;
+	}
+
+	*out = present;
+	return MANIFEST_SUCCESS;
+}
+
+static enum manifest_return_code read_string(const struct fdt_node *node,
+					     const char *property,
+					     struct string *out)
+{
+	const char *data;
+	uint32_t size;
+
+	if (!fdt_read_property(node, property, &data, &size)) {
+		return MANIFEST_ERROR_PROPERTY_NOT_FOUND;
+	}
+
+	switch (string_init(out, data, size)) {
+	case STRING_SUCCESS:
+		return MANIFEST_SUCCESS;
+	case STRING_ERROR_INVALID_INPUT:
+		return MANIFEST_ERROR_MALFORMED_STRING;
+	case STRING_ERROR_TOO_LONG:
+		return MANIFEST_ERROR_STRING_TOO_LONG;
+	}
+}
+
+static enum manifest_return_code read_optional_string(
+	const struct fdt_node *node, const char *property, struct string *out)
+{
+	enum manifest_return_code ret;
+
+	ret = read_string(node, property, out);
+	if (ret == MANIFEST_ERROR_PROPERTY_NOT_FOUND) {
+		string_init_empty(out);
+		ret = MANIFEST_SUCCESS;
+	}
+	return ret;
+}
+
+static enum manifest_return_code read_uint64(const struct fdt_node *node,
+					     const char *property,
+					     uint64_t *out)
+{
+	const char *data;
+	uint32_t size;
+
+	if (!fdt_read_property(node, property, &data, &size)) {
+		return MANIFEST_ERROR_PROPERTY_NOT_FOUND;
+	}
+
+	if (!fdt_parse_number(data, size, out)) {
+		return MANIFEST_ERROR_MALFORMED_INTEGER;
+	}
+
+	return MANIFEST_SUCCESS;
+}
+
+static enum manifest_return_code read_uint16(const struct fdt_node *node,
+					     const char *property,
+					     uint16_t *out)
+{
+	uint64_t value;
+
+	TRY(read_uint64(node, property, &value));
+
+	if (value > UINT16_MAX) {
+		return MANIFEST_ERROR_INTEGER_OVERFLOW;
+	}
+
+	*out = (uint16_t)value;
+	return MANIFEST_SUCCESS;
+}
+
+struct uint32list_iter {
+	struct memiter mem_it;
+};
+
+static enum manifest_return_code read_optional_uint32list(
+	const struct fdt_node *node, const char *property,
+	struct uint32list_iter *out)
+{
+	const char *data;
+	uint32_t size;
+
+	if (!fdt_read_property(node, property, &data, &size)) {
+		memiter_init(&out->mem_it, NULL, 0);
+		return MANIFEST_SUCCESS;
+	}
+
+	if ((size % sizeof(uint32_t)) != 0) {
+		return MANIFEST_ERROR_MALFORMED_INTEGER_LIST;
+	}
+
+	memiter_init(&out->mem_it, data, size);
+	return MANIFEST_SUCCESS;
+}
+
+/**
+ * Represents the value of property whose type is a list of strings. These are
+ * encoded as one contiguous byte buffer with NULL-separated entries.
+ */
+struct stringlist_iter {
+	struct memiter mem_it;
+};
+
+static enum manifest_return_code read_stringlist(const struct fdt_node *node,
+						 const char *property,
+						 struct stringlist_iter *out)
+{
+	const char *data;
+	uint32_t size;
+
+	if (!fdt_read_property(node, property, &data, &size)) {
+		return MANIFEST_ERROR_PROPERTY_NOT_FOUND;
+	}
+
+	/*
+	 * Require that the value ends with a NULL terminator. Other NULL
+	 * characters separate the string list entries.
+	 */
+	if (data[size - 1] != '\0') {
+		return MANIFEST_ERROR_MALFORMED_STRING_LIST;
+	}
+
+	memiter_init(&out->mem_it, data, size - 1);
+	return MANIFEST_SUCCESS;
+}
+
+static bool uint32list_has_next(const struct uint32list_iter *list)
+{
+	return memiter_size(&list->mem_it) > 0;
+}
+
+static uint32_t uint32list_get_next(struct uint32list_iter *list)
+{
+	const char *mem_base = memiter_base(&list->mem_it);
+	uint64_t num;
+
+	CHECK(uint32list_has_next(list));
+
+	if (!fdt_parse_number(mem_base, sizeof(uint32_t), &num)) {
+		return MANIFEST_ERROR_MALFORMED_INTEGER;
+	}
+
+	memiter_advance(&list->mem_it, sizeof(uint32_t));
+	return num;
+}
+
+static bool stringlist_has_next(const struct stringlist_iter *list)
+{
+	return memiter_size(&list->mem_it) > 0;
+}
+
+static void stringlist_get_next(struct stringlist_iter *list,
+				struct memiter *out)
+{
+	const char *mem_base = memiter_base(&list->mem_it);
+	size_t mem_size = memiter_size(&list->mem_it);
+	const char *null_term;
+
+	CHECK(stringlist_has_next(list));
+
+	null_term = memchr(mem_base, '\0', mem_size);
+	if (null_term == NULL) {
+		/*
+		 * NULL terminator not found, this is the last entry.
+		 * Set entry memiter to the entire byte range and advance list
+		 * memiter to the end of the byte range.
+		 */
+		memiter_init(out, mem_base, mem_size);
+		memiter_advance(&list->mem_it, mem_size);
+	} else {
+		/*
+		 * Found NULL terminator. Set entry memiter to byte range
+		 * [base, null) and move list memiter past the terminator.
+		 */
+		size_t entry_size = null_term - mem_base;
+
+		memiter_init(out, mem_base, entry_size);
+		memiter_advance(&list->mem_it, entry_size + 1);
+	}
+}
+
+static bool stringlist_contains(const struct stringlist_iter *list,
+				const char *str)
+{
+	struct stringlist_iter it = *list;
+	struct memiter entry;
+
+	while (stringlist_has_next(&it)) {
+		stringlist_get_next(&it, &entry);
+		if (memiter_iseq(&entry, str)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static enum manifest_return_code parse_vm(struct fdt_node *node,
+					  struct manifest_vm *vm,
+					  spci_vm_id_t vm_id)
+{
+	struct uint32list_iter smcs;
+
+	TRY(read_string(node, "debug_name", &vm->debug_name));
+	TRY(read_optional_string(node, "kernel_filename",
+				 &vm->kernel_filename));
+
+	TRY(read_optional_uint32list(node, "smc_whitelist", &smcs));
+	while (uint32list_has_next(&smcs) &&
+	       vm->smc_whitelist.smc_count < MAX_SMCS) {
+		vm->smc_whitelist.smcs[vm->smc_whitelist.smc_count++] =
+			uint32list_get_next(&smcs);
+	}
+
+	if (uint32list_has_next(&smcs)) {
+		dlog("%s SMC whitelist too long.\n", vm->debug_name);
+	}
+
+	TRY(read_bool(node, "smc_whitelist_permissive",
+		      &vm->smc_whitelist.permissive));
+
+	if (vm_id == HF_PRIMARY_VM_ID) {
+		TRY(read_optional_string(node, "ramdisk_filename",
+					 &vm->primary.ramdisk_filename));
+	} else {
+		TRY(read_uint64(node, "mem_size", &vm->secondary.mem_size));
+		TRY(read_uint16(node, "vcpu_count", &vm->secondary.vcpu_count));
+	}
+	return MANIFEST_SUCCESS;
+}
+
+/**
+ * Parse manifest from FDT.
+ */
+enum manifest_return_code manifest_init(struct manifest *manifest,
+					const struct fdt_node *fdt_root)
+{
+	char vm_name_buf[VM_NAME_BUF_SIZE];
+	struct fdt_node hyp_node;
+	struct stringlist_iter compatible_list;
+	size_t i = 0;
+	bool found_primary_vm = false;
+
+	memset_s(manifest, sizeof(*manifest), 0, sizeof(*manifest));
+
+	/* Find hypervisor node. */
+	hyp_node = *fdt_root;
+	if (!fdt_find_child(&hyp_node, "hypervisor")) {
+		return MANIFEST_ERROR_NO_HYPERVISOR_FDT_NODE;
+	}
+
+	/* Check "compatible" property. */
+	TRY(read_stringlist(&hyp_node, "compatible", &compatible_list));
+	if (!stringlist_contains(&compatible_list, "hafnium,hafnium")) {
+		return MANIFEST_ERROR_NOT_COMPATIBLE;
+	}
+
+	/* Iterate over reserved VM IDs and check no such nodes exist. */
+	for (i = 0; i < HF_VM_ID_OFFSET; i++) {
+		spci_vm_id_t vm_id = (spci_vm_id_t)i;
+		struct fdt_node vm_node = hyp_node;
+		const char *vm_name = generate_vm_node_name(vm_name_buf, vm_id);
+
+		if (fdt_find_child(&vm_node, vm_name)) {
+			return MANIFEST_ERROR_RESERVED_VM_ID;
+		}
+	}
+
+	/* Iterate over VM nodes until we find one that does not exist. */
+	for (i = 0; i <= MAX_VMS; ++i) {
+		spci_vm_id_t vm_id = HF_VM_ID_OFFSET + i;
+		struct fdt_node vm_node = hyp_node;
+		const char *vm_name = generate_vm_node_name(vm_name_buf, vm_id);
+
+		if (!fdt_find_child(&vm_node, vm_name)) {
+			break;
+		}
+
+		if (i == MAX_VMS) {
+			return MANIFEST_ERROR_TOO_MANY_VMS;
+		}
+
+		if (vm_id == HF_PRIMARY_VM_ID) {
+			CHECK(found_primary_vm == false); /* sanity check */
+			found_primary_vm = true;
+		}
+
+		manifest->vm_count = i + 1;
+		TRY(parse_vm(&vm_node, &manifest->vm[i], vm_id));
+	}
+
+	if (!found_primary_vm) {
+		return MANIFEST_ERROR_NO_PRIMARY_VM;
+	}
+
+	return MANIFEST_SUCCESS;
+}
+
+const char *manifest_strerror(enum manifest_return_code ret_code)
+{
+	switch (ret_code) {
+	case MANIFEST_SUCCESS:
+		return "Success";
+	case MANIFEST_ERROR_NO_HYPERVISOR_FDT_NODE:
+		return "Could not find \"hypervisor\" node in manifest";
+	case MANIFEST_ERROR_NOT_COMPATIBLE:
+		return "Hypervisor manifest entry not compatible with Hafnium";
+	case MANIFEST_ERROR_RESERVED_VM_ID:
+		return "Manifest defines a VM with a reserved ID";
+	case MANIFEST_ERROR_NO_PRIMARY_VM:
+		return "Manifest does not contain a primary VM entry";
+	case MANIFEST_ERROR_TOO_MANY_VMS:
+		return "Manifest specifies more VMs than Hafnium has "
+		       "statically allocated space for";
+	case MANIFEST_ERROR_PROPERTY_NOT_FOUND:
+		return "Property not found";
+	case MANIFEST_ERROR_MALFORMED_STRING:
+		return "Malformed string property";
+	case MANIFEST_ERROR_STRING_TOO_LONG:
+		return "String too long";
+	case MANIFEST_ERROR_MALFORMED_STRING_LIST:
+		return "Malformed string list property";
+	case MANIFEST_ERROR_MALFORMED_INTEGER:
+		return "Malformed integer property";
+	case MANIFEST_ERROR_INTEGER_OVERFLOW:
+		return "Integer overflow";
+	case MANIFEST_ERROR_MALFORMED_INTEGER_LIST:
+		return "Malformed integer list property";
+	case MANIFEST_ERROR_MALFORMED_BOOLEAN:
+		return "Malformed boolean property";
+	}
+
+	panic("Unexpected manifest return code.");
+}
diff --git a/src/manifest_test.cc b/src/manifest_test.cc
new file mode 100644
index 0000000..fdfd7f5
--- /dev/null
+++ b/src/manifest_test.cc
@@ -0,0 +1,589 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <array>
+#include <cstdio>
+#include <span>
+#include <sstream>
+
+#include <gmock/gmock.h>
+
+extern "C" {
+#include "hf/manifest.h"
+}
+
+namespace
+{
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::IsEmpty;
+using ::testing::NotNull;
+
+template <typename T>
+void exec(const char *program, const char *args[], const T &stdin,
+	  std::vector<char> *stdout)
+{
+	/* Create two pipes, one for stdin and one for stdout. */
+	int pipes[2][2];
+	pipe(pipes[0]);
+	pipe(pipes[1]);
+
+	/* Assign FDs for reading/writing by the parent/child. */
+	int parent_read_fd = pipes[1][0];  /* stdout pipe, read FD */
+	int parent_write_fd = pipes[0][1]; /* stdin pipe, write FD */
+	int child_read_fd = pipes[0][0];   /* stdin pipe, read FD */
+	int child_write_fd = pipes[1][1];  /* stdout pipe, write FD */
+
+	if (fork()) {
+		/* Parent process. */
+		std::array<char, 128> buf;
+		ssize_t res;
+
+		/* Close child FDs which won't be used. */
+		close(child_read_fd);
+		close(child_write_fd);
+
+		/* Write to stdin. */
+		for (size_t count = 0; count < stdin.size();) {
+			res = write(parent_write_fd, stdin.data() + count,
+				    stdin.size() - count);
+			if (res < 0) {
+				std::cerr << "IO error" << std::endl;
+				exit(1);
+			}
+			count += res;
+		}
+		close(parent_write_fd);
+
+		/* Read from stdout. */
+		while (true) {
+			res = read(parent_read_fd, buf.data(), buf.size());
+			if (res == 0) {
+				/* EOF */
+				break;
+			} else if (res < 0) {
+				std::cerr << "IO error" << std::endl;
+				exit(1);
+			}
+			stdout->insert(stdout->end(), buf.begin(),
+				       buf.begin() + res);
+		}
+		close(parent_read_fd);
+	} else {
+		/* Child process. */
+
+		/* Redirect stdin/stdout to read/write FDs. */
+		dup2(child_read_fd, STDIN_FILENO);
+		dup2(child_write_fd, STDOUT_FILENO);
+
+		/* Close all FDs which are now unused. */
+		close(child_read_fd);
+		close(child_write_fd);
+		close(parent_read_fd);
+		close(parent_write_fd);
+
+		/* Execute the given program. */
+		execv(program, const_cast<char *const *>(args));
+	}
+}
+
+/**
+ * Class for programatically building a Device Tree.
+ *
+ * Usage:
+ *   std::vector<char> dtb = ManifestDtBuilder()
+ *       .Command1()
+ *       .Command2()
+ *       ...
+ *       .CommandN()
+ *       .Build();
+ */
+class ManifestDtBuilder
+{
+       public:
+	ManifestDtBuilder()
+	{
+		dts_ << "/dts-v1/;" << std::endl;
+		dts_ << std::endl;
+
+		/* Start root node. */
+		StartChild("/");
+	}
+
+	std::vector<char> Build(bool dump = false)
+	{
+		const char *program = "./build/image/dtc.py";
+		const char *dtc_args[] = {program, "compile", NULL};
+		std::vector<char> dtc_stdout;
+
+		/* Finish root node. */
+		EndChild();
+
+		if (dump) {
+			Dump();
+		}
+
+		exec(program, dtc_args, dts_.str(), &dtc_stdout);
+		return dtc_stdout;
+	}
+
+	void Dump()
+	{
+		std::cerr << dts_.str() << std::endl;
+	}
+
+	ManifestDtBuilder &StartChild(const std::string_view &name)
+	{
+		dts_ << name << " {" << std::endl;
+		return *this;
+	}
+
+	ManifestDtBuilder &EndChild()
+	{
+		dts_ << "};" << std::endl;
+		return *this;
+	}
+
+	ManifestDtBuilder &Compatible(const std::vector<std::string_view>
+					      &value = {"hafnium,hafnium"})
+	{
+		return StringListProperty("compatible", value);
+	}
+
+	ManifestDtBuilder &DebugName(const std::string_view &value)
+	{
+		return StringProperty("debug_name", value);
+	}
+
+	ManifestDtBuilder &KernelFilename(const std::string_view &value)
+	{
+		return StringProperty("kernel_filename", value);
+	}
+
+	ManifestDtBuilder &RamdiskFilename(const std::string_view &value)
+	{
+		return StringProperty("ramdisk_filename", value);
+	}
+
+	ManifestDtBuilder &VcpuCount(uint32_t value)
+	{
+		return IntegerProperty("vcpu_count", value);
+	}
+
+	ManifestDtBuilder &MemSize(uint32_t value)
+	{
+		return IntegerProperty("mem_size", value);
+	}
+
+	ManifestDtBuilder &SmcWhitelist(const std::vector<uint32_t> &value)
+	{
+		return IntegerListProperty("smc_whitelist", value);
+	}
+
+	ManifestDtBuilder &SmcWhitelistPermissive()
+	{
+		return BooleanProperty("smc_whitelist_permissive");
+	}
+
+	ManifestDtBuilder &Property(const std::string_view &name,
+				    const std::string_view &value)
+	{
+		dts_ << name << " = " << value << ";" << std::endl;
+		return *this;
+	}
+
+       private:
+	ManifestDtBuilder &StringProperty(const std::string_view &name,
+					  const std::string_view &value)
+	{
+		dts_ << name << " = \"" << value << "\";" << std::endl;
+		return *this;
+	}
+
+	ManifestDtBuilder &StringListProperty(
+		const std::string_view &name,
+		const std::vector<std::string_view> &value)
+	{
+		bool is_first = true;
+
+		dts_ << name << " = ";
+		for (const std::string_view &entry : value) {
+			if (is_first) {
+				is_first = false;
+			} else {
+				dts_ << ", ";
+			}
+			dts_ << "\"" << entry << "\"";
+		}
+		dts_ << ";" << std::endl;
+		return *this;
+	}
+
+	ManifestDtBuilder &IntegerProperty(const std::string_view &name,
+					   uint32_t value)
+	{
+		dts_ << name << " = <" << value << ">;" << std::endl;
+		return *this;
+	}
+
+	ManifestDtBuilder &IntegerListProperty(
+		const std::string_view &name,
+		const std::vector<uint32_t> &value)
+	{
+		dts_ << name << " = < ";
+		for (const uint32_t entry : value) {
+			dts_ << entry << " ";
+		}
+		dts_ << ">;" << std::endl;
+		return *this;
+	}
+
+	ManifestDtBuilder &BooleanProperty(const std::string_view &name)
+	{
+		dts_ << name << ";" << std::endl;
+		return *this;
+	}
+
+	std::stringstream dts_;
+};
+
+static bool get_fdt_root(const std::vector<char> &dtb,
+			 struct fdt_node *fdt_root)
+{
+	const struct fdt_header *fdt_header;
+
+	fdt_header = reinterpret_cast<const struct fdt_header *>(dtb.data());
+	return fdt_root_node(fdt_root, fdt_header) &&
+	       fdt_find_child(fdt_root, "");
+}
+
+TEST(manifest, no_hypervisor_node)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+	std::vector<char> dtb = ManifestDtBuilder().Build();
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root),
+		  MANIFEST_ERROR_NO_HYPERVISOR_FDT_NODE);
+}
+
+TEST(manifest, no_compatible_property)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	/* clang-format off */
+	std::vector<char> dtb = ManifestDtBuilder()
+		.StartChild("hypervisor")
+		.EndChild()
+		.Build();
+	/* clang-format on */
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root),
+		  MANIFEST_ERROR_PROPERTY_NOT_FOUND);
+}
+
+TEST(manifest, not_compatible)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	/* clang-format off */
+	std::vector<char> dtb = ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible({ "foo,bar" })
+		.EndChild()
+		.Build();
+	/* clang-format on */
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_ERROR_NOT_COMPATIBLE);
+}
+
+TEST(manifest, compatible_one_of_many)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	/* clang-format off */
+	std::vector<char> dtb = ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible({ "foo,bar", "hafnium,hafnium" })
+			.StartChild("vm1")
+				.DebugName("primary")
+			.EndChild()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_SUCCESS);
+}
+
+TEST(manifest, no_vm_nodes)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	/* clang-format off */
+	std::vector<char> dtb = ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_ERROR_NO_PRIMARY_VM);
+}
+
+static std::vector<char> gen_long_string_dtb(bool valid)
+{
+	const char last_valid[] = "1234567890123456789012345678901";
+	const char first_invalid[] = "12345678901234567890123456789012";
+	static_assert(sizeof(last_valid) == STRING_MAX_SIZE);
+	static_assert(sizeof(first_invalid) == STRING_MAX_SIZE + 1);
+
+	/* clang-format off */
+	return ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible()
+			.StartChild("vm1")
+				.DebugName(valid ? last_valid : first_invalid)
+			.EndChild()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+}
+
+TEST(manifest, long_string)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	std::vector<char> dtb_last_valid = gen_long_string_dtb(true);
+	std::vector<char> dtb_first_invalid = gen_long_string_dtb(false);
+
+	ASSERT_TRUE(get_fdt_root(dtb_last_valid, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_SUCCESS);
+
+	ASSERT_TRUE(get_fdt_root(dtb_first_invalid, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_ERROR_STRING_TOO_LONG);
+}
+
+TEST(manifest, reserved_vm_id)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	/* clang-format off */
+	std::vector<char> dtb = ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible()
+			.StartChild("vm1")
+				.DebugName("primary_vm")
+			.EndChild()
+			.StartChild("vm0")
+				.DebugName("reserved_vm")
+				.VcpuCount(1)
+				.MemSize(0x1000)
+				.KernelFilename("kernel")
+			.EndChild()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_ERROR_RESERVED_VM_ID);
+}
+
+static std::vector<char> gen_vcpu_count_limit_dtb(uint32_t vcpu_count)
+{
+	/* clang-format off */
+	return ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible()
+			.StartChild("vm1")
+				.DebugName("primary_vm")
+			.EndChild()
+			.StartChild("vm2")
+				.DebugName("secondary_vm")
+				.VcpuCount(vcpu_count)
+				.MemSize(0x1000)
+				.KernelFilename("kernel")
+			.EndChild()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+}
+
+TEST(manifest, vcpu_count_limit)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+	std::vector<char> dtb_last_valid = gen_vcpu_count_limit_dtb(UINT16_MAX);
+	std::vector<char> dtb_first_invalid =
+		gen_vcpu_count_limit_dtb(UINT16_MAX + 1);
+
+	ASSERT_TRUE(get_fdt_root(dtb_last_valid, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_SUCCESS);
+	ASSERT_EQ(m.vm_count, 2);
+	ASSERT_EQ(m.vm[1].secondary.vcpu_count, UINT16_MAX);
+
+	ASSERT_TRUE(get_fdt_root(dtb_first_invalid, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root),
+		  MANIFEST_ERROR_INTEGER_OVERFLOW);
+}
+
+TEST(manifest, no_ramdisk_primary)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	/* clang-format off */
+	std::vector<char> dtb = ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible()
+			.StartChild("vm1")
+				.DebugName("primary_vm")
+			.EndChild()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_SUCCESS);
+	ASSERT_EQ(m.vm_count, 1);
+	ASSERT_STREQ(string_data(&m.vm[0].debug_name), "primary_vm");
+	ASSERT_STREQ(string_data(&m.vm[0].primary.ramdisk_filename), "");
+}
+
+static std::vector<char> gen_malformed_boolean_dtb(
+	const std::string_view &value)
+{
+	/* clang-format off */
+	return  ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible()
+			.StartChild("vm1")
+				.DebugName("primary_vm")
+				.Property("smc_whitelist_permissive", value)
+			.EndChild()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+}
+
+TEST(manifest, malformed_booleans)
+{
+	struct manifest m;
+	struct fdt_node fdt_root;
+
+	std::vector<char> dtb_false = gen_malformed_boolean_dtb("\"false\"");
+	std::vector<char> dtb_true = gen_malformed_boolean_dtb("\"true\"");
+	std::vector<char> dtb_0 = gen_malformed_boolean_dtb("\"<0>\"");
+	std::vector<char> dtb_1 = gen_malformed_boolean_dtb("\"<1>\"");
+
+	ASSERT_TRUE(get_fdt_root(dtb_false, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root),
+		  MANIFEST_ERROR_MALFORMED_BOOLEAN);
+
+	ASSERT_TRUE(get_fdt_root(dtb_true, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root),
+		  MANIFEST_ERROR_MALFORMED_BOOLEAN);
+
+	ASSERT_TRUE(get_fdt_root(dtb_0, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root),
+		  MANIFEST_ERROR_MALFORMED_BOOLEAN);
+
+	ASSERT_TRUE(get_fdt_root(dtb_1, &fdt_root));
+	ASSERT_EQ(manifest_init(&m, &fdt_root),
+		  MANIFEST_ERROR_MALFORMED_BOOLEAN);
+}
+
+TEST(manifest, valid)
+{
+	struct manifest m;
+	struct manifest_vm *vm;
+	struct fdt_node fdt_root;
+
+	/* clang-format off */
+	std::vector<char> dtb = ManifestDtBuilder()
+		.StartChild("hypervisor")
+			.Compatible()
+			.StartChild("vm1")
+				.DebugName("primary_vm")
+				.KernelFilename("primary_kernel")
+				.RamdiskFilename("primary_ramdisk")
+				.SmcWhitelist({0x32000000, 0x33001111})
+			.EndChild()
+			.StartChild("vm3")
+				.DebugName("second_secondary_vm")
+				.VcpuCount(43)
+				.MemSize(0x12345)
+				.KernelFilename("second_secondary_kernel")
+			.EndChild()
+			.StartChild("vm2")
+				.DebugName("first_secondary_vm")
+				.VcpuCount(42)
+				.MemSize(12345)
+				.SmcWhitelist({0x04000000, 0x30002222, 0x31445566})
+				.SmcWhitelistPermissive()
+			.EndChild()
+		.EndChild()
+		.Build();
+	/* clang-format on */
+
+	ASSERT_TRUE(get_fdt_root(dtb, &fdt_root));
+
+	ASSERT_EQ(manifest_init(&m, &fdt_root), MANIFEST_SUCCESS);
+	ASSERT_EQ(m.vm_count, 3);
+
+	vm = &m.vm[0];
+	ASSERT_STREQ(string_data(&vm->debug_name), "primary_vm");
+	ASSERT_STREQ(string_data(&vm->kernel_filename), "primary_kernel");
+	ASSERT_STREQ(string_data(&vm->primary.ramdisk_filename),
+		     "primary_ramdisk");
+	ASSERT_THAT(
+		std::span(vm->smc_whitelist.smcs, vm->smc_whitelist.smc_count),
+		ElementsAre(0x32000000, 0x33001111));
+	ASSERT_FALSE(vm->smc_whitelist.permissive);
+
+	vm = &m.vm[1];
+	ASSERT_STREQ(string_data(&vm->debug_name), "first_secondary_vm");
+	ASSERT_STREQ(string_data(&vm->kernel_filename), "");
+	ASSERT_EQ(vm->secondary.vcpu_count, 42);
+	ASSERT_EQ(vm->secondary.mem_size, 12345);
+	ASSERT_THAT(
+		std::span(vm->smc_whitelist.smcs, vm->smc_whitelist.smc_count),
+		ElementsAre(0x04000000, 0x30002222, 0x31445566));
+	ASSERT_TRUE(vm->smc_whitelist.permissive);
+
+	vm = &m.vm[2];
+	ASSERT_STREQ(string_data(&vm->debug_name), "second_secondary_vm");
+	ASSERT_STREQ(string_data(&vm->kernel_filename),
+		     "second_secondary_kernel");
+	ASSERT_EQ(vm->secondary.vcpu_count, 43);
+	ASSERT_EQ(vm->secondary.mem_size, 0x12345);
+	ASSERT_THAT(
+		std::span(vm->smc_whitelist.smcs, vm->smc_whitelist.smc_count),
+		IsEmpty());
+	ASSERT_FALSE(vm->smc_whitelist.permissive);
+}
+
+} /* namespace */
diff --git a/src/memiter.c b/src/memiter.c
new file mode 100644
index 0000000..d72250c
--- /dev/null
+++ b/src/memiter.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/memiter.h"
+
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+/**
+ * Initialises the given memory iterator.
+ */
+void memiter_init(struct memiter *it, const void *data, size_t size)
+{
+	it->next = data;
+	it->limit = it->next + size;
+}
+
+/**
+ * Determines if the next character is a whitespace.
+ */
+static bool memiter_isspace(struct memiter *it)
+{
+	switch (*it->next) {
+	case ' ':
+	case '\t':
+	case '\n':
+	case '\r':
+		return true;
+	default:
+		return false;
+	}
+}
+
+/**
+ * Moves iterator to the next non-whitespace character.
+ */
+static void memiter_skip_space(struct memiter *it)
+{
+	while (it->next < it->limit && memiter_isspace(it)) {
+		it->next++;
+	}
+}
+
+/**
+ * Compares the iterator to a null-terminated string.
+ */
+bool memiter_iseq(const struct memiter *it, const char *str)
+{
+	size_t it_len = it->limit - it->next;
+	size_t len = strnlen_s(str, it_len + 1);
+
+	if (len != it_len) {
+		return false;
+	}
+
+	return memcmp(it->next, str, len) == 0;
+}
+
+/**
+ * Retrieves the next string that is delimited by whitespaces. The result is
+ * stored in "str".
+ */
+bool memiter_parse_str(struct memiter *it, struct memiter *str)
+{
+	/* Skip all white space and fail if we reach the end of the buffer. */
+	memiter_skip_space(it);
+	if (it->next >= it->limit) {
+		return false;
+	}
+
+	str->next = it->next;
+
+	/* Find the end of the string. */
+	while (it->next < it->limit && !memiter_isspace(it)) {
+		it->next++;
+	}
+
+	str->limit = it->next;
+
+	return true;
+}
+
+/**
+ * Parses the next string that represents a 64-bit number.
+ */
+bool memiter_parse_uint(struct memiter *it, uint64_t *value)
+{
+	uint64_t v = 0;
+
+	/* Skip all white space and fail if we reach the end of the buffer. */
+	memiter_skip_space(it);
+	if (it->next >= it->limit) {
+		return false;
+	}
+
+	/* Fail if it's not a number. */
+	if (*it->next < '0' || *it->next > '9') {
+		return false;
+	}
+
+	/* Parse the number. */
+	do {
+		v = v * 10 + *it->next - '0';
+		it->next++;
+	} while (it->next < it->limit && *it->next >= '0' && *it->next <= '9');
+
+	*value = v;
+
+	return true;
+}
+
+/**
+ * Advances the iterator by the given number of bytes. Returns true if the
+ * iterator was advanced without going over its limit; returns false and leaves
+ * the iterator unmodified otherwise.
+ */
+bool memiter_advance(struct memiter *it, size_t v)
+{
+	const char *p = it->next + v;
+
+	if (p < it->next || p > it->limit) {
+		return false;
+	}
+
+	it->next = p;
+
+	return true;
+}
+
+const void *memiter_base(const struct memiter *it)
+{
+	return (const void *)it->next;
+}
+
+/**
+ * Returns the number of bytes in interval [it.next, it.limit).
+ */
+size_t memiter_size(const struct memiter *it)
+{
+	return it->limit - it->next;
+}
diff --git a/src/mm.c b/src/mm.c
new file mode 100644
index 0000000..83bf9bd
--- /dev/null
+++ b/src/mm.c
@@ -0,0 +1,1053 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/mm.h"
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/layout.h"
+#include "hf/plat/console.h"
+#include "hf/static_assert.h"
+
+/**
+ * This file has functions for managing the level 1 and 2 page tables used by
+ * Hafnium. There is a level 1 mapping used by Hafnium itself to access memory,
+ * and then a level 2 mapping per VM. The design assumes that all page tables
+ * contain only 1-1 mappings, aligned on the block boundaries.
+ */
+
+/*
+ * For stage 2, the input is an intermediate physical addresses rather than a
+ * virtual address so:
+ */
+static_assert(
+	sizeof(ptable_addr_t) == sizeof(uintpaddr_t),
+	"Currently, the same code manages the stage 1 and stage 2 page tables "
+	"which only works if the virtual and intermediate physical addresses "
+	"are the same size. It looks like that assumption might not be holding "
+	"so we need to check that everything is going to be ok.");
+
+static struct mm_ptable ptable;
+static struct spinlock ptable_lock;
+
+static bool mm_stage2_invalidate = false;
+
+/**
+ * After calling this function, modifications to stage-2 page tables will use
+ * break-before-make and invalidate the TLB for the affected range.
+ */
+void mm_vm_enable_invalidation(void)
+{
+	mm_stage2_invalidate = true;
+}
+
+/**
+ * Get the page table from the physical address.
+ */
+static struct mm_page_table *mm_page_table_from_pa(paddr_t pa)
+{
+	return ptr_from_va(va_from_pa(pa));
+}
+
+/**
+ * Rounds an address down to a page boundary.
+ */
+static ptable_addr_t mm_round_down_to_page(ptable_addr_t addr)
+{
+	return addr & ~((ptable_addr_t)(PAGE_SIZE - 1));
+}
+
+/**
+ * Rounds an address up to a page boundary.
+ */
+static ptable_addr_t mm_round_up_to_page(ptable_addr_t addr)
+{
+	return mm_round_down_to_page(addr + PAGE_SIZE - 1);
+}
+
+/**
+ * Calculates the size of the address space represented by a page table entry at
+ * the given level.
+ */
+static size_t mm_entry_size(uint8_t level)
+{
+	return UINT64_C(1) << (PAGE_BITS + level * PAGE_LEVEL_BITS);
+}
+
+/**
+ * Gets the address of the start of the next block of the given size. The size
+ * must be a power of two.
+ */
+static ptable_addr_t mm_start_of_next_block(ptable_addr_t addr,
+					    size_t block_size)
+{
+	return (addr + block_size) & ~(block_size - 1);
+}
+
+/**
+ * Gets the physical address of the start of the next block of the given size.
+ * The size must be a power of two.
+ */
+static paddr_t mm_pa_start_of_next_block(paddr_t pa, size_t block_size)
+{
+	return pa_init((pa_addr(pa) + block_size) & ~(block_size - 1));
+}
+
+/**
+ * For a given address, calculates the maximum (plus one) address that can be
+ * represented by the same table at the given level.
+ */
+static ptable_addr_t mm_level_end(ptable_addr_t addr, uint8_t level)
+{
+	size_t offset = PAGE_BITS + (level + 1) * PAGE_LEVEL_BITS;
+
+	return ((addr >> offset) + 1) << offset;
+}
+
+/**
+ * For a given address, calculates the index at which its entry is stored in a
+ * table at the given level.
+ */
+static size_t mm_index(ptable_addr_t addr, uint8_t level)
+{
+	ptable_addr_t v = addr >> (PAGE_BITS + level * PAGE_LEVEL_BITS);
+
+	return v & ((UINT64_C(1) << PAGE_LEVEL_BITS) - 1);
+}
+
+/**
+ * Allocates a new page table.
+ */
+static struct mm_page_table *mm_alloc_page_tables(size_t count,
+						  struct mpool *ppool)
+{
+	if (count == 1) {
+		return mpool_alloc(ppool);
+	}
+
+	return mpool_alloc_contiguous(ppool, count, count);
+}
+
+/**
+ * Returns the maximum level in the page table given the flags.
+ */
+static uint8_t mm_max_level(int flags)
+{
+	return (flags & MM_FLAG_STAGE1) ? arch_mm_stage1_max_level()
+					: arch_mm_stage2_max_level();
+}
+
+/**
+ * Returns the number of root-level tables given the flags.
+ */
+static uint8_t mm_root_table_count(int flags)
+{
+	return (flags & MM_FLAG_STAGE1) ? arch_mm_stage1_root_table_count()
+					: arch_mm_stage2_root_table_count();
+}
+
+/**
+ * Invalidates the TLB for the given address range.
+ */
+static void mm_invalidate_tlb(ptable_addr_t begin, ptable_addr_t end, int flags)
+{
+	if (flags & MM_FLAG_STAGE1) {
+		arch_mm_invalidate_stage1_range(va_init(begin), va_init(end));
+	} else {
+		arch_mm_invalidate_stage2_range(ipa_init(begin), ipa_init(end));
+	}
+}
+
+/**
+ * Frees all page-table-related memory associated with the given pte at the
+ * given level, including any subtables recursively.
+ */
+static void mm_free_page_pte(pte_t pte, uint8_t level, struct mpool *ppool)
+{
+	struct mm_page_table *table;
+	uint64_t i;
+
+	if (!arch_mm_pte_is_table(pte, level)) {
+		return;
+	}
+
+	/* Recursively free any subtables. */
+	table = mm_page_table_from_pa(arch_mm_table_from_pte(pte, level));
+	for (i = 0; i < MM_PTE_PER_PAGE; ++i) {
+		mm_free_page_pte(table->entries[i], level - 1, ppool);
+	}
+
+	/* Free the table itself. */
+	mpool_free(ppool, table);
+}
+
+/**
+ * Returns the first address which cannot be encoded in page tables given by
+ * `flags`. It is the exclusive end of the address space created by the tables.
+ */
+ptable_addr_t mm_ptable_addr_space_end(int flags)
+{
+	return mm_root_table_count(flags) *
+	       mm_entry_size(mm_max_level(flags) + 1);
+}
+
+/**
+ * Initialises the given page table.
+ */
+bool mm_ptable_init(struct mm_ptable *t, int flags, struct mpool *ppool)
+{
+	uint8_t i;
+	size_t j;
+	struct mm_page_table *tables;
+	uint8_t root_table_count = mm_root_table_count(flags);
+
+	tables = mm_alloc_page_tables(root_table_count, ppool);
+	if (tables == NULL) {
+		return false;
+	}
+
+	for (i = 0; i < root_table_count; i++) {
+		for (j = 0; j < MM_PTE_PER_PAGE; j++) {
+			tables[i].entries[j] =
+				arch_mm_absent_pte(mm_max_level(flags));
+		}
+	}
+
+	/*
+	 * TODO: halloc could return a virtual or physical address if mm not
+	 * enabled?
+	 */
+	t->root = pa_init((uintpaddr_t)tables);
+
+	return true;
+}
+
+/**
+ * Frees all memory associated with the give page table.
+ */
+static void mm_ptable_fini(struct mm_ptable *t, int flags, struct mpool *ppool)
+{
+	struct mm_page_table *tables = mm_page_table_from_pa(t->root);
+	uint8_t level = mm_max_level(flags);
+	uint8_t root_table_count = mm_root_table_count(flags);
+	uint8_t i;
+	uint64_t j;
+
+	for (i = 0; i < root_table_count; ++i) {
+		for (j = 0; j < MM_PTE_PER_PAGE; ++j) {
+			mm_free_page_pte(tables[i].entries[j], level, ppool);
+		}
+	}
+
+	mpool_add_chunk(ppool, tables,
+			sizeof(struct mm_page_table) * root_table_count);
+}
+
+/**
+ * Replaces a page table entry with the given value. If both old and new values
+ * are valid, it performs a break-before-make sequence where it first writes an
+ * invalid value to the PTE, flushes the TLB, then writes the actual new value.
+ * This is to prevent cases where CPUs have different 'valid' values in their
+ * TLBs, which may result in issues for example in cache coherency.
+ */
+static void mm_replace_entry(ptable_addr_t begin, pte_t *pte, pte_t new_pte,
+			     uint8_t level, int flags, struct mpool *ppool)
+{
+	pte_t v = *pte;
+
+	/*
+	 * We need to do the break-before-make sequence if both values are
+	 * present and the TLB is being invalidated.
+	 */
+	if (((flags & MM_FLAG_STAGE1) || mm_stage2_invalidate) &&
+	    arch_mm_pte_is_valid(v, level) &&
+	    arch_mm_pte_is_valid(new_pte, level)) {
+		*pte = arch_mm_absent_pte(level);
+		mm_invalidate_tlb(begin, begin + mm_entry_size(level), flags);
+	}
+
+	/* Assign the new pte. */
+	*pte = new_pte;
+
+	/* Free pages that aren't in use anymore. */
+	mm_free_page_pte(v, level, ppool);
+}
+
+/**
+ * Populates the provided page table entry with a reference to another table if
+ * needed, that is, if it does not yet point to another table.
+ *
+ * Returns a pointer to the table the entry now points to.
+ */
+static struct mm_page_table *mm_populate_table_pte(ptable_addr_t begin,
+						   pte_t *pte, uint8_t level,
+						   int flags,
+						   struct mpool *ppool)
+{
+	struct mm_page_table *ntable;
+	pte_t v = *pte;
+	pte_t new_pte;
+	size_t i;
+	size_t inc;
+	uint8_t level_below = level - 1;
+
+	/* Just return pointer to table if it's already populated. */
+	if (arch_mm_pte_is_table(v, level)) {
+		return mm_page_table_from_pa(arch_mm_table_from_pte(v, level));
+	}
+
+	/* Allocate a new table. */
+	ntable = mm_alloc_page_tables(1, ppool);
+	if (ntable == NULL) {
+		dlog("Failed to allocate memory for page table\n");
+		return NULL;
+	}
+
+	/* Determine template for new pte and its increment. */
+	if (arch_mm_pte_is_block(v, level)) {
+		inc = mm_entry_size(level_below);
+		new_pte = arch_mm_block_pte(level_below,
+					    arch_mm_block_from_pte(v, level),
+					    arch_mm_pte_attrs(v, level));
+	} else {
+		inc = 0;
+		new_pte = arch_mm_absent_pte(level_below);
+	}
+
+	/* Initialise entries in the new table. */
+	for (i = 0; i < MM_PTE_PER_PAGE; i++) {
+		ntable->entries[i] = new_pte;
+		new_pte += inc;
+	}
+
+	/* Ensure initialisation is visible before updating the pte. */
+	atomic_thread_fence(memory_order_release);
+
+	/* Replace the pte entry, doing a break-before-make if needed. */
+	mm_replace_entry(begin, pte,
+			 arch_mm_table_pte(level, pa_init((uintpaddr_t)ntable)),
+			 level, flags, ppool);
+
+	return ntable;
+}
+
+/**
+ * Updates the page table at the given level to map the given address range to a
+ * physical range using the provided (architecture-specific) attributes. Or if
+ * MM_FLAG_UNMAP is set, unmap the given range instead.
+ *
+ * This function calls itself recursively if it needs to update additional
+ * levels, but the recursion is bound by the maximum number of levels in a page
+ * table.
+ */
+static bool mm_map_level(ptable_addr_t begin, ptable_addr_t end, paddr_t pa,
+			 uint64_t attrs, struct mm_page_table *table,
+			 uint8_t level, int flags, struct mpool *ppool)
+{
+	pte_t *pte = &table->entries[mm_index(begin, level)];
+	ptable_addr_t level_end = mm_level_end(begin, level);
+	size_t entry_size = mm_entry_size(level);
+	bool commit = flags & MM_FLAG_COMMIT;
+	bool unmap = flags & MM_FLAG_UNMAP;
+
+	/* Cap end so that we don't go over the current level max. */
+	if (end > level_end) {
+		end = level_end;
+	}
+
+	/* Fill each entry in the table. */
+	while (begin < end) {
+		if (unmap ? !arch_mm_pte_is_present(*pte, level)
+			  : arch_mm_pte_is_block(*pte, level) &&
+				    arch_mm_pte_attrs(*pte, level) == attrs) {
+			/*
+			 * If the entry is already mapped with the right
+			 * attributes, or already absent in the case of
+			 * unmapping, no need to do anything; carry on to the
+			 * next entry.
+			 */
+		} else if ((end - begin) >= entry_size &&
+			   (unmap || arch_mm_is_block_allowed(level)) &&
+			   (begin & (entry_size - 1)) == 0) {
+			/*
+			 * If the entire entry is within the region we want to
+			 * map, map/unmap the whole entry.
+			 */
+			if (commit) {
+				pte_t new_pte =
+					unmap ? arch_mm_absent_pte(level)
+					      : arch_mm_block_pte(level, pa,
+								  attrs);
+				mm_replace_entry(begin, pte, new_pte, level,
+						 flags, ppool);
+			}
+		} else {
+			/*
+			 * If the entry is already a subtable get it; otherwise
+			 * replace it with an equivalent subtable and get that.
+			 */
+			struct mm_page_table *nt = mm_populate_table_pte(
+				begin, pte, level, flags, ppool);
+			if (nt == NULL) {
+				return false;
+			}
+
+			/*
+			 * Recurse to map/unmap the appropriate entries within
+			 * the subtable.
+			 */
+			if (!mm_map_level(begin, end, pa, attrs, nt, level - 1,
+					  flags, ppool)) {
+				return false;
+			}
+		}
+
+		begin = mm_start_of_next_block(begin, entry_size);
+		pa = mm_pa_start_of_next_block(pa, entry_size);
+		pte++;
+	}
+
+	return true;
+}
+
+/**
+ * Updates the page table from the root to map the given address range to a
+ * physical range using the provided (architecture-specific) attributes. Or if
+ * MM_FLAG_UNMAP is set, unmap the given range instead.
+ */
+static bool mm_map_root(struct mm_ptable *t, ptable_addr_t begin,
+			ptable_addr_t end, uint64_t attrs, uint8_t root_level,
+			int flags, struct mpool *ppool)
+{
+	size_t root_table_size = mm_entry_size(root_level);
+	struct mm_page_table *table =
+		&mm_page_table_from_pa(t->root)[mm_index(begin, root_level)];
+
+	while (begin < end) {
+		if (!mm_map_level(begin, end, pa_init(begin), attrs, table,
+				  root_level - 1, flags, ppool)) {
+			return false;
+		}
+		begin = mm_start_of_next_block(begin, root_table_size);
+		table++;
+	}
+
+	return true;
+}
+
+/**
+ * Updates the given table such that the given physical address range is mapped
+ * or not mapped into the address space with the architecture-agnostic mode
+ * provided. Only commits the change if MM_FLAG_COMMIT is set.
+ */
+static bool mm_ptable_identity_map(struct mm_ptable *t, paddr_t pa_begin,
+				   paddr_t pa_end, uint64_t attrs, int flags,
+				   struct mpool *ppool)
+{
+	uint8_t root_level = mm_max_level(flags) + 1;
+	ptable_addr_t ptable_end = mm_ptable_addr_space_end(flags);
+	ptable_addr_t end = mm_round_up_to_page(pa_addr(pa_end));
+	ptable_addr_t begin = pa_addr(arch_mm_clear_pa(pa_begin));
+
+	/*
+	 * Assert condition to communicate the API constraint of mm_max_level(),
+	 * that isn't encoded in the types, to the static analyzer.
+	 */
+	CHECK(root_level >= 2);
+
+	/* Cap end to stay within the bounds of the page table. */
+	if (end > ptable_end) {
+		end = ptable_end;
+	}
+
+	if (!mm_map_root(t, begin, end, attrs, root_level, flags, ppool)) {
+		return false;
+	}
+
+	/* Invalidate the TLB. */
+	if ((flags & MM_FLAG_COMMIT) &&
+	    ((flags & MM_FLAG_STAGE1) || mm_stage2_invalidate)) {
+		mm_invalidate_tlb(begin, end, flags);
+	}
+
+	return true;
+}
+
+/*
+ * Prepares the given page table for the given address mapping such that it
+ * will be able to commit the change without failure. It does so by ensuring
+ * the smallest granularity needed is available. This remains valid provided
+ * subsequent operations no not decrease the granularity.
+ *
+ * In particular, multiple calls to this function will result in the
+ * corresponding calls to commit the changes to succeed.
+ */
+static bool mm_ptable_identity_prepare(struct mm_ptable *t, paddr_t pa_begin,
+				       paddr_t pa_end, uint64_t attrs,
+				       int flags, struct mpool *ppool)
+{
+	flags &= ~MM_FLAG_COMMIT;
+	return mm_ptable_identity_map(t, pa_begin, pa_end, attrs, flags, ppool);
+}
+
+/**
+ * Commits the given address mapping to the page table assuming the operation
+ * cannot fail. `mm_ptable_identity_prepare` must used correctly before this to
+ * ensure this condition.
+ *
+ * Without the table being properly prepared, the commit may only partially
+ * complete if it runs out of memory resulting in an inconsistent state that
+ * isn't handled.
+ *
+ * Since the non-failure assumtion is used in the reasoning about the atomicity
+ * of higher level memory operations, any detected violations result in a panic.
+ *
+ * TODO: remove ppool argument to be sure no changes are made.
+ */
+static void mm_ptable_identity_commit(struct mm_ptable *t, paddr_t pa_begin,
+				      paddr_t pa_end, uint64_t attrs, int flags,
+				      struct mpool *ppool)
+{
+	CHECK(mm_ptable_identity_map(t, pa_begin, pa_end, attrs,
+				     flags | MM_FLAG_COMMIT, ppool));
+}
+
+/**
+ * Updates the given table such that the given physical address range is mapped
+ * or not mapped into the address space with the architecture-agnostic mode
+ * provided.
+ *
+ * The page table is updated using the separate prepare and commit stages so
+ * that, on failure, a partial update of the address space cannot happen. The
+ * table may be left with extra internal tables but the address space is
+ * unchanged.
+ */
+static bool mm_ptable_identity_update(struct mm_ptable *t, paddr_t pa_begin,
+				      paddr_t pa_end, uint64_t attrs, int flags,
+				      struct mpool *ppool)
+{
+	if (!mm_ptable_identity_prepare(t, pa_begin, pa_end, attrs, flags,
+					ppool)) {
+		return false;
+	}
+
+	mm_ptable_identity_commit(t, pa_begin, pa_end, attrs, flags, ppool);
+
+	return true;
+}
+
+/**
+ * Writes the given table to the debug log, calling itself recursively to
+ * write sub-tables.
+ */
+static void mm_dump_table_recursive(struct mm_page_table *table, uint8_t level,
+				    int max_level)
+{
+	uint64_t i;
+
+	for (i = 0; i < MM_PTE_PER_PAGE; i++) {
+		if (!arch_mm_pte_is_present(table->entries[i], level)) {
+			continue;
+		}
+
+		dlog("%*s%x: %x\n", 4 * (max_level - level), "", i,
+		     table->entries[i]);
+
+		if (arch_mm_pte_is_table(table->entries[i], level)) {
+			mm_dump_table_recursive(
+				mm_page_table_from_pa(arch_mm_table_from_pte(
+					table->entries[i], level)),
+				level - 1, max_level);
+		}
+	}
+}
+
+/**
+ * Writes the given table to the debug log.
+ */
+static void mm_ptable_dump(struct mm_ptable *t, int flags)
+{
+	struct mm_page_table *tables = mm_page_table_from_pa(t->root);
+	uint8_t max_level = mm_max_level(flags);
+	uint8_t root_table_count = mm_root_table_count(flags);
+	uint8_t i;
+
+	for (i = 0; i < root_table_count; ++i) {
+		mm_dump_table_recursive(&tables[i], max_level, max_level);
+	}
+}
+
+/**
+ * Given the table PTE entries all have identical attributes, returns the single
+ * entry with which it can be replaced. Note that the table PTE will no longer
+ * be valid after calling this function as the table may have been freed.
+ *
+ * If the table is freed, the memory is freed directly rather than calling
+ * `mm_free_page_pte()` as it is known to not have subtables.
+ */
+static pte_t mm_merge_table_pte(pte_t table_pte, uint8_t level,
+				struct mpool *ppool)
+{
+	struct mm_page_table *table;
+	uint64_t block_attrs;
+	uint64_t table_attrs;
+	uint64_t combined_attrs;
+	paddr_t block_address;
+
+	table = mm_page_table_from_pa(arch_mm_table_from_pte(table_pte, level));
+
+	if (!arch_mm_pte_is_present(table->entries[0], level - 1)) {
+		/* Free the table and return an absent entry. */
+		mpool_free(ppool, table);
+		return arch_mm_absent_pte(level);
+	}
+
+	/* Might not be possible to merge the table into a single block. */
+	if (!arch_mm_is_block_allowed(level)) {
+		return table_pte;
+	}
+
+	/* Replace table with a single block, with equivalent attributes. */
+	block_attrs = arch_mm_pte_attrs(table->entries[0], level - 1);
+	table_attrs = arch_mm_pte_attrs(table_pte, level);
+	combined_attrs =
+		arch_mm_combine_table_entry_attrs(table_attrs, block_attrs);
+	block_address = arch_mm_block_from_pte(table->entries[0], level - 1);
+
+	/* Free the table and return a block. */
+	mpool_free(ppool, table);
+	return arch_mm_block_pte(level, block_address, combined_attrs);
+}
+
+/**
+ * Defragments the given PTE by recursively replacing any tables with blocks or
+ * absent entries where possible.
+ */
+static pte_t mm_ptable_defrag_entry(pte_t entry, uint8_t level,
+				    struct mpool *ppool)
+{
+	struct mm_page_table *table;
+	uint64_t i;
+	bool mergeable;
+	bool base_present;
+	uint64_t base_attrs;
+
+	if (!arch_mm_pte_is_table(entry, level)) {
+		return entry;
+	}
+
+	table = mm_page_table_from_pa(arch_mm_table_from_pte(entry, level));
+
+	/* Defrag the first entry in the table and use it as the base entry. */
+	static_assert(MM_PTE_PER_PAGE >= 1, "There must be at least one PTE.");
+	table->entries[0] =
+		mm_ptable_defrag_entry(table->entries[0], level - 1, ppool);
+	base_present = arch_mm_pte_is_present(table->entries[0], level - 1);
+	base_attrs = arch_mm_pte_attrs(table->entries[0], level - 1);
+
+	/*
+	 * Defrag the remaining entries in the table and check whether they are
+	 * compatible with the base entry meaning the table can be merged into a
+	 * block entry. It assumes addresses are contiguous due to identity
+	 * mapping.
+	 */
+	mergeable = true;
+	for (i = 1; i < MM_PTE_PER_PAGE; ++i) {
+		bool present;
+
+		table->entries[i] = mm_ptable_defrag_entry(table->entries[i],
+							   level - 1, ppool);
+		present = arch_mm_pte_is_present(table->entries[i], level - 1);
+
+		if (present != base_present) {
+			mergeable = false;
+			continue;
+		}
+
+		if (!present) {
+			continue;
+		}
+
+		if (!arch_mm_pte_is_block(table->entries[i], level - 1)) {
+			mergeable = false;
+			continue;
+		}
+
+		if (arch_mm_pte_attrs(table->entries[i], level - 1) !=
+		    base_attrs) {
+			mergeable = false;
+			continue;
+		}
+	}
+
+	if (mergeable) {
+		return mm_merge_table_pte(entry, level, ppool);
+	}
+
+	return entry;
+}
+
+/**
+ * Defragments the given page table by converting page table references to
+ * blocks whenever possible.
+ */
+static void mm_ptable_defrag(struct mm_ptable *t, int flags,
+			     struct mpool *ppool)
+{
+	struct mm_page_table *tables = mm_page_table_from_pa(t->root);
+	uint8_t level = mm_max_level(flags);
+	uint8_t root_table_count = mm_root_table_count(flags);
+	uint8_t i;
+	uint64_t j;
+
+	/*
+	 * Loop through each entry in the table. If it points to another table,
+	 * check if that table can be replaced by a block or an absent entry.
+	 */
+	for (i = 0; i < root_table_count; ++i) {
+		for (j = 0; j < MM_PTE_PER_PAGE; ++j) {
+			tables[i].entries[j] = mm_ptable_defrag_entry(
+				tables[i].entries[j], level, ppool);
+		}
+	}
+}
+
+/**
+ * Gets the attributes applied to the given range of stage-2 addresses at the
+ * given level.
+ *
+ * The `got_attrs` argument is initially passed as false until `attrs` contains
+ * attributes of the memory region at which point it is passed as true.
+ *
+ * The value returned in `attrs` is only valid if the function returns true.
+ *
+ * Returns true if the whole range has the same attributes and false otherwise.
+ */
+static bool mm_ptable_get_attrs_level(struct mm_page_table *table,
+				      ptable_addr_t begin, ptable_addr_t end,
+				      uint8_t level, bool got_attrs,
+				      uint64_t *attrs)
+{
+	pte_t *pte = &table->entries[mm_index(begin, level)];
+	ptable_addr_t level_end = mm_level_end(begin, level);
+	size_t entry_size = mm_entry_size(level);
+
+	/* Cap end so that we don't go over the current level max. */
+	if (end > level_end) {
+		end = level_end;
+	}
+
+	/* Check that each entry is owned. */
+	while (begin < end) {
+		if (arch_mm_pte_is_table(*pte, level)) {
+			if (!mm_ptable_get_attrs_level(
+				    mm_page_table_from_pa(
+					    arch_mm_table_from_pte(*pte,
+								   level)),
+				    begin, end, level - 1, got_attrs, attrs)) {
+				return false;
+			}
+			got_attrs = true;
+		} else {
+			if (!got_attrs) {
+				*attrs = arch_mm_pte_attrs(*pte, level);
+				got_attrs = true;
+			} else if (arch_mm_pte_attrs(*pte, level) != *attrs) {
+				return false;
+			}
+		}
+
+		begin = mm_start_of_next_block(begin, entry_size);
+		pte++;
+	}
+
+	/* The entry is a valid block. */
+	return got_attrs;
+}
+
+/**
+ * Gets the attributes applies to the given range of addresses in the stage-2
+ * table.
+ *
+ * The value returned in `attrs` is only valid if the function returns true.
+ *
+ * Returns true if the whole range has the same attributes and false otherwise.
+ */
+static bool mm_vm_get_attrs(struct mm_ptable *t, ptable_addr_t begin,
+			    ptable_addr_t end, uint64_t *attrs)
+{
+	int flags = 0;
+	uint8_t max_level = mm_max_level(flags);
+	uint8_t root_level = max_level + 1;
+	size_t root_table_size = mm_entry_size(root_level);
+	ptable_addr_t ptable_end =
+		mm_root_table_count(flags) * mm_entry_size(root_level);
+	struct mm_page_table *table;
+	bool got_attrs = false;
+
+	begin = mm_round_down_to_page(begin);
+	end = mm_round_up_to_page(end);
+
+	/* Fail if the addresses are out of range. */
+	if (end > ptable_end) {
+		return false;
+	}
+
+	table = &mm_page_table_from_pa(t->root)[mm_index(begin, root_level)];
+	while (begin < end) {
+		if (!mm_ptable_get_attrs_level(table, begin, end, max_level,
+					       got_attrs, attrs)) {
+			return false;
+		}
+
+		got_attrs = true;
+		begin = mm_start_of_next_block(begin, root_table_size);
+		table++;
+	}
+
+	return got_attrs;
+}
+
+bool mm_vm_init(struct mm_ptable *t, struct mpool *ppool)
+{
+	return mm_ptable_init(t, 0, ppool);
+}
+
+void mm_vm_fini(struct mm_ptable *t, struct mpool *ppool)
+{
+	mm_ptable_fini(t, 0, ppool);
+}
+
+/**
+ * Selects flags to pass to the page table manipulation operation based on the
+ * mapping mode.
+ */
+static int mm_mode_to_flags(uint32_t mode)
+{
+	if ((mode & MM_MODE_UNMAPPED_MASK) == MM_MODE_UNMAPPED_MASK) {
+		return MM_FLAG_UNMAP;
+	}
+
+	return 0;
+}
+
+/**
+ * See `mm_ptable_identity_prepare`.
+ *
+ * This must be called before `mm_vm_identity_commit` for the same mapping.
+ *
+ * Returns true on success, or false if the update would fail.
+ */
+bool mm_vm_identity_prepare(struct mm_ptable *t, paddr_t begin, paddr_t end,
+			    uint32_t mode, struct mpool *ppool)
+{
+	int flags = mm_mode_to_flags(mode);
+
+	return mm_ptable_identity_prepare(t, begin, end,
+					  arch_mm_mode_to_stage2_attrs(mode),
+					  flags, ppool);
+}
+
+/**
+ * See `mm_ptable_identity_commit`.
+ *
+ * `mm_vm_identity_prepare` must be called before this for the same mapping.
+ */
+void mm_vm_identity_commit(struct mm_ptable *t, paddr_t begin, paddr_t end,
+			   uint32_t mode, struct mpool *ppool, ipaddr_t *ipa)
+{
+	int flags = mm_mode_to_flags(mode);
+
+	mm_ptable_identity_commit(t, begin, end,
+				  arch_mm_mode_to_stage2_attrs(mode), flags,
+				  ppool);
+
+	if (ipa != NULL) {
+		*ipa = ipa_from_pa(begin);
+	}
+}
+
+/**
+ * Updates a VM's page table such that the given physical address range is
+ * mapped in the address space at the corresponding address range in the
+ * architecture-agnostic mode provided.
+ *
+ * mm_vm_defrag should always be called after a series of page table updates,
+ * whether they succeed or fail. This is because on failure extra page table
+ * entries may have been allocated and then not used, while on success it may be
+ * possible to compact the page table by merging several entries into a block.
+ *
+ * Returns true on success, or false if the update failed and no changes were
+ * made.
+ */
+bool mm_vm_identity_map(struct mm_ptable *t, paddr_t begin, paddr_t end,
+			uint32_t mode, struct mpool *ppool, ipaddr_t *ipa)
+{
+	int flags = mm_mode_to_flags(mode);
+	bool success = mm_ptable_identity_update(
+		t, begin, end, arch_mm_mode_to_stage2_attrs(mode), flags,
+		ppool);
+
+	if (success && ipa != NULL) {
+		*ipa = ipa_from_pa(begin);
+	}
+
+	return success;
+}
+
+/**
+ * Updates the VM's table such that the given physical address range has no
+ * connection to the VM.
+ */
+bool mm_vm_unmap(struct mm_ptable *t, paddr_t begin, paddr_t end,
+		 struct mpool *ppool)
+{
+	uint32_t mode = MM_MODE_UNMAPPED_MASK;
+
+	return mm_vm_identity_map(t, begin, end, mode, ppool, NULL);
+}
+
+/**
+ * Write the given page table of a VM to the debug log.
+ */
+void mm_vm_dump(struct mm_ptable *t)
+{
+	mm_ptable_dump(t, 0);
+}
+
+/**
+ * Defragments the VM page table.
+ */
+void mm_vm_defrag(struct mm_ptable *t, struct mpool *ppool)
+{
+	mm_ptable_defrag(t, 0, ppool);
+}
+
+/**
+ * Gets the mode of the give range of intermediate physical addresses if they
+ * are mapped with the same mode.
+ *
+ * Returns true if the range is mapped with the same mode and false otherwise.
+ */
+bool mm_vm_get_mode(struct mm_ptable *t, ipaddr_t begin, ipaddr_t end,
+		    uint32_t *mode)
+{
+	uint64_t attrs;
+	bool ret;
+
+	ret = mm_vm_get_attrs(t, ipa_addr(begin), ipa_addr(end), &attrs);
+	if (ret) {
+		*mode = arch_mm_stage2_attrs_to_mode(attrs);
+	}
+
+	return ret;
+}
+
+static struct mm_stage1_locked mm_stage1_lock_unsafe(void)
+{
+	return (struct mm_stage1_locked){.ptable = &ptable};
+}
+
+struct mm_stage1_locked mm_lock_stage1(void)
+{
+	sl_lock(&ptable_lock);
+	return mm_stage1_lock_unsafe();
+}
+
+void mm_unlock_stage1(struct mm_stage1_locked *lock)
+{
+	CHECK(lock->ptable == &ptable);
+	sl_unlock(&ptable_lock);
+	lock->ptable = NULL;
+}
+
+/**
+ * Updates the hypervisor page table such that the given physical address range
+ * is mapped into the address space at the corresponding address range in the
+ * architecture-agnostic mode provided.
+ */
+void *mm_identity_map(struct mm_stage1_locked stage1_locked, paddr_t begin,
+		      paddr_t end, uint32_t mode, struct mpool *ppool)
+{
+	int flags = MM_FLAG_STAGE1 | mm_mode_to_flags(mode);
+
+	if (mm_ptable_identity_update(stage1_locked.ptable, begin, end,
+				      arch_mm_mode_to_stage1_attrs(mode), flags,
+				      ppool)) {
+		return ptr_from_va(va_from_pa(begin));
+	}
+
+	return NULL;
+}
+
+/**
+ * Updates the hypervisor table such that the given physical address range is
+ * not mapped in the address space.
+ */
+bool mm_unmap(struct mm_stage1_locked stage1_locked, paddr_t begin, paddr_t end,
+	      struct mpool *ppool)
+{
+	uint32_t mode = MM_MODE_UNMAPPED_MASK;
+
+	return mm_identity_map(stage1_locked, begin, end, mode, ppool);
+}
+
+/**
+ * Defragments the hypervisor page table.
+ */
+void mm_defrag(struct mm_stage1_locked stage1_locked, struct mpool *ppool)
+{
+	mm_ptable_defrag(stage1_locked.ptable, MM_FLAG_STAGE1, ppool);
+}
+
+/**
+ * Initialises memory management for the hypervisor itself.
+ */
+bool mm_init(struct mpool *ppool)
+{
+	/* Locking is not enabled yet so fake it, */
+	struct mm_stage1_locked stage1_locked = mm_stage1_lock_unsafe();
+
+	dlog("text: %#x - %#x\n", pa_addr(layout_text_begin()),
+	     pa_addr(layout_text_end()));
+	dlog("rodata: %#x - %#x\n", pa_addr(layout_rodata_begin()),
+	     pa_addr(layout_rodata_end()));
+	dlog("data: %#x - %#x\n", pa_addr(layout_data_begin()),
+	     pa_addr(layout_data_end()));
+
+	if (!mm_ptable_init(&ptable, MM_FLAG_STAGE1, ppool)) {
+		dlog("Unable to allocate memory for page table.\n");
+		return false;
+	}
+
+	/* Let console driver map pages for itself. */
+	plat_console_mm_init(stage1_locked, ppool);
+
+	/* Map each section. */
+	mm_identity_map(stage1_locked, layout_text_begin(), layout_text_end(),
+			MM_MODE_X, ppool);
+
+	mm_identity_map(stage1_locked, layout_rodata_begin(),
+			layout_rodata_end(), MM_MODE_R, ppool);
+
+	mm_identity_map(stage1_locked, layout_data_begin(), layout_data_end(),
+			MM_MODE_R | MM_MODE_W, ppool);
+
+	return arch_mm_init(ptable.root);
+}
diff --git a/src/mm_test.cc b/src/mm_test.cc
new file mode 100644
index 0000000..887d16c
--- /dev/null
+++ b/src/mm_test.cc
@@ -0,0 +1,1195 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gmock/gmock.h>
+
+extern "C" {
+#include "hf/arch/mm.h"
+
+#include "hf/mm.h"
+#include "hf/mpool.h"
+}
+
+#include <limits>
+#include <memory>
+#include <span>
+#include <vector>
+
+#include "mm_test.hh"
+
+namespace
+{
+using namespace ::std::placeholders;
+
+using ::testing::AllOf;
+using ::testing::Contains;
+using ::testing::Each;
+using ::testing::Eq;
+using ::testing::Not;
+using ::testing::SizeIs;
+using ::testing::Truly;
+
+using ::mm_test::get_ptable;
+
+constexpr size_t TEST_HEAP_SIZE = PAGE_SIZE * 16;
+const int TOP_LEVEL = arch_mm_stage2_max_level();
+const paddr_t VM_MEM_END = pa_init(0x200'0000'0000);
+
+/**
+ * Calculates the size of the address space represented by a page table entry at
+ * the given level.
+ */
+size_t mm_entry_size(int level)
+{
+	return UINT64_C(1) << (PAGE_BITS + level * PAGE_LEVEL_BITS);
+}
+
+/**
+ * Checks whether the address is mapped in the address space.
+ */
+bool mm_vm_is_mapped(struct mm_ptable *t, ipaddr_t ipa)
+{
+	uint32_t mode;
+	return mm_vm_get_mode(t, ipa, ipa_add(ipa, 1), &mode) &&
+	       (mode & MM_MODE_INVALID) == 0;
+}
+
+/**
+ * Get an STL representation of the page table.
+ */
+std::span<pte_t, MM_PTE_PER_PAGE> get_table(paddr_t pa)
+{
+	auto table = reinterpret_cast<struct mm_page_table *>(
+		ptr_from_va(va_from_pa(pa)));
+	return std::span<pte_t>(table->entries, std::end(table->entries));
+}
+
+class mm : public ::testing::Test
+{
+	void SetUp() override
+	{
+		/*
+		 * TODO: replace with direct use of stdlib allocator so
+		 * sanitizers are more effective.
+		 */
+		test_heap = std::make_unique<uint8_t[]>(TEST_HEAP_SIZE);
+		mpool_init(&ppool, sizeof(struct mm_page_table));
+		mpool_add_chunk(&ppool, test_heap.get(), TEST_HEAP_SIZE);
+	}
+
+	std::unique_ptr<uint8_t[]> test_heap;
+
+       protected:
+	struct mpool ppool;
+};
+
+/**
+ * A new table is initially empty.
+ */
+TEST_F(mm, ptable_init_empty)
+{
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Each new concatenated table is initially empty.
+ */
+TEST_F(mm, ptable_init_concatenated_empty)
+{
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Only the first page is mapped with all others left absent.
+ */
+TEST_F(mm, map_first_page)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t page_begin = pa_init(0);
+	const paddr_t page_end = pa_add(page_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, page_begin, page_end, mode,
+				       &ppool, nullptr));
+
+	auto tables = get_ptable(ptable);
+	EXPECT_THAT(tables, SizeIs(4));
+	ASSERT_THAT(TOP_LEVEL, Eq(2));
+
+	/* Check that the first page is mapped and nothing else. */
+	EXPECT_THAT(std::span(tables).last(3),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+
+	auto table_l2 = tables.front();
+	EXPECT_THAT(table_l2.subspan(1), Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l2[0], TOP_LEVEL));
+
+	auto table_l1 =
+		get_table(arch_mm_table_from_pte(table_l2[0], TOP_LEVEL));
+	EXPECT_THAT(table_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l1[0], TOP_LEVEL - 1));
+
+	auto table_l0 =
+		get_table(arch_mm_table_from_pte(table_l1[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table_l0.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table_l0[0], TOP_LEVEL - 2));
+	EXPECT_THAT(pa_addr(arch_mm_block_from_pte(table_l0[0], TOP_LEVEL - 2)),
+		    Eq(pa_addr(page_begin)));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * The start address is rounded down and the end address is rounded up to page
+ * boundaries.
+ */
+TEST_F(mm, map_round_to_page)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t map_begin = pa_init(0x200'0000'0000 - PAGE_SIZE + 23);
+	const paddr_t map_end = pa_add(map_begin, 268);
+	ipaddr_t ipa = ipa_init(-1);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, map_begin, map_end, mode,
+				       &ppool, &ipa));
+	EXPECT_THAT(ipa_addr(ipa), Eq(pa_addr(map_begin)));
+
+	auto tables = get_ptable(ptable);
+	EXPECT_THAT(tables, SizeIs(4));
+	ASSERT_THAT(TOP_LEVEL, Eq(2));
+
+	/* Check that the last page is mapped, and nothing else. */
+	EXPECT_THAT(std::span(tables).first(3),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+
+	auto table_l2 = tables.back();
+	EXPECT_THAT(table_l2.first(table_l2.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l2.last(1)[0], TOP_LEVEL));
+
+	auto table_l1 = get_table(
+		arch_mm_table_from_pte(table_l2.last(1)[0], TOP_LEVEL));
+	EXPECT_THAT(table_l1.first(table_l1.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l1.last(1)[0], TOP_LEVEL - 1));
+
+	auto table_l0 = get_table(
+		arch_mm_table_from_pte(table_l1.last(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table_l0.first(table_l0.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table_l0.last(1)[0], TOP_LEVEL - 2));
+	EXPECT_THAT(pa_addr(arch_mm_block_from_pte(table_l0.last(1)[0],
+						   TOP_LEVEL - 2)),
+		    Eq(0x200'0000'0000 - PAGE_SIZE));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Map a two page range over the boundary of two tables.
+ */
+TEST_F(mm, map_across_tables)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t map_begin = pa_init(0x80'0000'0000 - PAGE_SIZE);
+	const paddr_t map_end = pa_add(map_begin, 2 * PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, map_begin, map_end, mode,
+				       &ppool, nullptr));
+
+	auto tables = get_ptable(ptable);
+	EXPECT_THAT(tables, SizeIs(4));
+	EXPECT_THAT(std::span(tables).last(2),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+	ASSERT_THAT(TOP_LEVEL, Eq(2));
+
+	/* Check only the last page of the first table is mapped. */
+	auto table0_l2 = tables.front();
+	EXPECT_THAT(table0_l2.first(table0_l2.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table0_l2.last(1)[0], TOP_LEVEL));
+
+	auto table0_l1 = get_table(
+		arch_mm_table_from_pte(table0_l2.last(1)[0], TOP_LEVEL));
+	EXPECT_THAT(table0_l1.first(table0_l1.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table0_l1.last(1)[0], TOP_LEVEL - 1));
+
+	auto table0_l0 = get_table(
+		arch_mm_table_from_pte(table0_l1.last(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table0_l0.first(table0_l0.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table0_l0.last(1)[0], TOP_LEVEL - 2));
+	EXPECT_THAT(pa_addr(arch_mm_block_from_pte(table0_l0.last(1)[0],
+						   TOP_LEVEL - 2)),
+		    Eq(pa_addr(map_begin)));
+
+	/* Check only the first page of the second table is mapped. */
+	auto table1_l2 = tables[1];
+	EXPECT_THAT(table1_l2.subspan(1), Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table1_l2[0], TOP_LEVEL));
+
+	auto table1_l1 =
+		get_table(arch_mm_table_from_pte(table1_l2[0], TOP_LEVEL));
+	EXPECT_THAT(table1_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table1_l1[0], TOP_LEVEL - 1));
+
+	auto table1_l0 =
+		get_table(arch_mm_table_from_pte(table1_l1[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table1_l0.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table1_l0[0], TOP_LEVEL - 2));
+	EXPECT_THAT(
+		pa_addr(arch_mm_block_from_pte(table1_l0[0], TOP_LEVEL - 2)),
+		Eq(pa_addr(pa_add(map_begin, PAGE_SIZE))));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping all of memory creates blocks at the highest level.
+ */
+TEST_F(mm, map_all_at_top_level)
+{
+	constexpr uint32_t mode = 0;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	auto tables = get_ptable(ptable);
+	EXPECT_THAT(
+		tables,
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	for (uint64_t i = 0; i < tables.size(); ++i) {
+		for (uint64_t j = 0; j < MM_PTE_PER_PAGE; ++j) {
+			EXPECT_THAT(pa_addr(arch_mm_block_from_pte(tables[i][j],
+								   TOP_LEVEL)),
+				    Eq((i * mm_entry_size(TOP_LEVEL + 1)) +
+				       (j * mm_entry_size(TOP_LEVEL))))
+				<< "i=" << i << " j=" << j;
+		}
+	}
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Map all memory then trying to map a page again doesn't introduce a special
+ * mapping for that particular page.
+ */
+TEST_F(mm, map_already_mapped)
+{
+	constexpr uint32_t mode = 0;
+	ipaddr_t ipa = ipa_init(-1);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), pa_init(PAGE_SIZE),
+				       mode, &ppool, &ipa));
+	EXPECT_THAT(ipa_addr(ipa), Eq(0));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping a reverse range, i.e. the end comes before the start, is treated as
+ * an empty range so no mappings are made.
+ */
+TEST_F(mm, map_reverse_range)
+{
+	constexpr uint32_t mode = 0;
+	ipaddr_t ipa = ipa_init(-1);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0x1234'5678),
+				       pa_init(0x5000), mode, &ppool, &ipa));
+	EXPECT_THAT(ipa_addr(ipa), Eq(0x1234'5678));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping a reverse range in the same page will map the page because the start
+ * of the range is rounded down and the end is rounded up.
+ *
+ * This serves as a form of documentation of behaviour rather than a
+ * requirement. Check whether any code relies on this before changing it.
+ */
+TEST_F(mm, map_reverse_range_quirk)
+{
+	constexpr uint32_t mode = 0;
+	ipaddr_t ipa = ipa_init(-1);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(20), pa_init(10), mode,
+				       &ppool, &ipa));
+	EXPECT_THAT(ipa_addr(ipa), Eq(20));
+	EXPECT_TRUE(mm_vm_is_mapped(&ptable, ipa));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping a range up to the maximum address causes the range end to wrap to
+ * zero as it is rounded up to a page boundary meaning no memory is mapped.
+ *
+ * This serves as a form of documentation of behaviour rather than a
+ * requirement. Check whether any code relies on this before changing it.
+ */
+TEST_F(mm, map_last_address_quirk)
+{
+	constexpr uint32_t mode = 0;
+	ipaddr_t ipa = ipa_init(-1);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(
+		&ptable, pa_init(0),
+		pa_init(std::numeric_limits<uintpaddr_t>::max()), mode, &ppool,
+		&ipa));
+	EXPECT_THAT(ipa_addr(ipa), Eq(0));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping a range that goes beyond the available memory clamps to the available
+ * range.
+ */
+TEST_F(mm, map_clamp_to_range)
+{
+	constexpr uint32_t mode = 0;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0),
+				       pa_init(0xf32'0000'0000'0000), mode,
+				       &ppool, nullptr));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping a range outside of the available memory is ignored and doesn't alter
+ * the page tables.
+ */
+TEST_F(mm, map_ignore_out_of_range)
+{
+	constexpr uint32_t mode = 0;
+	ipaddr_t ipa = ipa_init(-1);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, VM_MEM_END,
+				       pa_init(0xf0'0000'0000'0000), mode,
+				       &ppool, &ipa));
+	EXPECT_THAT(ipa_addr(ipa), Eq(pa_addr(VM_MEM_END)));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Map a single page and then map all of memory which replaces the single page
+ * mapping with a higher level block mapping.
+ */
+TEST_F(mm, map_block_replaces_table)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t page_begin = pa_init(34567 * PAGE_SIZE);
+	const paddr_t page_end = pa_add(page_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, page_begin, page_end, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Map all memory at the top level, unmapping a page and remapping at a lower
+ * level does not result in all memory being mapped at the top level again.
+ */
+TEST_F(mm, map_does_not_defrag)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t page_begin = pa_init(12000 * PAGE_SIZE);
+	const paddr_t page_end = pa_add(page_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, page_begin, page_end, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, page_begin, page_end, mode,
+				       &ppool, nullptr));
+	EXPECT_THAT(get_ptable(ptable),
+		    AllOf(SizeIs(4),
+			  Each(Each(Truly(std::bind(arch_mm_pte_is_present, _1,
+						    TOP_LEVEL)))),
+			  Contains(Contains(Truly(std::bind(
+				  arch_mm_pte_is_block, _1, TOP_LEVEL)))),
+			  Contains(Contains(Truly(std::bind(
+				  arch_mm_pte_is_table, _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping with a mode that indicates unmapping results in the addresses being
+ * unmapped with absent entries.
+ */
+TEST_F(mm, map_to_unmap)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t l0_begin = pa_init(uintpaddr_t(524421) * PAGE_SIZE);
+	const paddr_t l0_end = pa_add(l0_begin, 17 * PAGE_SIZE);
+	const paddr_t l1_begin = pa_init(3 * mm_entry_size(1));
+	const paddr_t l1_end = pa_add(l1_begin, 5 * mm_entry_size(1));
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l0_begin, l0_end, mode, &ppool,
+				       nullptr));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l1_begin, l1_end, mode, &ppool,
+				       nullptr));
+	EXPECT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END,
+				       MM_MODE_UNMAPPED_MASK, &ppool, nullptr));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/*
+ * Preparing and committing an address range works the same as mapping it.
+ */
+TEST_F(mm, prepare_and_commit_first_page)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t page_begin = pa_init(0);
+	const paddr_t page_end = pa_add(page_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_prepare(&ptable, page_begin, page_end, mode,
+					   &ppool));
+	mm_vm_identity_commit(&ptable, page_begin, page_end, mode, &ppool,
+			      nullptr);
+
+	auto tables = get_ptable(ptable);
+	EXPECT_THAT(tables, SizeIs(4));
+	ASSERT_THAT(TOP_LEVEL, Eq(2));
+
+	/* Check that the first page is mapped and nothing else. */
+	EXPECT_THAT(std::span(tables).last(3),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+
+	auto table_l2 = tables.front();
+	EXPECT_THAT(table_l2.subspan(1), Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l2[0], TOP_LEVEL));
+
+	auto table_l1 =
+		get_table(arch_mm_table_from_pte(table_l2[0], TOP_LEVEL));
+	EXPECT_THAT(table_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l1[0], TOP_LEVEL - 1));
+
+	auto table_l0 =
+		get_table(arch_mm_table_from_pte(table_l1[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table_l0.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table_l0[0], TOP_LEVEL - 2));
+	EXPECT_THAT(pa_addr(arch_mm_block_from_pte(table_l0[0], TOP_LEVEL - 2)),
+		    Eq(pa_addr(page_begin)));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Disjoint address ranges can be prepared and committed together.
+ */
+TEST_F(mm, prepare_and_commit_disjoint_regions)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t first_begin = pa_init(0);
+	const paddr_t first_end = pa_add(first_begin, PAGE_SIZE);
+	const paddr_t last_begin = pa_init(pa_addr(VM_MEM_END) - PAGE_SIZE);
+	const paddr_t last_end = VM_MEM_END;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_prepare(&ptable, first_begin, first_end,
+					   mode, &ppool));
+	ASSERT_TRUE(mm_vm_identity_prepare(&ptable, last_begin, last_end, mode,
+					   &ppool));
+	mm_vm_identity_commit(&ptable, first_begin, first_end, mode, &ppool,
+			      nullptr);
+	mm_vm_identity_commit(&ptable, last_begin, last_end, mode, &ppool,
+			      nullptr);
+
+	auto tables = get_ptable(ptable);
+	EXPECT_THAT(tables, SizeIs(4));
+	ASSERT_THAT(TOP_LEVEL, Eq(2));
+
+	/* Check that the first and last pages are mapped and nothing else. */
+	EXPECT_THAT(std::span(tables).subspan(1, 2),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+
+	/* Check the first page. */
+	auto table0_l2 = tables.front();
+	EXPECT_THAT(table0_l2.subspan(1), Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table0_l2[0], TOP_LEVEL));
+
+	auto table0_l1 =
+		get_table(arch_mm_table_from_pte(table0_l2[0], TOP_LEVEL));
+	EXPECT_THAT(table0_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table0_l1[0], TOP_LEVEL - 1));
+
+	auto table0_l0 =
+		get_table(arch_mm_table_from_pte(table0_l1[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table0_l0.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table0_l0[0], TOP_LEVEL - 2));
+	EXPECT_THAT(
+		pa_addr(arch_mm_block_from_pte(table0_l0[0], TOP_LEVEL - 2)),
+		Eq(pa_addr(first_begin)));
+
+	/* Check the last page. */
+	auto table3_l2 = tables.back();
+	EXPECT_THAT(table3_l2.first(table3_l2.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table3_l2.last(1)[0], TOP_LEVEL));
+
+	auto table3_l1 = get_table(
+		arch_mm_table_from_pte(table3_l2.last(1)[0], TOP_LEVEL));
+	EXPECT_THAT(table3_l1.first(table3_l1.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table3_l1.last(1)[0], TOP_LEVEL - 1));
+
+	auto table3_l0 = get_table(
+		arch_mm_table_from_pte(table3_l1.last(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table3_l0.first(table3_l0.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table3_l0.last(1)[0], TOP_LEVEL - 2));
+	EXPECT_THAT(pa_addr(arch_mm_block_from_pte(table3_l0.last(1)[0],
+						   TOP_LEVEL - 2)),
+		    Eq(pa_addr(last_begin)));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Overlapping address ranges can be prepared and committed together.
+ */
+TEST_F(mm, prepare_and_commit_overlapping_regions)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t low_begin = pa_init(0x80'0000'0000 - PAGE_SIZE);
+	const paddr_t high_begin = pa_add(low_begin, PAGE_SIZE);
+	const paddr_t map_end = pa_add(high_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_prepare(&ptable, high_begin, map_end, mode,
+					   &ppool));
+	ASSERT_TRUE(mm_vm_identity_prepare(&ptable, low_begin, map_end, mode,
+					   &ppool));
+	mm_vm_identity_commit(&ptable, high_begin, map_end, mode, &ppool,
+			      nullptr);
+	mm_vm_identity_commit(&ptable, low_begin, map_end, mode, &ppool,
+			      nullptr);
+
+	auto tables = get_ptable(ptable);
+	EXPECT_THAT(tables, SizeIs(4));
+	EXPECT_THAT(std::span(tables).last(2),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+	ASSERT_THAT(TOP_LEVEL, Eq(2));
+
+	/* Check only the last page of the first table is mapped. */
+	auto table0_l2 = tables.front();
+	EXPECT_THAT(table0_l2.first(table0_l2.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table0_l2.last(1)[0], TOP_LEVEL));
+
+	auto table0_l1 = get_table(
+		arch_mm_table_from_pte(table0_l2.last(1)[0], TOP_LEVEL));
+	EXPECT_THAT(table0_l1.first(table0_l1.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table0_l1.last(1)[0], TOP_LEVEL - 1));
+
+	auto table0_l0 = get_table(
+		arch_mm_table_from_pte(table0_l1.last(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table0_l0.first(table0_l0.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table0_l0.last(1)[0], TOP_LEVEL - 2));
+	EXPECT_THAT(pa_addr(arch_mm_block_from_pte(table0_l0.last(1)[0],
+						   TOP_LEVEL - 2)),
+		    Eq(pa_addr(low_begin)));
+
+	/* Check only the first page of the second table is mapped. */
+	auto table1_l2 = tables[1];
+	EXPECT_THAT(table1_l2.subspan(1), Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table1_l2[0], TOP_LEVEL));
+
+	auto table1_l1 =
+		get_table(arch_mm_table_from_pte(table1_l2[0], TOP_LEVEL));
+	EXPECT_THAT(table1_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table1_l1[0], TOP_LEVEL - 1));
+
+	auto table1_l0 =
+		get_table(arch_mm_table_from_pte(table1_l1[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table1_l0.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+	ASSERT_TRUE(arch_mm_pte_is_block(table1_l0[0], TOP_LEVEL - 2));
+	EXPECT_THAT(
+		pa_addr(arch_mm_block_from_pte(table1_l0[0], TOP_LEVEL - 2)),
+		Eq(pa_addr(high_begin)));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * If range is not mapped, unmapping has no effect.
+ */
+TEST_F(mm, unmap_not_mapped)
+{
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	EXPECT_TRUE(
+		mm_vm_unmap(&ptable, pa_init(12345), pa_init(987652), &ppool));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Unmapping everything should result in an empty page table with no subtables.
+ */
+TEST_F(mm, unmap_all)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t l0_begin = pa_init(uintpaddr_t(524421) * PAGE_SIZE);
+	const paddr_t l0_end = pa_add(l0_begin, 17 * PAGE_SIZE);
+	const paddr_t l1_begin = pa_init(3 * mm_entry_size(1));
+	const paddr_t l1_end = pa_add(l1_begin, 5 * mm_entry_size(1));
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l0_begin, l0_end, mode, &ppool,
+				       nullptr));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l1_begin, l1_end, mode, &ppool,
+				       nullptr));
+	EXPECT_TRUE(mm_vm_unmap(&ptable, pa_init(0), VM_MEM_END, &ppool));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Unmap range is rounded to the containing pages.
+ */
+TEST_F(mm, unmap_round_to_page)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t map_begin = pa_init(0x160'0000'0000 + PAGE_SIZE);
+	const paddr_t map_end = pa_add(map_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, map_begin, map_end, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, pa_add(map_begin, 93),
+				pa_add(map_begin, 99), &ppool));
+
+	auto tables = get_ptable(ptable);
+	constexpr auto l3_index = 2;
+
+	/* Check all other top level entries are empty... */
+	EXPECT_THAT(std::span(tables).first(l3_index),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+	EXPECT_THAT(std::span(tables).subspan(l3_index + 1),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+
+	/* Except the mapped page which is absent. */
+	auto table_l2 = tables[l3_index];
+	constexpr auto l2_index = 384;
+	EXPECT_THAT(table_l2.first(l2_index),
+		    Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l2[l2_index], TOP_LEVEL));
+	EXPECT_THAT(table_l2.subspan(l2_index + 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL)));
+
+	auto table_l1 = get_table(
+		arch_mm_table_from_pte(table_l2[l2_index], TOP_LEVEL));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l1.first(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+
+	auto table_l0 = get_table(
+		arch_mm_table_from_pte(table_l1.first(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table_l0, Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Unmap a range that of page mappings that spans multiple concatenated tables.
+ */
+TEST_F(mm, unmap_across_tables)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t map_begin = pa_init(0x180'0000'0000 - PAGE_SIZE);
+	const paddr_t map_end = pa_add(map_begin, 2 * PAGE_SIZE);
+	struct mm_ptable ptable;
+
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, map_begin, map_end, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, map_begin, map_end, &ppool));
+
+	auto tables = get_ptable(ptable);
+
+	/* Check the untouched tables are empty. */
+	EXPECT_THAT(std::span(tables).first(2),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+
+	/* Check the last page is explicity marked as absent. */
+	auto table2_l2 = tables[2];
+	EXPECT_THAT(table2_l2.first(table2_l2.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table2_l2.last(1)[0], TOP_LEVEL));
+
+	auto table2_l1 = get_table(
+		arch_mm_table_from_pte(table2_l2.last(1)[0], TOP_LEVEL));
+	EXPECT_THAT(table2_l1.first(table2_l1.size() - 1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+	ASSERT_TRUE(arch_mm_pte_is_table(table2_l1.last(1)[0], TOP_LEVEL - 1));
+
+	auto table2_l0 = get_table(
+		arch_mm_table_from_pte(table2_l1.last(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table2_l0, Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+
+	/* Check the first page is explicitly marked as absent. */
+	auto table3_l2 = tables[3];
+	ASSERT_TRUE(arch_mm_pte_is_table(table3_l2.first(1)[0], TOP_LEVEL));
+	EXPECT_THAT(table3_l2.subspan(1), Each(arch_mm_absent_pte(TOP_LEVEL)));
+
+	auto table3_l1 = get_table(
+		arch_mm_table_from_pte(table3_l2.first(1)[0], TOP_LEVEL));
+	ASSERT_TRUE(arch_mm_pte_is_table(table3_l1.first(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table3_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+
+	auto table3_l0 = get_table(
+		arch_mm_table_from_pte(table3_l1.first(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table3_l0, Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Unmapping outside the range of memory had no effect.
+ */
+TEST_F(mm, unmap_out_of_range)
+{
+	constexpr uint32_t mode = 0;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, VM_MEM_END, pa_init(0x4000'0000'0000),
+				&ppool));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Unmapping a reverse range, i.e. the end comes before the start, is treated as
+ * an empty range so no change is made.
+ */
+TEST_F(mm, unmap_reverse_range)
+{
+	constexpr uint32_t mode = 0;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, pa_init(0x80'a000'0000), pa_init(27),
+				&ppool));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Unmapping a reverse range in the same page will unmap the page because the
+ * start of the range is rounded down and the end is rounded up.
+ *
+ * This serves as a form of documentation of behaviour rather than a
+ * requirement. Check whether any code relies on this before changing it.
+ */
+TEST_F(mm, unmap_reverse_range_quirk)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t page_begin = pa_init(0x180'0000'0000);
+	const paddr_t page_end = pa_add(page_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, page_begin, page_end, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, pa_add(page_begin, 100),
+				pa_add(page_begin, 50), &ppool));
+
+	auto tables = get_ptable(ptable);
+	constexpr auto l3_index = 3;
+
+	/* Check all other top level entries are empty... */
+	EXPECT_THAT(std::span(tables).first(l3_index),
+		    Each(Each(arch_mm_absent_pte(TOP_LEVEL))));
+
+	/* Except the mapped page which is absent. */
+	auto table_l2 = tables[l3_index];
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l2.first(1)[0], TOP_LEVEL));
+	EXPECT_THAT(table_l2.subspan(1), Each(arch_mm_absent_pte(TOP_LEVEL)));
+
+	auto table_l1 = get_table(
+		arch_mm_table_from_pte(table_l2.first(1)[0], TOP_LEVEL));
+	ASSERT_TRUE(arch_mm_pte_is_table(table_l1.first(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table_l1.subspan(1),
+		    Each(arch_mm_absent_pte(TOP_LEVEL - 1)));
+
+	auto table_l0 = get_table(
+		arch_mm_table_from_pte(table_l1.first(1)[0], TOP_LEVEL - 1));
+	EXPECT_THAT(table_l0, Each(arch_mm_absent_pte(TOP_LEVEL - 2)));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Unmapping a range up to the maximum address causes the range end to wrap to
+ * zero as it is rounded up to a page boundary meaning no change is made.
+ *
+ * This serves as a form of documentation of behaviour rather than a
+ * requirement. Check whether any code relies on this before changing it.
+ */
+TEST_F(mm, unmap_last_address_quirk)
+{
+	constexpr uint32_t mode = 0;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(
+		&ptable, pa_init(0),
+		pa_init(std::numeric_limits<uintpaddr_t>::max()), &ppool));
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Mapping then unmapping a page does not defrag the table.
+ */
+TEST_F(mm, unmap_does_not_defrag)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t l0_begin = pa_init(5555 * PAGE_SIZE);
+	const paddr_t l0_end = pa_add(l0_begin, 13 * PAGE_SIZE);
+	const paddr_t l1_begin = pa_init(666 * mm_entry_size(1));
+	const paddr_t l1_end = pa_add(l1_begin, 5 * mm_entry_size(1));
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l0_begin, l0_end, mode, &ppool,
+				       nullptr));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l1_begin, l1_end, mode, &ppool,
+				       nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, l0_begin, l0_end, &ppool));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, l1_begin, l1_end, &ppool));
+	EXPECT_THAT(get_ptable(ptable),
+		    AllOf(SizeIs(4),
+			  Not(Each(Each(arch_mm_absent_pte(TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Nothing is mapped in an empty table.
+ */
+TEST_F(mm, is_mapped_empty)
+{
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	EXPECT_FALSE(mm_vm_is_mapped(&ptable, ipa_init(0)));
+	EXPECT_FALSE(mm_vm_is_mapped(&ptable, ipa_init(0x8123'2344)));
+	EXPECT_FALSE(mm_vm_is_mapped(&ptable, ipa_init(0x1e0'0000'0073)));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Everything is mapped in a full table.
+ */
+TEST_F(mm, is_mapped_all)
+{
+	constexpr uint32_t mode = 0;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	EXPECT_TRUE(mm_vm_is_mapped(&ptable, ipa_init(0)));
+	EXPECT_TRUE(mm_vm_is_mapped(&ptable, ipa_init(0xf247'a7b3)));
+	EXPECT_TRUE(mm_vm_is_mapped(&ptable, ipa_init(0x1ff'7bfa'983b)));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * A page is mapped for the range [begin, end).
+ */
+TEST_F(mm, is_mapped_page)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t page_begin = pa_init(0x100'0000'0000);
+	const paddr_t page_end = pa_add(page_begin, PAGE_SIZE);
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, page_begin, page_end, mode,
+				       &ppool, nullptr));
+	EXPECT_TRUE(mm_vm_is_mapped(&ptable, ipa_from_pa(page_begin)));
+	EXPECT_TRUE(
+		mm_vm_is_mapped(&ptable, ipa_from_pa(pa_add(page_begin, 127))));
+	EXPECT_FALSE(mm_vm_is_mapped(&ptable, ipa_from_pa(page_end)));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Everything out of range is not mapped.
+ */
+TEST_F(mm, is_mapped_out_of_range)
+{
+	constexpr uint32_t mode = 0;
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	EXPECT_FALSE(mm_vm_is_mapped(&ptable, ipa_from_pa(VM_MEM_END)));
+	EXPECT_FALSE(mm_vm_is_mapped(&ptable, ipa_init(0x1000'adb7'8123)));
+	EXPECT_FALSE(mm_vm_is_mapped(
+		&ptable, ipa_init(std::numeric_limits<uintpaddr_t>::max())));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * The mode of unmapped addresses can be retrieved and is set to invalid,
+ * unowned and shared.
+ */
+TEST_F(mm, get_mode_empty)
+{
+	constexpr int default_mode =
+		MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
+	struct mm_ptable ptable;
+	uint32_t read_mode;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+
+	read_mode = 0;
+	EXPECT_TRUE(
+		mm_vm_get_mode(&ptable, ipa_init(0), ipa_init(20), &read_mode));
+	EXPECT_THAT(read_mode, Eq(default_mode));
+
+	read_mode = 0;
+	EXPECT_TRUE(mm_vm_get_mode(&ptable, ipa_init(0x3c97'654d),
+				   ipa_init(0x3c97'e000), &read_mode));
+	EXPECT_THAT(read_mode, Eq(default_mode));
+
+	read_mode = 0;
+	EXPECT_TRUE(mm_vm_get_mode(&ptable, ipa_init(0x5f'ffff'ffff),
+				   ipa_init(0x1ff'ffff'ffff), &read_mode));
+	EXPECT_THAT(read_mode, Eq(default_mode));
+
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Get the mode of a range comprised of individual pages which are either side
+ * of a root table boundary.
+ */
+TEST_F(mm, get_mode_pages_across_tables)
+{
+	constexpr uint32_t mode = MM_MODE_INVALID | MM_MODE_SHARED;
+	const paddr_t map_begin = pa_init(0x180'0000'0000 - PAGE_SIZE);
+	const paddr_t map_end = pa_add(map_begin, 2 * PAGE_SIZE);
+	struct mm_ptable ptable;
+	uint32_t read_mode;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, map_begin, map_end, mode,
+				       &ppool, nullptr));
+
+	read_mode = 0;
+	EXPECT_TRUE(mm_vm_get_mode(&ptable, ipa_from_pa(map_begin),
+				   ipa_from_pa(pa_add(map_begin, PAGE_SIZE)),
+				   &read_mode));
+	EXPECT_THAT(read_mode, Eq(mode));
+
+	EXPECT_FALSE(mm_vm_get_mode(&ptable, ipa_init(0),
+				    ipa_from_pa(pa_add(map_begin, PAGE_SIZE)),
+				    &read_mode));
+
+	read_mode = 0;
+	EXPECT_TRUE(mm_vm_get_mode(&ptable, ipa_from_pa(map_begin),
+				   ipa_from_pa(map_end), &read_mode));
+	EXPECT_THAT(read_mode, Eq(mode));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Anything out of range fail to retrieve the mode.
+ */
+TEST_F(mm, get_mode_out_of_range)
+{
+	constexpr uint32_t mode = MM_MODE_UNOWNED;
+	struct mm_ptable ptable;
+	uint32_t read_mode;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	EXPECT_FALSE(mm_vm_get_mode(&ptable, ipa_init(0),
+				    ipa_from_pa(pa_add(VM_MEM_END, 1)),
+				    &read_mode));
+	EXPECT_FALSE(mm_vm_get_mode(&ptable, ipa_from_pa(VM_MEM_END),
+				    ipa_from_pa(pa_add(VM_MEM_END, 1)),
+				    &read_mode));
+	EXPECT_FALSE(mm_vm_get_mode(&ptable, ipa_init(0x1'1234'1234'1234),
+				    ipa_init(2'0000'0000'0000), &read_mode));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Defragging an entirely empty table has no effect.
+ */
+TEST_F(mm, defrag_empty)
+{
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	mm_vm_defrag(&ptable, &ppool);
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Defragging a table with some empty subtables (even nested) results in
+ * an empty table.
+ */
+TEST_F(mm, defrag_empty_subtables)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t l0_begin = pa_init(120000 * PAGE_SIZE);
+	const paddr_t l0_end = pa_add(l0_begin, PAGE_SIZE);
+	const paddr_t l1_begin = pa_init(3 * mm_entry_size(1));
+	const paddr_t l1_end = pa_add(l1_begin, 5 * mm_entry_size(1));
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l0_begin, l0_end, mode, &ppool,
+				       nullptr));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, l1_begin, l1_end, mode, &ppool,
+				       nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, l0_begin, l0_end, &ppool));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, l1_begin, l1_end, &ppool));
+	mm_vm_defrag(&ptable, &ppool);
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+/**
+ * Any subtable with all blocks with the same attributes should be replaced
+ * with a single block.
+ */
+TEST_F(mm, defrag_block_subtables)
+{
+	constexpr uint32_t mode = 0;
+	const paddr_t begin = pa_init(39456 * mm_entry_size(1));
+	const paddr_t middle = pa_add(begin, 67 * PAGE_SIZE);
+	const paddr_t end = pa_add(begin, 4 * mm_entry_size(1));
+	struct mm_ptable ptable;
+	ASSERT_TRUE(mm_vm_init(&ptable, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, pa_init(0), VM_MEM_END, mode,
+				       &ppool, nullptr));
+	ASSERT_TRUE(mm_vm_unmap(&ptable, begin, end, &ppool));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, begin, middle, mode, &ppool,
+				       nullptr));
+	ASSERT_TRUE(mm_vm_identity_map(&ptable, middle, end, mode, &ppool,
+				       nullptr));
+	mm_vm_defrag(&ptable, &ppool);
+	EXPECT_THAT(
+		get_ptable(ptable),
+		AllOf(SizeIs(4), Each(Each(Truly(std::bind(arch_mm_pte_is_block,
+							   _1, TOP_LEVEL))))));
+	mm_vm_fini(&ptable, &ppool);
+}
+
+} /* namespace */
+
+namespace mm_test
+{
+/**
+ * Get an STL representation of the ptable.
+ */
+std::vector<std::span<pte_t, MM_PTE_PER_PAGE>> get_ptable(
+	const struct mm_ptable &ptable)
+{
+	std::vector<std::span<pte_t, MM_PTE_PER_PAGE>> all;
+	const uint8_t root_table_count = arch_mm_stage2_root_table_count();
+	for (uint8_t i = 0; i < root_table_count; ++i) {
+		all.push_back(get_table(
+			pa_add(ptable.root, i * sizeof(struct mm_page_table))));
+	}
+	return all;
+}
+
+} /* namespace mm_test */
diff --git a/src/mm_test.hh b/src/mm_test.hh
new file mode 100644
index 0000000..2e906aa
--- /dev/null
+++ b/src/mm_test.hh
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <span>
+#include <vector>
+
+namespace mm_test
+{
+std::vector<std::span<pte_t, MM_PTE_PER_PAGE>> get_ptable(
+	const struct mm_ptable &ptable);
+
+} /* namespace mm_test */
diff --git a/src/mpool.c b/src/mpool.c
new file mode 100644
index 0000000..df954e8
--- /dev/null
+++ b/src/mpool.c
@@ -0,0 +1,355 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/mpool.h"
+
+#include <stdbool.h>
+
+struct mpool_chunk {
+	struct mpool_chunk *next_chunk;
+	struct mpool_chunk *limit;
+};
+
+struct mpool_entry {
+	struct mpool_entry *next;
+};
+
+static bool mpool_locks_enabled = false;
+
+/**
+ * Enables the locks protecting memory pools. Before this function is called,
+ * the locks are disabled, that is, acquiring/releasing them is a no-op.
+ */
+void mpool_enable_locks(void)
+{
+	mpool_locks_enabled = true;
+}
+
+/**
+ * Acquires the lock protecting the given memory pool, if locks are enabled.
+ */
+static void mpool_lock(struct mpool *p)
+{
+	if (mpool_locks_enabled) {
+		sl_lock(&p->lock);
+	}
+}
+
+/**
+ * Releases the lock protecting the given memory pool, if locks are enabled.
+ */
+static void mpool_unlock(struct mpool *p)
+{
+	if (mpool_locks_enabled) {
+		sl_unlock(&p->lock);
+	}
+}
+
+/**
+ * Initialises the given memory pool with the given entry size, which must be
+ * at least the size of two pointers.
+ *
+ * All entries stored in the memory pool will be aligned to at least the entry
+ * size.
+ */
+void mpool_init(struct mpool *p, size_t entry_size)
+{
+	p->entry_size = entry_size;
+	p->chunk_list = NULL;
+	p->entry_list = NULL;
+	p->fallback = NULL;
+	sl_init(&p->lock);
+}
+
+/**
+ * Initialises the given memory pool by replicating the properties of `from`. It
+ * also pulls the chunk and free lists from `from`, consuming all its resources
+ * and making them available via the new memory pool.
+ */
+void mpool_init_from(struct mpool *p, struct mpool *from)
+{
+	mpool_init(p, from->entry_size);
+
+	mpool_lock(from);
+	p->chunk_list = from->chunk_list;
+	p->entry_list = from->entry_list;
+	p->fallback = from->fallback;
+
+	from->chunk_list = NULL;
+	from->entry_list = NULL;
+	from->fallback = NULL;
+	mpool_unlock(from);
+}
+
+/**
+ * Initialises the given memory pool with a fallback memory pool if this pool
+ * runs out of memory.
+ */
+void mpool_init_with_fallback(struct mpool *p, struct mpool *fallback)
+{
+	mpool_init(p, fallback->entry_size);
+	p->fallback = fallback;
+}
+
+/**
+ * Finishes the given memory pool, giving all free memory to the fallback pool
+ * if there is one.
+ */
+void mpool_fini(struct mpool *p)
+{
+	struct mpool_entry *entry;
+	struct mpool_chunk *chunk;
+
+	if (!p->fallback) {
+		return;
+	}
+
+	mpool_lock(p);
+
+	/* Merge the freelist into the fallback. */
+	entry = p->entry_list;
+	while (entry != NULL) {
+		void *ptr = entry;
+
+		entry = entry->next;
+		mpool_free(p->fallback, ptr);
+	}
+
+	/* Merge the chunk list into the fallback. */
+	chunk = p->chunk_list;
+	while (chunk != NULL) {
+		void *ptr = chunk;
+		size_t size = (uintptr_t)chunk->limit - (uintptr_t)chunk;
+
+		chunk = chunk->next_chunk;
+		mpool_add_chunk(p->fallback, ptr, size);
+	}
+
+	p->chunk_list = NULL;
+	p->entry_list = NULL;
+	p->fallback = NULL;
+
+	mpool_unlock(p);
+}
+
+/**
+ * Adds a contiguous chunk of memory to the given memory pool. The chunk will
+ * eventually be broken up into entries of the size held by the memory pool.
+ *
+ * Only the portions aligned to the entry size will be added to the pool.
+ *
+ * Returns true if at least a portion of the chunk was added to pool, or false
+ * if none of the buffer was usable in the pool.
+ */
+bool mpool_add_chunk(struct mpool *p, void *begin, size_t size)
+{
+	struct mpool_chunk *chunk;
+	uintptr_t new_begin;
+	uintptr_t new_end;
+
+	/* Round begin address up, and end address down. */
+	new_begin = ((uintptr_t)begin + p->entry_size - 1) / p->entry_size *
+		    p->entry_size;
+	new_end = ((uintptr_t)begin + size) / p->entry_size * p->entry_size;
+
+	/* Nothing to do if there isn't enough room for an entry. */
+	if (new_begin >= new_end || new_end - new_begin < p->entry_size) {
+		return false;
+	}
+
+	chunk = (struct mpool_chunk *)new_begin;
+	chunk->limit = (struct mpool_chunk *)new_end;
+
+	mpool_lock(p);
+	chunk->next_chunk = p->chunk_list;
+	p->chunk_list = chunk;
+	mpool_unlock(p);
+
+	return true;
+}
+
+/**
+ * Allocates an entry from the given memory pool, if one is available. The
+ * fallback will not be used even if there is one.
+ */
+static void *mpool_alloc_no_fallback(struct mpool *p)
+{
+	void *ret;
+	struct mpool_chunk *chunk;
+	struct mpool_chunk *new_chunk;
+
+	/* Fetch an entry from the free list if one is available. */
+	mpool_lock(p);
+	if (p->entry_list != NULL) {
+		struct mpool_entry *entry = p->entry_list;
+
+		p->entry_list = entry->next;
+		ret = entry;
+		goto exit;
+	}
+
+	/* There was no free list available. Try a chunk instead. */
+	chunk = p->chunk_list;
+	if (chunk == NULL) {
+		/* The chunk list is also empty, we're out of entries. */
+		ret = NULL;
+		goto exit;
+	}
+
+	new_chunk = (struct mpool_chunk *)((uintptr_t)chunk + p->entry_size);
+	if (new_chunk >= chunk->limit) {
+		p->chunk_list = chunk->next_chunk;
+	} else {
+		*new_chunk = *chunk;
+		p->chunk_list = new_chunk;
+	}
+
+	ret = chunk;
+
+exit:
+	mpool_unlock(p);
+
+	return ret;
+}
+
+/**
+ * Allocates an entry from the given memory pool, if one is available. If there
+ * isn't one available, try and allocate from the fallback if there is one.
+ */
+void *mpool_alloc(struct mpool *p)
+{
+	do {
+		void *ret = mpool_alloc_no_fallback(p);
+
+		if (ret != NULL) {
+			return ret;
+		}
+
+		p = p->fallback;
+	} while (p != NULL);
+
+	return NULL;
+}
+
+/**
+ * Frees an entry back into the memory pool, making it available for reuse.
+ *
+ * This is meant to be used for freeing single entries. To free multiple
+ * entries, one must call mpool_add_chunk instead.
+ */
+void mpool_free(struct mpool *p, void *ptr)
+{
+	struct mpool_entry *e = ptr;
+
+	/* Store the newly freed entry in the front of the free list. */
+	mpool_lock(p);
+	e->next = p->entry_list;
+	p->entry_list = e;
+	mpool_unlock(p);
+}
+
+/**
+ * Allocates a number of contiguous and aligned entries. If a suitable
+ * allocation could not be found, the fallback will not be used even if there is
+ * one.
+ */
+void *mpool_alloc_contiguous_no_fallback(struct mpool *p, size_t count,
+					 size_t align)
+{
+	struct mpool_chunk **prev;
+	void *ret = NULL;
+
+	align *= p->entry_size;
+
+	mpool_lock(p);
+
+	/*
+	 * Go through the chunk list in search of one with enough room for the
+	 * requested allocation
+	 */
+	prev = &p->chunk_list;
+	while (*prev != NULL) {
+		uintptr_t start;
+		struct mpool_chunk *new_chunk;
+		struct mpool_chunk *chunk = *prev;
+
+		/* Round start address up to the required alignment. */
+		start = (((uintptr_t)chunk + align - 1) / align) * align;
+
+		/*
+		 * Calculate where the new chunk would be if we consume the
+		 * requested number of entries. Then check if this chunk is big
+		 * enough to satisfy the request.
+		 */
+		new_chunk =
+			(struct mpool_chunk *)(start + (count * p->entry_size));
+		if (new_chunk <= chunk->limit) {
+			/* Remove the consumed area. */
+			if (new_chunk == chunk->limit) {
+				*prev = chunk->next_chunk;
+			} else {
+				*new_chunk = *chunk;
+				*prev = new_chunk;
+			}
+
+			/*
+			 * Add back the space consumed by the alignment
+			 * requirement, if it's big enough to fit an entry.
+			 */
+			if (start - (uintptr_t)chunk >= p->entry_size) {
+				chunk->next_chunk = *prev;
+				*prev = chunk;
+				chunk->limit = (struct mpool_chunk *)start;
+			}
+
+			ret = (void *)start;
+			break;
+		}
+
+		prev = &chunk->next_chunk;
+	}
+
+	mpool_unlock(p);
+
+	return ret;
+}
+
+/**
+ * Allocates a number of contiguous and aligned entries. This is a best-effort
+ * operation and only succeeds if such entries can be found in the chunks list
+ * or the chunks of the fallbacks (i.e., the entry list is never used to satisfy
+ * these allocations).
+ *
+ * The alignment is specified as the number of entries, that is, if `align` is
+ * 4, the alignment in bytes will be 4 * entry_size.
+ *
+ * The caller can enventually free the returned entries by calling
+ * mpool_add_chunk.
+ */
+void *mpool_alloc_contiguous(struct mpool *p, size_t count, size_t align)
+{
+	do {
+		void *ret = mpool_alloc_contiguous_no_fallback(p, count, align);
+
+		if (ret != NULL) {
+			return ret;
+		}
+
+		p = p->fallback;
+	} while (p != NULL);
+
+	return NULL;
+}
diff --git a/src/mpool_test.cc b/src/mpool_test.cc
new file mode 100644
index 0000000..53a45ed
--- /dev/null
+++ b/src/mpool_test.cc
@@ -0,0 +1,360 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+
+#include <gmock/gmock.h>
+
+extern "C" {
+#include "hf/mpool.h"
+}
+
+namespace
+{
+using ::testing::Eq;
+using ::testing::IsNull;
+using ::testing::NotNull;
+
+/**
+ * Checks that the given allocations come from the given chunks.
+ */
+bool check_allocs(std::vector<std::unique_ptr<char[]>>& chunks,
+		  std::vector<uintptr_t>& allocs, size_t entries_per_chunk,
+		  size_t entry_size)
+{
+	size_t i, j;
+
+	if (allocs.size() != chunks.size() * entries_per_chunk) {
+		return false;
+	}
+
+	sort(allocs.begin(), allocs.end());
+	sort(chunks.begin(), chunks.end(),
+	     [](const std::unique_ptr<char[]>& a,
+		const std::unique_ptr<char[]>& b) {
+		     return a.get() < b.get();
+	     });
+
+	for (i = 0; i < chunks.size(); i++) {
+		if ((uintptr_t)chunks[i].get() !=
+		    allocs[i * entries_per_chunk]) {
+			return false;
+		}
+
+		for (j = 1; j < entries_per_chunk; j++) {
+			size_t k = i * entries_per_chunk + j;
+			if (allocs[k] != allocs[k - 1] + entry_size) {
+				return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+/**
+ * Add chunks to the given mem pool and chunk vector.
+ */
+static void add_chunks(std::vector<std::unique_ptr<char[]>>& chunks,
+		       struct mpool* p, size_t count, size_t size)
+{
+	size_t i;
+
+	for (i = 0; i < count; i++) {
+		chunks.emplace_back(std::make_unique<char[]>(size));
+		mpool_add_chunk(p, chunks.back().get(), size);
+	}
+}
+
+/**
+ * Validates allocations from a memory pool.
+ */
+TEST(mpool, allocation)
+{
+	struct mpool p;
+	constexpr size_t entry_size = 16;
+	constexpr size_t entries_per_chunk = 10;
+	constexpr size_t chunk_count = 10;
+	std::vector<std::unique_ptr<char[]>> chunks;
+	std::vector<uintptr_t> allocs;
+	void* ret;
+
+	mpool_init(&p, entry_size);
+
+	/* Allocate from an empty pool. */
+	EXPECT_THAT(mpool_alloc(&p), IsNull());
+
+	/*
+	 * Add a chunk that is too small, it should be ignored, and allocation
+	 * should return NULL.
+	 */
+	mpool_add_chunk(&p, NULL, entry_size - 1);
+	EXPECT_THAT(mpool_alloc(&p), IsNull());
+
+	/* Allocate a number of chunks and add them to the pool. */
+	add_chunks(chunks, &p, chunk_count, entries_per_chunk * entry_size);
+
+	/* Allocate from the pool until we run out of memory. */
+	while ((ret = mpool_alloc(&p))) {
+		allocs.push_back((uintptr_t)ret);
+	}
+
+	/* Check that returned entries are within chunks that were added. */
+	ASSERT_THAT(check_allocs(chunks, allocs, entries_per_chunk, entry_size),
+		    true);
+}
+
+/**
+ * Validates frees into a memory pool.
+ */
+TEST(mpool, freeing)
+{
+	struct mpool p;
+	constexpr size_t entry_size = 16;
+	constexpr size_t entries_per_chunk = 12;
+	constexpr size_t chunk_count = 10;
+	std::vector<std::unique_ptr<char[]>> chunks;
+	std::vector<uintptr_t> allocs;
+	size_t i;
+	alignas(entry_size) char entry[entry_size];
+	void* ret;
+
+	mpool_init(&p, entry_size);
+
+	/* Allocate from an empty pool. */
+	EXPECT_THAT(mpool_alloc(&p), IsNull());
+
+	/* Free an entry into the pool, then allocate it back. */
+	mpool_free(&p, &entry[0]);
+	EXPECT_THAT(mpool_alloc(&p), (void*)&entry[0]);
+	EXPECT_THAT(mpool_alloc(&p), IsNull());
+
+	/* Allocate a number of chunks and add them to the pool. */
+	add_chunks(chunks, &p, chunk_count, entries_per_chunk * entry_size);
+
+	/*
+	 * Free again into the pool. Ensure that we get entry back on next
+	 * allocation instead of something from the chunks.
+	 */
+	mpool_free(&p, &entry[0]);
+	EXPECT_THAT(mpool_alloc(&p), (void*)&entry[0]);
+
+	/* Allocate from the pool until we run out of memory. */
+	while ((ret = mpool_alloc(&p))) {
+		allocs.push_back((uintptr_t)ret);
+	}
+
+	/*
+	 * Free again into the pool. Ensure that we get entry back on next
+	 * allocation instead of something from the chunks.
+	 */
+	mpool_free(&p, &entry[0]);
+	EXPECT_THAT(mpool_alloc(&p), (void*)&entry[0]);
+
+	/* Add entries back to the pool by freeing them. */
+	for (i = 0; i < allocs.size(); i++) {
+		mpool_free(&p, (void*)allocs[i]);
+	}
+	allocs.clear();
+
+	/* Allocate from the pool until we run out of memory. */
+	while ((ret = mpool_alloc(&p))) {
+		allocs.push_back((uintptr_t)ret);
+	}
+
+	/* Check that returned entries are within chunks that were added. */
+	ASSERT_THAT(check_allocs(chunks, allocs, entries_per_chunk, entry_size),
+		    true);
+}
+
+/**
+ * Initialises a memory pool from an existing one.
+ */
+TEST(mpool, init_from)
+{
+	struct mpool p, q;
+	constexpr size_t entry_size = 16;
+	constexpr size_t entries_per_chunk = 10;
+	constexpr size_t chunk_count = 10;
+	std::vector<std::unique_ptr<char[]>> chunks;
+	std::vector<uintptr_t> allocs;
+	size_t i;
+	void* ret;
+
+	mpool_init(&p, entry_size);
+
+	/* Allocate a number of chunks and add them to the pool. */
+	add_chunks(chunks, &p, chunk_count, entries_per_chunk * entry_size);
+
+	/* Allocate half of the elements. */
+	for (i = 0; i < entries_per_chunk * chunk_count / 2; i++) {
+		void* ret = mpool_alloc(&p);
+		ASSERT_THAT(ret, NotNull());
+		allocs.push_back((uintptr_t)ret);
+	}
+
+	/* Add entries back to the pool by freeing them. */
+	for (i = 0; i < allocs.size(); i++) {
+		mpool_free(&p, (void*)allocs[i]);
+	}
+	allocs.clear();
+
+	/* Initialise q from p. */
+	mpool_init_from(&q, &p);
+
+	/* Allocation from p must now fail. */
+	EXPECT_THAT(mpool_alloc(&p), IsNull());
+
+	/* Allocate from q until we run out of memory. */
+	while ((ret = mpool_alloc(&q))) {
+		allocs.push_back((uintptr_t)ret);
+	}
+
+	/* Check that returned entries are within chunks that were added. */
+	ASSERT_THAT(check_allocs(chunks, allocs, entries_per_chunk, entry_size),
+		    true);
+}
+
+/**
+ * Initialises a memory pool from an existing one.
+ */
+TEST(mpool, alloc_contiguous)
+{
+	struct mpool p;
+	constexpr size_t entry_size = 16;
+	constexpr size_t entries_per_chunk = 12;
+	constexpr size_t chunk_count = 10;
+	std::vector<std::unique_ptr<char[]>> chunks;
+	std::vector<uintptr_t> allocs;
+	size_t i;
+	void* ret;
+	uintptr_t next;
+
+	mpool_init(&p, entry_size);
+
+	/* Allocate a number of chunks and add them to the pool. */
+	add_chunks(chunks, &p, chunk_count, entries_per_chunk * entry_size);
+
+	/*
+	 * Allocate entries until the remaining chunk is aligned to 2 entries,
+	 * but not aligned to 4 entries.
+	 */
+	do {
+		ret = mpool_alloc(&p);
+		ASSERT_THAT(ret, NotNull());
+		allocs.push_back((uintptr_t)ret);
+		next = ((uintptr_t)ret / entry_size) + 1;
+	} while ((next % 4) != 2);
+
+	/* Allocate 5 entries with an alignment of 4. So two must be skipped. */
+	ret = mpool_alloc_contiguous(&p, 5, 4);
+	ASSERT_THAT(ret, NotNull());
+	ASSERT_THAT((uintptr_t)ret, (next + 2) * entry_size);
+	for (i = 0; i < 5; i++) {
+		allocs.push_back((uintptr_t)ret + i * entry_size);
+	}
+
+	/* Allocate a whole chunk. */
+	ret = mpool_alloc_contiguous(&p, entries_per_chunk, 1);
+	ASSERT_THAT(ret, NotNull());
+	for (i = 0; i < entries_per_chunk; i++) {
+		allocs.push_back((uintptr_t)ret + i * entry_size);
+	}
+
+	/* Allocate 2 entries that are already aligned. */
+	ret = mpool_alloc_contiguous(&p, 2, 1);
+	ASSERT_THAT(ret, NotNull());
+	allocs.push_back((uintptr_t)ret);
+	allocs.push_back((uintptr_t)ret + entry_size);
+
+	/* Allocate from p until we run out of memory. */
+	while ((ret = mpool_alloc(&p))) {
+		allocs.push_back((uintptr_t)ret);
+	}
+
+	/* Check that returned entries are within chunks that were added. */
+	ASSERT_THAT(check_allocs(chunks, allocs, entries_per_chunk, entry_size),
+		    true);
+}
+
+TEST(mpool, allocation_with_fallback)
+{
+	struct mpool fallback;
+	struct mpool p;
+	constexpr size_t entry_size = 16;
+	constexpr size_t entries_per_chunk = 10;
+	constexpr size_t chunk_count = 10;
+	std::vector<std::unique_ptr<char[]>> chunks;
+	std::vector<uintptr_t> allocs;
+	void* ret;
+
+	mpool_init(&fallback, entry_size);
+	mpool_init_with_fallback(&p, &fallback);
+
+	/* Allocate from an empty pool. */
+	EXPECT_THAT(mpool_alloc(&p), IsNull());
+
+	/* Allocate a number of chunks and add them to the fallback pool. */
+	add_chunks(chunks, &fallback, chunk_count,
+		   entries_per_chunk * entry_size);
+
+	/* Allocate from the pool until we run out of memory. */
+	while ((ret = mpool_alloc(&p))) {
+		allocs.push_back((uintptr_t)ret);
+	}
+
+	/* Check that returned entries are within chunks that were added. */
+	ASSERT_THAT(check_allocs(chunks, allocs, entries_per_chunk, entry_size),
+		    true);
+}
+
+TEST(mpool, free_with_fallback)
+{
+	struct mpool fallback;
+	struct mpool p;
+	constexpr size_t entry_size = 16;
+	constexpr size_t entries_per_chunk = 1;
+	constexpr size_t chunk_count = 1;
+	std::vector<std::unique_ptr<char[]>> chunks;
+	std::vector<uintptr_t> allocs;
+	void* ret;
+
+	mpool_init(&fallback, entry_size);
+	mpool_init_with_fallback(&p, &fallback);
+
+	/* Allocate a number of chunks and add them to the fallback pool. */
+	add_chunks(chunks, &fallback, chunk_count,
+		   entries_per_chunk * entry_size);
+
+	/* Allocate, making use of the fallback and free again. */
+	ret = mpool_alloc(&p);
+	mpool_free(&p, ret);
+
+	/* The entry is not available in the fallback. */
+	EXPECT_THAT(mpool_alloc(&fallback), IsNull());
+
+	/* The entry will be allocated by the local pool. */
+	EXPECT_THAT(mpool_alloc(&p), Eq(ret));
+
+	/* Return the memory to the local pool and then to the fallback. */
+	mpool_free(&p, ret);
+	mpool_fini(&p);
+
+	/* The fallback can now allocate the entry. */
+	EXPECT_THAT(mpool_alloc(&fallback), Eq(ret));
+}
+
+} /* namespace */
diff --git a/src/panic.c b/src/panic.c
new file mode 100644
index 0000000..9c0a4f6
--- /dev/null
+++ b/src/panic.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/panic.h"
+
+#include <stdarg.h>
+
+#include "hf/abort.h"
+#include "hf/dlog.h"
+
+/**
+ * Logs a reason before calling abort.
+ *
+ * TODO: Determine if we want to omit strings on non-debug builds.
+ */
+noreturn void panic(const char *fmt, ...)
+{
+	va_list args;
+
+	dlog("Panic: ");
+
+	va_start(args, fmt);
+	vdlog(fmt, args);
+	va_end(args);
+
+	dlog("\n");
+
+	abort();
+}
diff --git a/src/spci_architected_message.c b/src/spci_architected_message.c
new file mode 100644
index 0000000..06106d5
--- /dev/null
+++ b/src/spci_architected_message.c
@@ -0,0 +1,659 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/api.h"
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/spci_internal.h"
+#include "hf/std.h"
+#include "hf/vm.h"
+
+/**
+ * Obtain the next mode to apply to the two VMs.
+ *
+ * Returns true iff a state transition was found.
+ */
+static bool spci_msg_get_next_state(
+	const struct spci_mem_transitions *transitions,
+	uint32_t transition_count, uint32_t memory_to_attributes,
+	uint32_t orig_from_mode, uint32_t orig_to_mode, uint32_t *from_mode,
+	uint32_t *to_mode)
+{
+	const uint32_t state_mask =
+		MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
+	const uint32_t orig_from_state = orig_from_mode & state_mask;
+
+	for (uint32_t index = 0; index < transition_count; index++) {
+		uint32_t table_orig_from_mode =
+			transitions[index].orig_from_mode;
+		uint32_t table_orig_to_mode = transitions[index].orig_to_mode;
+
+		if (((orig_from_state) == table_orig_from_mode) &&
+		    ((orig_to_mode & state_mask) == table_orig_to_mode)) {
+			*to_mode = transitions[index].to_mode |
+				   memory_to_attributes;
+
+			*from_mode = transitions[index].from_mode |
+				     (~state_mask & orig_from_mode);
+
+			return true;
+		}
+	}
+	return false;
+}
+
+/**
+ * Verify that all pages have the same mode, that the starting mode
+ * constitutes a valid state and obtain the next mode to apply
+ * to the two VMs.
+ *
+ * Returns:
+ *  The error code false indicates that:
+ *   1) a state transition was not found;
+ *   2) the pages being shared do not have the same mode within the <to>
+ *     or <form> VMs;
+ *   3) The beginning and end IPAs are not page aligned;
+ *   4) The requested share type was not handled.
+ *  Success is indicated by true.
+ *
+ */
+static bool spci_msg_check_transition(struct vm *to, struct vm *from,
+				      uint32_t share_type,
+				      uint32_t *orig_from_mode,
+				      struct spci_memory_region *memory_region,
+				      uint32_t memory_to_attributes,
+				      uint32_t *from_mode, uint32_t *to_mode)
+{
+	uint32_t orig_to_mode;
+	const struct spci_mem_transitions *mem_transition_table;
+	uint32_t transition_table_size;
+	uint32_t i;
+
+	/*
+	 * TODO: Transition table does not currently consider the multiple
+	 * shared case.
+	 */
+	static const struct spci_mem_transitions donate_transitions[] = {
+		{
+			/* 1) {O-EA, !O-NA} -> {!O-NA, O-EA} */
+			.orig_from_mode = 0,
+			.orig_to_mode = MM_MODE_INVALID | MM_MODE_UNOWNED,
+			.from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED,
+			.to_mode = 0,
+		},
+		{
+			/* 2) {O-NA, !O-EA} -> {!O-NA, O-EA} */
+			.orig_from_mode = MM_MODE_INVALID,
+			.orig_to_mode = MM_MODE_UNOWNED,
+			.from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED,
+			.to_mode = 0,
+		},
+		{
+			/* 3) {O-SA, !O-SA} -> {!O-NA, O-EA} */
+			.orig_from_mode = MM_MODE_SHARED,
+			.orig_to_mode = MM_MODE_UNOWNED | MM_MODE_SHARED,
+			.from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED,
+			.to_mode = 0,
+		},
+		{
+			/*
+			 * Duplicate of 1) in order to cater for an alternative
+			 * representation of !O-NA:
+			 * (INVALID | UNOWNED | SHARED) and (INVALID | UNOWNED)
+			 * are both alternate representations of !O-NA.
+			 */
+			/* 4) {O-EA, !O-NA} -> {!O-NA, O-EA} */
+			.orig_from_mode = 0,
+			.orig_to_mode = MM_MODE_INVALID | MM_MODE_UNOWNED |
+					MM_MODE_SHARED,
+			.from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED |
+				     MM_MODE_SHARED,
+			.to_mode = 0,
+		},
+	};
+
+	static const uint32_t size_donate_transitions =
+		ARRAY_SIZE(donate_transitions);
+
+	/*
+	 * This data structure holds the allowed state transitions for the
+	 * "lend" state machine. In this state machine the owner keeps ownership
+	 * but loses access to the lent pages.
+	 */
+	static const struct spci_mem_transitions lend_transitions[] = {
+		{
+			/* 1) {O-EA, !O-NA} -> {O-NA, !O-EA} */
+			.orig_from_mode = 0,
+			.orig_to_mode = MM_MODE_INVALID | MM_MODE_UNOWNED |
+					MM_MODE_SHARED,
+			.from_mode = MM_MODE_INVALID,
+			.to_mode = MM_MODE_UNOWNED,
+		},
+		{
+			/*
+			 * Duplicate of 1) in order to cater for an alternative
+			 * representation of !O-NA:
+			 * (INVALID | UNOWNED | SHARED) and (INVALID | UNOWNED)
+			 * are both alternate representations of !O-NA.
+			 */
+			/* 2) {O-EA, !O-NA} -> {O-NA, !O-EA} */
+			.orig_from_mode = 0,
+			.orig_to_mode = MM_MODE_INVALID | MM_MODE_UNOWNED,
+			.from_mode = MM_MODE_INVALID,
+			.to_mode = MM_MODE_UNOWNED,
+		},
+	};
+
+	static const uint32_t size_lend_transitions =
+		ARRAY_SIZE(lend_transitions);
+
+	/*
+	 * This data structure holds the allowed state transitions for the
+	 * "share" state machine. In this state machine the owner keeps the
+	 * shared pages mapped on its stage2 table and keeps access as well.
+	 */
+	static const struct spci_mem_transitions share_transitions[] = {
+		{
+			/* 1) {O-EA, !O-NA} -> {O-SA, !O-SA} */
+			.orig_from_mode = 0,
+			.orig_to_mode = MM_MODE_INVALID | MM_MODE_UNOWNED |
+					MM_MODE_SHARED,
+			.from_mode = MM_MODE_SHARED,
+			.to_mode = MM_MODE_UNOWNED | MM_MODE_SHARED,
+		},
+		{
+			/*
+			 * Duplicate of 1) in order to cater for an alternative
+			 * representation of !O-NA:
+			 * (INVALID | UNOWNED | SHARED) and (INVALID | UNOWNED)
+			 * are both alternate representations of !O-NA.
+			 */
+			/* 2) {O-EA, !O-NA} -> {O-SA, !O-SA} */
+			.orig_from_mode = 0,
+			.orig_to_mode = MM_MODE_INVALID | MM_MODE_UNOWNED,
+			.from_mode = MM_MODE_SHARED,
+			.to_mode = MM_MODE_UNOWNED | MM_MODE_SHARED,
+		},
+	};
+
+	static const uint32_t size_share_transitions =
+		ARRAY_SIZE(share_transitions);
+
+	static const struct spci_mem_transitions relinquish_transitions[] = {
+		{
+			/* 1) {!O-EA, O-NA} -> {!O-NA, O-EA} */
+			.orig_from_mode = MM_MODE_UNOWNED,
+			.orig_to_mode = MM_MODE_INVALID,
+			.from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED |
+				     MM_MODE_SHARED,
+			.to_mode = 0,
+		},
+		{
+			/* 2) {!O-SA, O-SA} -> {!O-NA, O-EA} */
+			.orig_from_mode = MM_MODE_UNOWNED | MM_MODE_SHARED,
+			.orig_to_mode = MM_MODE_SHARED,
+			.from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED |
+				     MM_MODE_SHARED,
+			.to_mode = 0,
+		},
+	};
+
+	static const uint32_t size_relinquish_transitions =
+		ARRAY_SIZE(relinquish_transitions);
+
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	if (memory_region->constituent_count == 0) {
+		/*
+		 * Fail if there are no constituents. Otherwise
+		 * spci_msg_get_next_state would get an unitialised
+		 * *orig_from_mode and orig_to_mode.
+		 */
+		return false;
+	}
+
+	for (i = 0; i < memory_region->constituent_count; ++i) {
+		ipaddr_t begin = ipa_init(constituents[i].address);
+		size_t size = constituents[i].page_count * PAGE_SIZE;
+		ipaddr_t end = ipa_add(begin, size);
+		uint32_t current_from_mode;
+		uint32_t current_to_mode;
+
+		/* Fail if addresses are not page-aligned. */
+		if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
+		    !is_aligned(ipa_addr(end), PAGE_SIZE)) {
+			return false;
+		}
+
+		/*
+		 * Ensure that this constituent memory range is all mapped with
+		 * the same mode.
+		 */
+		if (!mm_vm_get_mode(&from->ptable, begin, end,
+				    &current_from_mode) ||
+		    !mm_vm_get_mode(&to->ptable, begin, end,
+				    &current_to_mode)) {
+			return false;
+		}
+
+		/*
+		 * Ensure that all constituents are mapped with the same mode.
+		 */
+		if (i == 0) {
+			*orig_from_mode = current_from_mode;
+			orig_to_mode = current_to_mode;
+		} else if (current_from_mode != *orig_from_mode ||
+			   current_to_mode != orig_to_mode) {
+			return false;
+		}
+	}
+
+	/* Ensure the address range is normal memory and not a device. */
+	if (*orig_from_mode & MM_MODE_D) {
+		return false;
+	}
+
+	switch (share_type) {
+	case SPCI_MSG_SEND_LEGACY_MEMORY_DONATE:
+		mem_transition_table = donate_transitions;
+		transition_table_size = size_donate_transitions;
+		break;
+
+	case SPCI_MSG_SEND_LEGACY_MEMORY_LEND:
+		mem_transition_table = lend_transitions;
+		transition_table_size = size_lend_transitions;
+		break;
+
+	case SPCI_MSG_SEND_LEGACY_MEMORY_SHARE:
+		mem_transition_table = share_transitions;
+		transition_table_size = size_share_transitions;
+		break;
+
+	case SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH:
+		mem_transition_table = relinquish_transitions;
+		transition_table_size = size_relinquish_transitions;
+		break;
+
+	default:
+		return false;
+	}
+
+	return spci_msg_get_next_state(mem_transition_table,
+				       transition_table_size,
+				       memory_to_attributes, *orig_from_mode,
+				       orig_to_mode, from_mode, to_mode);
+}
+
+/**
+ * Updates a VM's page table such that the given set of physical address ranges
+ * are mapped in the address space at the corresponding address ranges, in the
+ * mode provided.
+ *
+ * If commit is false, the page tables will be allocated from the mpool but no
+ * mappings will actually be updated. This function must always be called first
+ * with commit false to check that it will succeed before calling with commit
+ * true, to avoid leaving the page table in a half-updated state. To make a
+ * series of changes atomically you can call them all with commit false before
+ * calling them all with commit true.
+ *
+ * mm_vm_defrag should always be called after a series of page table updates,
+ * whether they succeed or fail.
+ *
+ * Returns true on success, or false if the update failed and no changes were
+ * made to memory mappings.
+ */
+static bool spci_region_group_identity_map(
+	struct vm_locked vm_locked, struct spci_memory_region *memory_region,
+	int mode, struct mpool *ppool, bool commit)
+{
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+	uint32_t memory_constituent_count = memory_region->constituent_count;
+
+	/* Iterate over the memory region constituents. */
+	for (uint32_t index = 0; index < memory_constituent_count; index++) {
+		size_t size = constituents[index].page_count * PAGE_SIZE;
+		paddr_t pa_begin =
+			pa_from_ipa(ipa_init(constituents[index].address));
+		paddr_t pa_end = pa_add(pa_begin, size);
+
+		if (commit) {
+			vm_identity_commit(vm_locked, pa_begin, pa_end, mode,
+					   ppool, NULL);
+		} else if (!vm_identity_prepare(vm_locked, pa_begin, pa_end,
+						mode, ppool)) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * Clears a region of physical memory by overwriting it with zeros. The data is
+ * flushed from the cache so the memory has been cleared across the system.
+ */
+static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool)
+{
+	/*
+	 * TODO: change this to a CPU local single page window rather than a
+	 *       global mapping of the whole range. Such an approach will limit
+	 *       the changes to stage-1 tables and will allow only local
+	 *       invalidation.
+	 */
+	bool ret;
+	struct mm_stage1_locked stage1_locked = mm_lock_stage1();
+	void *ptr =
+		mm_identity_map(stage1_locked, begin, end, MM_MODE_W, ppool);
+	size_t size = pa_difference(begin, end);
+
+	if (!ptr) {
+		/* TODO: partial defrag of failed range. */
+		/* Recover any memory consumed in failed mapping. */
+		mm_defrag(stage1_locked, ppool);
+		goto fail;
+	}
+
+	memset_s(ptr, size, 0, size);
+	arch_mm_flush_dcache(ptr, size);
+	mm_unmap(stage1_locked, begin, end, ppool);
+
+	ret = true;
+	goto out;
+
+fail:
+	ret = false;
+
+out:
+	mm_unlock_stage1(&stage1_locked);
+
+	return ret;
+}
+
+/**
+ * Clears a region of physical memory by overwriting it with zeros. The data is
+ * flushed from the cache so the memory has been cleared across the system.
+ */
+static bool spci_clear_memory_region(struct spci_memory_region *memory_region,
+				     struct mpool *api_page_pool)
+{
+	struct mpool local_page_pool;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+	uint32_t memory_constituent_count = memory_region->constituent_count;
+	struct mm_stage1_locked stage1_locked;
+	bool ret = false;
+
+	/*
+	 * Create a local pool so any freed memory can't be used by another
+	 * thread. This is to ensure each constituent that is mapped can be
+	 * unmapped again afterwards.
+	 */
+	mpool_init_with_fallback(&local_page_pool, api_page_pool);
+
+	/* Iterate over the memory region constituents. */
+	for (uint32_t i = 0; i < memory_constituent_count; ++i) {
+		size_t size = constituents[i].page_count * PAGE_SIZE;
+		paddr_t begin = pa_from_ipa(ipa_init(constituents[i].address));
+		paddr_t end = pa_add(begin, size);
+
+		if (!clear_memory(begin, end, &local_page_pool)) {
+			/*
+			 * api_clear_memory will defrag on failure, so no need
+			 * to do it here.
+			 */
+			goto out;
+		}
+	}
+
+	/*
+	 * Need to defrag after clearing, as it may have added extra mappings to
+	 * the stage 1 page table.
+	 */
+	stage1_locked = mm_lock_stage1();
+	mm_defrag(stage1_locked, &local_page_pool);
+	mm_unlock_stage1(&stage1_locked);
+
+	ret = true;
+
+out:
+	mpool_fini(&local_page_pool);
+	return ret;
+}
+
+/**
+ * Shares memory from the calling VM with another. The memory can be shared in
+ * different modes.
+ *
+ * This function requires the calling context to hold the <to> and <from> locks.
+ *
+ * Returns:
+ *  In case of error one of the following values is returned:
+ *   1) SPCI_INVALID_PARAMETERS - The endpoint provided parameters were
+ *     erroneous;
+ *   2) SPCI_NO_MEMORY - Hafnium did not have sufficient memory to complete
+ *     the request.
+ *  Success is indicated by SPCI_SUCCESS.
+ */
+static struct spci_value spci_share_memory(
+	struct vm_locked to_locked, struct vm_locked from_locked,
+	struct spci_memory_region *memory_region, uint32_t memory_to_attributes,
+	uint32_t share_type, struct mpool *api_page_pool)
+{
+	struct vm *to = to_locked.vm;
+	struct vm *from = from_locked.vm;
+	uint32_t orig_from_mode;
+	uint32_t from_mode;
+	uint32_t to_mode;
+	struct mpool local_page_pool;
+	struct spci_value ret;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	/*
+	 * Make sure constituents are properly aligned to a 64-bit boundary. If
+	 * not we would get alignment faults trying to read (64-bit) page
+	 * addresses.
+	 */
+	if (!is_aligned(constituents, 8)) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Disallow reflexive shares as this suggests an error in the VM. */
+	if (to == from) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/*
+	 * Check if the state transition is lawful for both VMs involved
+	 * in the memory exchange, ensure that all constituents of a memory
+	 * region being shared are at the same state.
+	 */
+	if (!spci_msg_check_transition(to, from, share_type, &orig_from_mode,
+				       memory_region, memory_to_attributes,
+				       &from_mode, &to_mode)) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/*
+	 * Create a local pool so any freed memory can't be used by another
+	 * thread. This is to ensure the original mapping can be restored if the
+	 * clear fails.
+	 */
+	mpool_init_with_fallback(&local_page_pool, api_page_pool);
+
+	/*
+	 * First reserve all required memory for the new page table entries in
+	 * both sender and recipient page tables without committing, to make
+	 * sure the entire operation will succeed without exhausting the page
+	 * pool.
+	 */
+	if (!spci_region_group_identity_map(from_locked, memory_region,
+					    from_mode, api_page_pool, false) ||
+	    !spci_region_group_identity_map(to_locked, memory_region, to_mode,
+					    api_page_pool, false)) {
+		/* TODO: partial defrag of failed range. */
+		ret = spci_error(SPCI_NO_MEMORY);
+		goto out;
+	}
+
+	/*
+	 * First update the mapping for the sender so there is no overlap with
+	 * the recipient. This won't allocate because the transaction was
+	 * already prepared above, but may free pages in the case that a whole
+	 * block is being unmapped that was previously partially mapped.
+	 */
+	CHECK(spci_region_group_identity_map(
+		from_locked, memory_region, from_mode, &local_page_pool, true));
+
+	/* Clear the memory so no VM or device can see the previous contents. */
+	if ((memory_region->flags & SPCI_MEMORY_REGION_FLAG_CLEAR) &&
+	    !spci_clear_memory_region(memory_region, api_page_pool)) {
+		/*
+		 * On failure, roll back by returning memory to the sender. This
+		 * may allocate pages which were previously freed into
+		 * `local_page_pool` by the call above, but will never allocate
+		 * more pages than that so can never fail.
+		 */
+		CHECK(spci_region_group_identity_map(from_locked, memory_region,
+						     orig_from_mode,
+						     &local_page_pool, true));
+
+		ret = spci_error(SPCI_NO_MEMORY);
+		goto out;
+	}
+
+	/*
+	 * Complete the transfer by mapping the memory into the recipient. This
+	 * won't allocate because the transaction was already prepared above, so
+	 * it doesn't need to use the `local_page_pool`.
+	 */
+	CHECK(spci_region_group_identity_map(to_locked, memory_region, to_mode,
+					     api_page_pool, true));
+
+	ret = (struct spci_value){.func = SPCI_SUCCESS_32};
+
+out:
+	mpool_fini(&local_page_pool);
+
+	/*
+	 * Tidy up the page tables by reclaiming failed mappings (if there was
+	 * an error) or merging entries into blocks where possible (on success).
+	 */
+	mm_vm_defrag(&to->ptable, api_page_pool);
+	mm_vm_defrag(&from->ptable, api_page_pool);
+
+	return ret;
+}
+
+/**
+ * Check if the message length and the number of memory region constituents
+ * match, if the check is correct call the memory sharing routine.
+ */
+static struct spci_value spci_validate_call_share_memory(
+	struct vm_locked to_locked, struct vm_locked from_locked,
+	struct spci_memory_region *memory_region, uint32_t memory_share_size,
+	uint32_t share_type, struct mpool *api_page_pool)
+{
+	uint32_t memory_to_attributes;
+	uint32_t attributes_size;
+	uint32_t constituents_size;
+
+	/*
+	 * Ensure the number of constituents are within the memory
+	 * bounds.
+	 */
+	attributes_size = sizeof(struct spci_memory_region_attributes) *
+			  memory_region->attribute_count;
+	constituents_size = sizeof(struct spci_memory_region_constituent) *
+			    memory_region->constituent_count;
+	if (memory_region->constituent_offset <
+		    sizeof(struct spci_memory_region) + attributes_size ||
+	    memory_share_size !=
+		    memory_region->constituent_offset + constituents_size) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* The sender must match the message sender. */
+	if (memory_region->sender != from_locked.vm->id) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* We only support a single recipient. */
+	if (memory_region->attribute_count != 1) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	/* The recipient must match the message recipient. */
+	if (memory_region->attributes[0].receiver != to_locked.vm->id) {
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	switch (share_type) {
+	case SPCI_MSG_SEND_LEGACY_MEMORY_DONATE:
+	case SPCI_MSG_SEND_LEGACY_MEMORY_LEND:
+	case SPCI_MSG_SEND_LEGACY_MEMORY_SHARE:
+		memory_to_attributes = spci_memory_attrs_to_mode(
+			memory_region->attributes[0].memory_attributes);
+		break;
+	case SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH:
+		memory_to_attributes = MM_MODE_R | MM_MODE_W | MM_MODE_X;
+		break;
+	default:
+		dlog("Invalid memory sharing message.\n");
+		return spci_error(SPCI_INVALID_PARAMETERS);
+	}
+
+	return spci_share_memory(to_locked, from_locked, memory_region,
+				 memory_to_attributes, share_type,
+				 api_page_pool);
+}
+
+/**
+ * Performs initial architected message information parsing. Calls the
+ * corresponding api functions implementing the functionality requested
+ * in the architected message.
+ */
+struct spci_value spci_msg_handle_architected_message(
+	struct vm_locked to_locked, struct vm_locked from_locked,
+	struct spci_memory_region *memory_region, uint32_t size,
+	uint32_t attributes, struct mpool *api_page_pool)
+{
+	uint32_t share_type = attributes & SPCI_MSG_SEND_LEGACY_MEMORY_MASK;
+	struct spci_value ret = spci_validate_call_share_memory(
+		to_locked, from_locked, memory_region, size, share_type,
+		api_page_pool);
+
+	/* Copy data to the destination Rx. */
+	/*
+	 * TODO: Translate the <from> IPA addresses to <to> IPA addresses.
+	 * Currently we assume identity mapping of the stage 2 translation.
+	 * Removing this assumption relies on a mechanism to handle scenarios
+	 * where the memory region fits in the source Tx buffer but cannot fit
+	 * in the destination Rx buffer. This mechanism will be defined at the
+	 * spec level.
+	 */
+	if (ret.func == SPCI_SUCCESS_32) {
+		memcpy_s(to_locked.vm->mailbox.recv, SPCI_MSG_PAYLOAD_MAX,
+			 memory_region, size);
+		to_locked.vm->mailbox.recv_size = size;
+		to_locked.vm->mailbox.recv_sender = from_locked.vm->id;
+		to_locked.vm->mailbox.recv_attributes = share_type;
+	}
+
+	return ret;
+}
diff --git a/src/std.c b/src/std.c
new file mode 100644
index 0000000..4576118
--- /dev/null
+++ b/src/std.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/std.h"
+
+#include "hf/check.h"
+
+/* Declare unsafe functions locally so they are not available globally. */
+void *memset(void *s, int c, size_t n);
+void *memcpy(void *dst, const void *src, size_t n);
+void *memmove(void *dst, const void *src, size_t n);
+
+/*
+ * As per the C11 specification, mem*_s() operations fill the destination buffer
+ * if runtime constraint validation fails, assuming that `dest` and `destsz`
+ * are both valid.
+ */
+#define CHECK_OR_FILL(cond, dest, destsz, ch)                               \
+	do {                                                                \
+		if (!(cond)) {                                              \
+			if ((dest) != NULL && (destsz) <= RSIZE_MAX) {      \
+				memset_s((dest), (destsz), (ch), (destsz)); \
+			}                                                   \
+			panic("%s failed: " #cond, __func__);               \
+		}                                                           \
+	} while (0)
+
+#define CHECK_OR_ZERO_FILL(cond, dest, destsz) \
+	CHECK_OR_FILL(cond, dest, destsz, '\0')
+
+void memset_s(void *dest, rsize_t destsz, int ch, rsize_t count)
+{
+	CHECK_OR_FILL(dest != NULL, dest, destsz, ch);
+
+	/* Check count <= destsz <= RSIZE_MAX. */
+	CHECK_OR_FILL(destsz <= RSIZE_MAX, dest, destsz, ch);
+	CHECK_OR_FILL(count <= destsz, dest, destsz, ch);
+
+	/*
+	 * Clang analyzer doesn't like us calling unsafe memory functions, so
+	 * make it ignore this call.
+	 */
+#ifndef __clang_analyzer__
+	memset(dest, ch, count);
+#endif
+}
+
+void memcpy_s(void *dest, rsize_t destsz, const void *src, rsize_t count)
+{
+	uintptr_t d = (uintptr_t)dest;
+	uintptr_t s = (uintptr_t)src;
+
+	CHECK_OR_ZERO_FILL(dest != NULL, dest, destsz);
+	CHECK_OR_ZERO_FILL(src != NULL, dest, destsz);
+
+	/* Check count <= destsz <= RSIZE_MAX. */
+	CHECK_OR_ZERO_FILL(destsz <= RSIZE_MAX, dest, destsz);
+	CHECK_OR_ZERO_FILL(count <= destsz, dest, destsz);
+
+	/*
+	 * Buffer overlap test.
+	 * case a) `d < s` implies `s >= d+count`
+	 * case b) `d > s` implies `d >= s+count`
+	 */
+	CHECK_OR_ZERO_FILL(d != s, dest, destsz);
+	CHECK_OR_ZERO_FILL(d < s || d >= (s + count), dest, destsz);
+	CHECK_OR_ZERO_FILL(d > s || s >= (d + count), dest, destsz);
+
+#ifndef __clang_analyzer__
+	memcpy(dest, src, count);
+#endif
+}
+
+void memmove_s(void *dest, rsize_t destsz, const void *src, rsize_t count)
+{
+	CHECK_OR_ZERO_FILL(dest != NULL, dest, destsz);
+	CHECK_OR_ZERO_FILL(src != NULL, dest, destsz);
+
+	/* Check count <= destsz <= RSIZE_MAX. */
+	CHECK_OR_ZERO_FILL(destsz <= RSIZE_MAX, dest, destsz);
+	CHECK_OR_ZERO_FILL(count <= destsz, dest, destsz);
+
+#ifndef __clang_analyzer__
+	memmove(dest, src, count);
+#endif
+}
+
+/**
+ * Finds the first occurrence of character `ch` in the first `count` bytes of
+ * memory pointed to by `ptr`.
+ *
+ * Returns NULL if `ch` is not found.
+ * Panics if `ptr` is NULL (undefined behaviour).
+ */
+void *memchr(const void *ptr, int ch, size_t count)
+{
+	size_t i;
+	const unsigned char *p = (const unsigned char *)ptr;
+
+	CHECK(ptr != NULL);
+
+	/* Iterate over at most `strsz` characters of `str`. */
+	for (i = 0; i < count; ++i) {
+		if (p[i] == (unsigned char)ch) {
+			return (void *)(&p[i]);
+		}
+	}
+
+	return NULL;
+}
+
+/**
+ * Returns the length of the null-terminated byte string `str`, examining at
+ * most `strsz` bytes.
+ *
+ * If `str` is a NULL pointer, it returns zero.
+ * If a NULL character is not found, it returns `strsz`.
+ */
+size_t strnlen_s(const char *str, size_t strsz)
+{
+	if (str == NULL) {
+		return 0;
+	}
+
+	for (size_t i = 0; i < strsz; ++i) {
+		if (str[i] == '\0') {
+			return i;
+		}
+	}
+
+	/* NULL character not found. */
+	return strsz;
+}
diff --git a/src/string.c b/src/string.c
new file mode 100644
index 0000000..4876a3f
--- /dev/null
+++ b/src/string.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/string.h"
+
+#include "hf/static_assert.h"
+#include "hf/std.h"
+
+void string_init_empty(struct string *str)
+{
+	static_assert(sizeof(str->data) >= 1, "String buffer too small");
+	str->data[0] = '\0';
+}
+
+/**
+ * Caller must guarantee that `data` points to a NULL-terminated string.
+ * The constructor checks that it fits into the internal buffer and copies
+ * the string there.
+ */
+enum string_return_code string_init(struct string *str, const char *data,
+				    size_t size)
+{
+	/*
+	 * Require that the value contains exactly one NULL character and that
+	 * it is the last byte.
+	 */
+	if (size < 1 || memchr(data, '\0', size) != &data[size - 1]) {
+		return STRING_ERROR_INVALID_INPUT;
+	}
+
+	if (size > sizeof(str->data)) {
+		return STRING_ERROR_TOO_LONG;
+	}
+
+	memcpy_s(str->data, sizeof(str->data), data, size);
+	return STRING_SUCCESS;
+}
+
+bool string_is_empty(const struct string *str)
+{
+	return str->data[0] == '\0';
+}
+
+const char *string_data(const struct string *str)
+{
+	return str->data;
+}
diff --git a/src/string_test.cc b/src/string_test.cc
new file mode 100644
index 0000000..f814636
--- /dev/null
+++ b/src/string_test.cc
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gmock/gmock.h>
+
+extern "C" {
+#include "hf/string.h"
+}
+
+namespace
+{
+TEST(string, valid)
+{
+	struct string str;
+	constexpr const char data[] = "test";
+
+	string_init_empty(&str);
+	ASSERT_TRUE(string_is_empty(&str));
+	ASSERT_STREQ(string_data(&str), "");
+
+	ASSERT_EQ(string_init(&str, data, sizeof(data)), STRING_SUCCESS);
+	ASSERT_FALSE(string_is_empty(&str));
+	ASSERT_STRNE(string_data(&str), "");
+	ASSERT_STREQ(string_data(&str), "test");
+}
+
+TEST(string, data_zero_size)
+{
+	struct string str;
+	constexpr const char data[] = "test";
+
+	ASSERT_EQ(string_init(&str, data, 0), STRING_ERROR_INVALID_INPUT);
+}
+
+TEST(string, data_no_null_terminator)
+{
+	struct string str;
+	constexpr const char data[] = {'t', 'e', 's', 't'};
+
+	ASSERT_EQ(string_init(&str, data, sizeof(data)),
+		  STRING_ERROR_INVALID_INPUT);
+}
+
+TEST(string, data_two_null_terminators)
+{
+	struct string str;
+	constexpr const char data[] = {'\0', 't', 'e', 's', 't', '\0'};
+
+	ASSERT_EQ(string_init(&str, data, sizeof(data)),
+		  STRING_ERROR_INVALID_INPUT);
+}
+
+} /* namespace */
diff --git a/src/vcpu.c b/src/vcpu.c
new file mode 100644
index 0000000..2f1b5fc
--- /dev/null
+++ b/src/vcpu.c
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/vcpu.h"
+
+#include "hf/check.h"
+#include "hf/dlog.h"
+#include "hf/std.h"
+#include "hf/vm.h"
+
+/**
+ * Locks the given vCPU and updates `locked` to hold the newly locked vCPU.
+ */
+struct vcpu_locked vcpu_lock(struct vcpu *vcpu)
+{
+	struct vcpu_locked locked = {
+		.vcpu = vcpu,
+	};
+
+	sl_lock(&vcpu->lock);
+
+	return locked;
+}
+
+/**
+ * Unlocks a vCPU previously locked with vpu_lock, and updates `locked` to
+ * reflect the fact that the vCPU is no longer locked.
+ */
+void vcpu_unlock(struct vcpu_locked *locked)
+{
+	sl_unlock(&locked->vcpu->lock);
+	locked->vcpu = NULL;
+}
+
+void vcpu_init(struct vcpu *vcpu, struct vm *vm)
+{
+	memset_s(vcpu, sizeof(*vcpu), 0, sizeof(*vcpu));
+	sl_init(&vcpu->lock);
+	vcpu->regs_available = true;
+	vcpu->vm = vm;
+	vcpu->state = VCPU_STATE_OFF;
+}
+
+/**
+ * Initialise the registers for the given vCPU and set the state to
+ * VCPU_STATE_READY. The caller must hold the vCPU lock while calling this.
+ */
+void vcpu_on(struct vcpu_locked vcpu, ipaddr_t entry, uintreg_t arg)
+{
+	arch_regs_set_pc_arg(&vcpu.vcpu->regs, entry, arg);
+	vcpu.vcpu->state = VCPU_STATE_READY;
+}
+
+spci_vcpu_index_t vcpu_index(const struct vcpu *vcpu)
+{
+	size_t index = vcpu - vcpu->vm->vcpus;
+
+	CHECK(index < UINT16_MAX);
+	return index;
+}
+
+/**
+ * Check whether the given vcpu_state is an off state, for the purpose of
+ * turning vCPUs on and off. Note that aborted still counts as on in this
+ * context.
+ */
+bool vcpu_is_off(struct vcpu_locked vcpu)
+{
+	switch (vcpu.vcpu->state) {
+	case VCPU_STATE_OFF:
+		return true;
+	case VCPU_STATE_READY:
+	case VCPU_STATE_RUNNING:
+	case VCPU_STATE_BLOCKED_MAILBOX:
+	case VCPU_STATE_BLOCKED_INTERRUPT:
+	case VCPU_STATE_ABORTED:
+		/*
+		 * Aborted still counts as ON for the purposes of PSCI,
+		 * because according to the PSCI specification (section
+		 * 5.7.1) a core is only considered to be off if it has
+		 * been turned off with a CPU_OFF call or hasn't yet
+		 * been turned on with a CPU_ON call.
+		 */
+		return false;
+	}
+}
+
+/**
+ * Starts a vCPU of a secondary VM.
+ *
+ * Returns true if the secondary was reset and started, or false if it was
+ * already on and so nothing was done.
+ */
+bool vcpu_secondary_reset_and_start(struct vcpu *vcpu, ipaddr_t entry,
+				    uintreg_t arg)
+{
+	struct vcpu_locked vcpu_locked;
+	struct vm *vm = vcpu->vm;
+	bool vcpu_was_off;
+
+	CHECK(vm->id != HF_PRIMARY_VM_ID);
+
+	vcpu_locked = vcpu_lock(vcpu);
+	vcpu_was_off = vcpu_is_off(vcpu_locked);
+	if (vcpu_was_off) {
+		/*
+		 * Set vCPU registers to a clean state ready for boot. As this
+		 * is a secondary which can migrate between pCPUs, the ID of the
+		 * vCPU is defined as the index and does not match the ID of the
+		 * pCPU it is running on.
+		 */
+		arch_regs_reset(vcpu);
+		vcpu_on(vcpu_locked, entry, arg);
+	}
+	vcpu_unlock(&vcpu_locked);
+
+	return vcpu_was_off;
+}
+
+/**
+ * Handles a page fault. It does so by determining if it's a legitimate or
+ * spurious fault, and recovering from the latter.
+ *
+ * Returns true if the caller should resume the current vCPU, or false if its VM
+ * should be aborted.
+ */
+bool vcpu_handle_page_fault(const struct vcpu *current,
+			    struct vcpu_fault_info *f)
+{
+	struct vm *vm = current->vm;
+	uint32_t mode;
+	uint32_t mask = f->mode | MM_MODE_INVALID;
+	bool resume;
+
+	sl_lock(&vm->lock);
+
+	/*
+	 * Check if this is a legitimate fault, i.e., if the page table doesn't
+	 * allow the access attempted by the VM.
+	 *
+	 * Otherwise, this is a spurious fault, likely because another CPU is
+	 * updating the page table. It is responsible for issuing global TLB
+	 * invalidations while holding the VM lock, so we don't need to do
+	 * anything else to recover from it. (Acquiring/releasing the lock
+	 * ensured that the invalidations have completed.)
+	 */
+	resume = mm_vm_get_mode(&vm->ptable, f->ipaddr, ipa_add(f->ipaddr, 1),
+				&mode) &&
+		 (mode & mask) == f->mode;
+
+	sl_unlock(&vm->lock);
+
+	if (!resume) {
+		dlog("Stage-2 page fault: pc=%#x, vmid=%u, vcpu=%u, "
+		     "vaddr=%#x, ipaddr=%#x, mode=%#x\n",
+		     f->pc, vm->id, vcpu_index(current), f->vaddr, f->ipaddr,
+		     f->mode);
+	}
+
+	return resume;
+}
diff --git a/src/vm.c b/src/vm.c
new file mode 100644
index 0000000..0f36e21
--- /dev/null
+++ b/src/vm.c
@@ -0,0 +1,255 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/vm.h"
+
+#include "hf/api.h"
+#include "hf/check.h"
+#include "hf/cpu.h"
+#include "hf/layout.h"
+#include "hf/plat/iommu.h"
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+static struct vm vms[MAX_VMS];
+static spci_vm_count_t vm_count;
+
+bool vm_init(spci_vcpu_count_t vcpu_count, struct mpool *ppool,
+	     struct vm **new_vm)
+{
+	uint32_t i;
+	struct vm *vm;
+
+	if (vm_count >= MAX_VMS) {
+		return false;
+	}
+
+	vm = &vms[vm_count];
+
+	memset_s(vm, sizeof(*vm), 0, sizeof(*vm));
+
+	list_init(&vm->mailbox.waiter_list);
+	list_init(&vm->mailbox.ready_list);
+	sl_init(&vm->lock);
+
+	/* Generate IDs based on an offset, as low IDs e.g., 0, are reserved */
+	vm->id = vm_count + HF_VM_ID_OFFSET;
+	vm->vcpu_count = vcpu_count;
+	vm->mailbox.state = MAILBOX_STATE_EMPTY;
+	atomic_init(&vm->aborting, false);
+
+	if (!mm_vm_init(&vm->ptable, ppool)) {
+		return false;
+	}
+
+	/* Initialise waiter entries. */
+	for (i = 0; i < MAX_VMS; i++) {
+		vm->wait_entries[i].waiting_vm = vm;
+		list_init(&vm->wait_entries[i].wait_links);
+		list_init(&vm->wait_entries[i].ready_links);
+	}
+
+	/* Do basic initialization of vCPUs. */
+	for (i = 0; i < vcpu_count; i++) {
+		vcpu_init(vm_get_vcpu(vm, i), vm);
+	}
+
+	++vm_count;
+	*new_vm = vm;
+
+	return true;
+}
+
+spci_vm_count_t vm_get_count(void)
+{
+	return vm_count;
+}
+
+struct vm *vm_find(spci_vm_id_t id)
+{
+	uint16_t index;
+
+	/* Check that this is not a reserved ID. */
+	if (id < HF_VM_ID_OFFSET) {
+		return NULL;
+	}
+
+	index = id - HF_VM_ID_OFFSET;
+
+	/* Ensure the VM is initialized. */
+	if (index >= vm_count) {
+		return NULL;
+	}
+
+	return &vms[index];
+}
+
+/**
+ * Locks the given VM and updates `locked` to hold the newly locked VM.
+ */
+struct vm_locked vm_lock(struct vm *vm)
+{
+	struct vm_locked locked = {
+		.vm = vm,
+	};
+
+	sl_lock(&vm->lock);
+
+	return locked;
+}
+
+/**
+ * Locks two VMs ensuring that the locking order is according to the locks'
+ * addresses.
+ */
+struct two_vm_locked vm_lock_both(struct vm *vm1, struct vm *vm2)
+{
+	struct two_vm_locked dual_lock;
+
+	sl_lock_both(&vm1->lock, &vm2->lock);
+	dual_lock.vm1.vm = vm1;
+	dual_lock.vm2.vm = vm2;
+
+	return dual_lock;
+}
+
+/**
+ * Unlocks a VM previously locked with vm_lock, and updates `locked` to reflect
+ * the fact that the VM is no longer locked.
+ */
+void vm_unlock(struct vm_locked *locked)
+{
+	sl_unlock(&locked->vm->lock);
+	locked->vm = NULL;
+}
+
+/**
+ * Get the vCPU with the given index from the given VM.
+ * This assumes the index is valid, i.e. less than vm->vcpu_count.
+ */
+struct vcpu *vm_get_vcpu(struct vm *vm, spci_vcpu_index_t vcpu_index)
+{
+	CHECK(vcpu_index < vm->vcpu_count);
+	return &vm->vcpus[vcpu_index];
+}
+
+/**
+ * Gets `vm`'s wait entry for waiting on the `for_vm`.
+ */
+struct wait_entry *vm_get_wait_entry(struct vm *vm, spci_vm_id_t for_vm)
+{
+	uint16_t index;
+
+	CHECK(for_vm >= HF_VM_ID_OFFSET);
+	index = for_vm - HF_VM_ID_OFFSET;
+	CHECK(index < MAX_VMS);
+
+	return &vm->wait_entries[index];
+}
+
+/**
+ * Gets the ID of the VM which the given VM's wait entry is for.
+ */
+spci_vm_id_t vm_id_for_wait_entry(struct vm *vm, struct wait_entry *entry)
+{
+	uint16_t index = entry - vm->wait_entries;
+
+	return index + HF_VM_ID_OFFSET;
+}
+
+/**
+ * Map a range of addresses to the VM in both the MMU and the IOMMU.
+ *
+ * mm_vm_defrag should always be called after a series of page table updates,
+ * whether they succeed or fail. This is because on failure extra page table
+ * entries may have been allocated and then not used, while on success it may be
+ * possible to compact the page table by merging several entries into a block.
+ *
+ * Returns true on success, or false if the update failed and no changes were
+ * made.
+ *
+ */
+bool vm_identity_map(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+		     uint32_t mode, struct mpool *ppool, ipaddr_t *ipa)
+{
+	if (!vm_identity_prepare(vm_locked, begin, end, mode, ppool)) {
+		return false;
+	}
+
+	vm_identity_commit(vm_locked, begin, end, mode, ppool, ipa);
+
+	return true;
+}
+
+/**
+ * Prepares the given VM for the given address mapping such that it will be able
+ * to commit the change without failure.
+ *
+ * In particular, multiple calls to this function will result in the
+ * corresponding calls to commit the changes to succeed.
+ *
+ * Returns true on success, or false if the update failed and no changes were
+ * made.
+ */
+bool vm_identity_prepare(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+			 uint32_t mode, struct mpool *ppool)
+{
+	return mm_vm_identity_prepare(&vm_locked.vm->ptable, begin, end, mode,
+				      ppool);
+}
+
+/**
+ * Commits the given address mapping to the VM assuming the operation cannot
+ * fail. `vm_identity_prepare` must used correctly before this to ensure
+ * this condition.
+ */
+void vm_identity_commit(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+			uint32_t mode, struct mpool *ppool, ipaddr_t *ipa)
+{
+	mm_vm_identity_commit(&vm_locked.vm->ptable, begin, end, mode, ppool,
+			      ipa);
+	plat_iommu_identity_map(vm_locked, begin, end, mode);
+}
+
+/**
+ * Unmap a range of addresses from the VM.
+ *
+ * Returns true on success, or false if the update failed and no changes were
+ * made.
+ */
+bool vm_unmap(struct vm_locked vm_locked, paddr_t begin, paddr_t end,
+	      struct mpool *ppool)
+{
+	uint32_t mode = MM_MODE_UNMAPPED_MASK;
+
+	return vm_identity_map(vm_locked, begin, end, mode, ppool, NULL);
+}
+
+/**
+ * Unmaps the hypervisor pages from the given page table.
+ */
+bool vm_unmap_hypervisor(struct vm_locked vm_locked, struct mpool *ppool)
+{
+	/* TODO: If we add pages dynamically, they must be included here too. */
+	return vm_unmap(vm_locked, layout_text_begin(), layout_text_end(),
+			ppool) &&
+	       vm_unmap(vm_locked, layout_rodata_begin(), layout_rodata_end(),
+			ppool) &&
+	       vm_unmap(vm_locked, layout_data_begin(), layout_data_end(),
+			ppool);
+}
diff --git a/src/vm_test.cc b/src/vm_test.cc
new file mode 100644
index 0000000..fb33ec4
--- /dev/null
+++ b/src/vm_test.cc
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gmock/gmock.h>
+
+extern "C" {
+#include "hf/mpool.h"
+#include "hf/vm.h"
+}
+
+#include <memory>
+#include <span>
+#include <vector>
+
+#include "mm_test.hh"
+
+namespace
+{
+using namespace ::std::placeholders;
+
+using ::testing::AllOf;
+using ::testing::Each;
+using ::testing::SizeIs;
+
+using struct_vm = struct vm;
+
+constexpr size_t TEST_HEAP_SIZE = PAGE_SIZE * 16;
+const int TOP_LEVEL = arch_mm_stage2_max_level();
+
+class vm : public ::testing::Test
+{
+	void SetUp() override
+	{
+		/*
+		 * TODO: replace with direct use of stdlib allocator so
+		 * sanitizers are more effective.
+		 */
+		test_heap = std::make_unique<uint8_t[]>(TEST_HEAP_SIZE);
+		mpool_init(&ppool, sizeof(struct mm_page_table));
+		mpool_add_chunk(&ppool, test_heap.get(), TEST_HEAP_SIZE);
+	}
+
+	std::unique_ptr<uint8_t[]> test_heap;
+
+       protected:
+	struct mpool ppool;
+};
+
+/**
+ * If nothing is mapped, unmapping the hypervisor has no effect.
+ */
+TEST_F(vm, vm_unmap_hypervisor_not_mapped)
+{
+	struct_vm *vm;
+	struct vm_locked vm_locked;
+
+	vm_init(1, &ppool, &vm);
+	vm_locked = vm_lock(vm);
+	ASSERT_TRUE(mm_vm_init(&vm->ptable, &ppool));
+	EXPECT_TRUE(vm_unmap_hypervisor(vm_locked, &ppool));
+	EXPECT_THAT(
+		mm_test::get_ptable(vm->ptable),
+		AllOf(SizeIs(4), Each(Each(arch_mm_absent_pte(TOP_LEVEL)))));
+	mm_vm_fini(&vm->ptable, &ppool);
+	vm_unlock(&vm_locked);
+}
+
+} /* namespace */
diff --git a/test/arch/BUILD.gn b/test/arch/BUILD.gn
new file mode 100644
index 0000000..c770dee
--- /dev/null
+++ b/test/arch/BUILD.gn
@@ -0,0 +1,38 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+import("//build/toolchain/platform.gni")
+
+group("arch") {
+  testonly = true
+
+  deps = [
+    ":arch_test",
+  ]
+}
+
+hypervisor("arch_test") {
+  testonly = true
+
+  sources = [
+    "dlog_test.c",
+    "mm_test.c",
+  ]
+
+  deps = [
+    "//src/arch/${plat_arch}:arch",
+    "//test/hftest:hftest_hypervisor",
+  ]
+}
diff --git a/test/arch/dlog_test.c b/test/arch/dlog_test.c
new file mode 100644
index 0000000..f31ba26
--- /dev/null
+++ b/test/arch/dlog_test.c
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/dlog.h"
+
+#include "test/hftest.h"
+
+/**
+ * Test that logs are written to the buffer, and the rest is empty.
+ */
+TEST(dlog, log_buffer)
+{
+	const char test_string[] = "Test string\n";
+
+	dlog(test_string);
+	ASSERT_EQ(strcmp(test_string, dlog_buffer), 0);
+	/* The \0 at the end shouldn't be counted. */
+	ASSERT_EQ(dlog_buffer_offset, sizeof(test_string) - 1);
+	for (int i = sizeof(test_string) - 1; i < DLOG_BUFFER_SIZE; ++i) {
+		EXPECT_EQ(dlog_buffer[i], '\0');
+	}
+}
diff --git a/test/arch/mm_test.c b/test/arch/mm_test.c
new file mode 100644
index 0000000..bf0c186
--- /dev/null
+++ b/test/arch/mm_test.c
@@ -0,0 +1,198 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/mm.h"
+
+#include "hf/arch/mm.h"
+
+#include "test/hftest.h"
+
+/** There must be at least two levels in the page table.  */
+#define MAX_LEVEL_LOWER_BOUND 1
+
+/**
+ * This is the number of levels that are tested and is constrained as it
+ * controls the depth of recursion in the memory management code.
+ */
+#define MAX_LEVEL_UPPER_BOUND 3
+
+/** X macro to expand tests for all levels. */
+#define EXPAND_LEVEL_TESTS \
+	LEVEL_TEST(0)      \
+	LEVEL_TEST(1)      \
+	LEVEL_TEST(2)      \
+	LEVEL_TEST(3)
+
+/* TODO: work out how to run these test against the host fake arch. */
+
+/**
+ * A block must be allowed at level 0 as this is the level which represents
+ * pages.
+ */
+TEST(arch_mm, block_allowed_at_level0)
+{
+	ASSERT_TRUE(arch_mm_is_block_allowed(0));
+}
+
+/**
+ * The maximum level must be within acceptable bounds.
+ */
+TEST(arch_mm, max_level_stage1)
+{
+	uint8_t max_level = arch_mm_stage1_max_level();
+	EXPECT_GE(max_level, MAX_LEVEL_LOWER_BOUND);
+	EXPECT_LE(max_level, MAX_LEVEL_UPPER_BOUND);
+}
+
+/* TODO: initialize arch_mm and check max level of stage-2. */
+
+/**
+ * An absent entry is not present, valid, a block nor a table.
+ */
+#define LEVEL_TEST(lvl)                                                  \
+	TEST(arch_mm, absent_properties_level##lvl)                      \
+	{                                                                \
+		uint8_t level = lvl;                                     \
+		pte_t absent_pte;                                        \
+                                                                         \
+		absent_pte = arch_mm_absent_pte(level);                  \
+                                                                         \
+		EXPECT_FALSE(arch_mm_pte_is_present(absent_pte, level)); \
+		EXPECT_FALSE(arch_mm_pte_is_valid(absent_pte, level));   \
+		EXPECT_FALSE(arch_mm_pte_is_block(absent_pte, level));   \
+		EXPECT_FALSE(arch_mm_pte_is_table(absent_pte, level));   \
+	}
+EXPAND_LEVEL_TESTS
+#undef LEVEL_TEST
+
+/**
+ * An invalid block is present and mutually exclusive from a table.
+ */
+#define LEVEL_TEST(lvl)                                                       \
+	TEST(arch_mm, invalid_block_properties_level##lvl)                    \
+	{                                                                     \
+		uint8_t level = lvl;                                          \
+		uint64_t attrs =                                              \
+			arch_mm_mode_to_stage2_attrs(MM_MODE_INVALID);        \
+		pte_t block_pte;                                              \
+                                                                              \
+		/* Test doesn't apply if a block is not allowed. */           \
+		if (!arch_mm_is_block_allowed(level)) {                       \
+			return;                                               \
+		}                                                             \
+                                                                              \
+		block_pte = arch_mm_block_pte(level, pa_init(PAGE_SIZE * 19), \
+					      attrs);                         \
+                                                                              \
+		EXPECT_TRUE(arch_mm_pte_is_present(block_pte, level));        \
+		EXPECT_FALSE(arch_mm_pte_is_valid(block_pte, level));         \
+		EXPECT_TRUE(arch_mm_pte_is_block(block_pte, level));          \
+		EXPECT_FALSE(arch_mm_pte_is_table(block_pte, level));         \
+	}
+EXPAND_LEVEL_TESTS
+#undef LEVEL_TEST
+
+/**
+ * A valid block is present and mutually exclusive from a table.
+ */
+#define LEVEL_TEST(lvl)                                                \
+	TEST(arch_mm, valid_block_properties_level##lvl)               \
+	{                                                              \
+		uint8_t level = lvl;                                   \
+		uint64_t attrs = arch_mm_mode_to_stage2_attrs(0);      \
+		pte_t block_pte;                                       \
+                                                                       \
+		/* Test doesn't apply if a block is not allowed. */    \
+		if (!arch_mm_is_block_allowed(level)) {                \
+			return;                                        \
+		}                                                      \
+                                                                       \
+		block_pte = arch_mm_block_pte(                         \
+			level, pa_init(PAGE_SIZE * 12345678), attrs);  \
+                                                                       \
+		EXPECT_TRUE(arch_mm_pte_is_present(block_pte, level)); \
+		EXPECT_TRUE(arch_mm_pte_is_valid(block_pte, level));   \
+		EXPECT_TRUE(arch_mm_pte_is_block(block_pte, level));   \
+		EXPECT_FALSE(arch_mm_pte_is_table(block_pte, level));  \
+	}
+EXPAND_LEVEL_TESTS
+#undef LEVEL_TEST
+
+/**
+ * A table is present, valid and mutually exclusive from a block.
+ */
+#define LEVEL_TEST(lvl)                                                        \
+	TEST(arch_mm, table_properties_level##lvl)                             \
+	{                                                                      \
+		uint8_t level = lvl;                                           \
+		pte_t table_pte;                                               \
+                                                                               \
+		/* Test doesn't apply to level 0 as there can't be a table. */ \
+		if (level == 0) {                                              \
+			return;                                                \
+		}                                                              \
+                                                                               \
+		table_pte = arch_mm_table_pte(level,                           \
+					      pa_init(PAGE_SIZE * 999999999)); \
+                                                                               \
+		EXPECT_TRUE(arch_mm_pte_is_present(table_pte, level));         \
+		EXPECT_TRUE(arch_mm_pte_is_valid(table_pte, level));           \
+		EXPECT_FALSE(arch_mm_pte_is_block(table_pte, level));          \
+		EXPECT_TRUE(arch_mm_pte_is_table(table_pte, level));           \
+	}
+EXPAND_LEVEL_TESTS
+#undef LEVEL_TEST
+
+/**
+ * The address and attributes of a block must be preserved when encoding and
+ * decoding.
+ */
+#define LEVEL_TEST(lvl)                                                      \
+	TEST(arch_mm, block_addr_and_attrs_preserved_level##lvl)             \
+	{                                                                    \
+		uint8_t level = lvl;                                         \
+		paddr_t addr;                                                \
+		uint64_t attrs;                                              \
+		pte_t block_pte;                                             \
+                                                                             \
+		/* Test doesn't apply if a block is not allowed. */          \
+		if (!arch_mm_is_block_allowed(level)) {                      \
+			return;                                              \
+		}                                                            \
+                                                                             \
+		addr = pa_init(0);                                           \
+		attrs = arch_mm_mode_to_stage2_attrs(0);                     \
+		block_pte = arch_mm_block_pte(level, addr, attrs);           \
+		EXPECT_EQ(arch_mm_pte_attrs(block_pte, level), attrs);       \
+		EXPECT_EQ(pa_addr(arch_mm_block_from_pte(block_pte, level)), \
+			  pa_addr(addr));                                    \
+                                                                             \
+		addr = pa_init(PAGE_SIZE * 17);                              \
+		attrs = arch_mm_mode_to_stage2_attrs(MM_MODE_INVALID);       \
+		block_pte = arch_mm_block_pte(level, addr, attrs);           \
+		EXPECT_EQ(arch_mm_pte_attrs(block_pte, level), attrs);       \
+		EXPECT_EQ(pa_addr(arch_mm_block_from_pte(block_pte, level)), \
+			  pa_addr(addr));                                    \
+                                                                             \
+		addr = pa_init(PAGE_SIZE * 500);                             \
+		attrs = arch_mm_mode_to_stage2_attrs(MM_MODE_R | MM_MODE_W); \
+		block_pte = arch_mm_block_pte(level, addr, attrs);           \
+		EXPECT_EQ(arch_mm_pte_attrs(block_pte, level), attrs);       \
+		EXPECT_EQ(pa_addr(arch_mm_block_from_pte(block_pte, level)), \
+			  pa_addr(addr));                                    \
+	}
+EXPAND_LEVEL_TESTS
+#undef LEVEL_TEST
diff --git a/test/hftest/BUILD.gn b/test/hftest/BUILD.gn
new file mode 100644
index 0000000..d65397b
--- /dev/null
+++ b/test/hftest/BUILD.gn
@@ -0,0 +1,155 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/platform.gni")
+
+config("hftest_config") {
+  include_dirs = [ "//test/inc" ]
+}
+
+# Testing framework for a primary VM.
+source_set("hftest_primary_vm") {
+  testonly = true
+
+  public_configs = [ ":hftest_config" ]
+
+  deps = [
+    ":hftest_standalone",
+    "//vmlib/${plat_arch}:call",
+  ]
+}
+
+# Testing framework for a secondary VM. It's currently just a slave VM and
+# can't affect the tests directly.
+source_set("hftest_secondary_vm") {
+  testonly = true
+
+  public_configs = [ ":hftest_config" ]
+
+  sources = [
+    "service.c",
+  ]
+
+  deps = [
+    ":mm",
+    ":power_mgmt",
+    "//src:dlog",
+    "//src:memiter",
+    "//src:panic",
+    "//src:std",
+    "//src/arch/${plat_arch}:entry",
+    "//src/arch/${plat_arch}/hftest:entry",
+    "//src/arch/${plat_arch}/hftest:power_mgmt",
+    "//vmlib/${plat_arch}:call",
+  ]
+}
+
+# Testing framework for a hypervisor.
+source_set("hftest_hypervisor") {
+  testonly = true
+  public_configs = [ ":hftest_config" ]
+  deps = [
+    ":hftest_standalone",
+  ]
+}
+
+# Testing framework for tests running under Linux in the primary VM.
+source_set("hftest_linux") {
+  testonly = true
+  public_configs = [ ":hftest_config" ]
+
+  sources = [
+    "linux_main.c",
+  ]
+
+  deps = [
+    ":common",
+    "//src:dlog",
+    "//src:memiter",
+    "//src/arch/${plat_arch}/hftest:power_mgmt",
+  ]
+}
+
+source_set("hftest_standalone") {
+  visibility = [ ":*" ]
+  testonly = true
+
+  public_configs = [ ":hftest_config" ]
+
+  sources = [
+    "standalone_main.c",
+  ]
+
+  deps = [
+    ":common",
+    ":mm",
+    ":power_mgmt",
+    "//src:dlog",
+    "//src:fdt",
+    "//src:memiter",
+    "//src/arch/${plat_arch}:entry",
+    "//src/arch/${plat_arch}/hftest:entry",
+    "//src/arch/${plat_arch}/hftest:interrupts",
+    "//src/arch/${plat_arch}/hftest:power_mgmt",
+  ]
+}
+
+# Common code for hftest, whether it is running under Linux, under Hafnium in
+# the primary VM, or directly on the hardware.
+source_set("common") {
+  visibility = [ ":*" ]
+  testonly = true
+  public_configs = [ ":hftest_config" ]
+  sources = [
+    "common.c",
+  ]
+  deps = [
+    "//src:fdt_handler",
+    "//src:memiter",
+    "//src:panic",
+    "//src:std",
+  ]
+}
+
+source_set("mm") {
+  testonly = true
+
+  public_configs = [ ":hftest_config" ]
+
+  sources = [
+    "mm.c",
+  ]
+
+  deps = [
+    "//src:layout",
+    "//src:mm",
+    "//src/arch/${plat_arch}:arch",
+    "//src/arch/${plat_arch}/hftest:mm",
+  ]
+}
+
+source_set("power_mgmt") {
+  testonly = true
+
+  public_configs = [ ":hftest_config" ]
+
+  sources = [
+    "power_mgmt.c",
+  ]
+
+  deps = [
+    ":mm",
+    "//src/arch/${plat_arch}/hftest:power_mgmt",
+  ]
+}
diff --git a/test/hftest/common.c b/test/hftest/common.c
new file mode 100644
index 0000000..877f1db
--- /dev/null
+++ b/test/hftest/common.c
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include "hf/boot_params.h"
+#include "hf/fdt_handler.h"
+#include "hf/memiter.h"
+#include "hf/std.h"
+
+#include "hftest_common.h"
+#include "test/hftest.h"
+
+HFTEST_ENABLE();
+
+static struct hftest_test hftest_constructed[HFTEST_MAX_TESTS];
+static size_t hftest_count;
+static struct hftest_test *hftest_list;
+
+static struct hftest_context global_context;
+
+struct hftest_context *hftest_get_context(void)
+{
+	return &global_context;
+}
+
+/**
+ * Adds the given test information to the global list, to be used by
+ * `hftest_use_registered_list`.
+ */
+void hftest_register(struct hftest_test test)
+{
+	if (hftest_count < HFTEST_MAX_TESTS) {
+		hftest_constructed[hftest_count++] = test;
+	} else {
+		HFTEST_FAIL(true, "Too many tests");
+	}
+}
+
+/**
+ * Uses the list of tests registered by `hftest_register(...)` as the ones to
+ * run.
+ */
+void hftest_use_registered_list(void)
+{
+	hftest_list = hftest_constructed;
+}
+
+/**
+ * Uses the given list of tests as the ones to run.
+ */
+void hftest_use_list(struct hftest_test list[], size_t count)
+{
+	hftest_list = list;
+	hftest_count = count;
+}
+
+/**
+ * Writes out a JSON structure describing the available tests.
+ */
+void hftest_json(void)
+{
+	const char *suite = NULL;
+	size_t i;
+	size_t suites_in_image = 0;
+	size_t tests_in_suite = 0;
+
+	HFTEST_LOG("{");
+	HFTEST_LOG("  \"suites\": [");
+	for (i = 0; i < hftest_count; ++i) {
+		struct hftest_test *test = &hftest_list[i];
+		if (test->suite != suite) {
+			/* Close out previously open suite. */
+			if (tests_in_suite) {
+				HFTEST_LOG("      ]");
+				HFTEST_LOG("    },");
+			}
+			/* Move onto new suite. */
+			++suites_in_image;
+			suite = test->suite;
+			tests_in_suite = 0;
+			HFTEST_LOG("    {");
+			HFTEST_LOG("      \"name\": \"%s\",", test->suite);
+		}
+		if (test->kind == HFTEST_KIND_SET_UP) {
+			HFTEST_LOG("      \"setup\": true,");
+		}
+		if (test->kind == HFTEST_KIND_TEAR_DOWN) {
+			HFTEST_LOG("      \"teardown\": true,");
+		}
+		if (test->kind == HFTEST_KIND_TEST) {
+			if (!tests_in_suite) {
+				HFTEST_LOG("      \"tests\": [");
+			}
+			/*
+			 * It's easier to put the comma at the start of the line
+			 * than the end even though the JSON looks a bit funky.
+			 */
+			HFTEST_LOG("       %c{", tests_in_suite ? ',' : ' ');
+			HFTEST_LOG("          \"name\": \"%s\",", test->name);
+			HFTEST_LOG("          \"is_long_running\": %s",
+				   test->is_long_running ? "true" : "false");
+			HFTEST_LOG("       }");
+			++tests_in_suite;
+		}
+	}
+	if (tests_in_suite) {
+		HFTEST_LOG("      ]");
+		HFTEST_LOG("    }");
+	}
+	HFTEST_LOG("  ]");
+	HFTEST_LOG("}");
+}
+
+/**
+ * Logs a failure message and shut down.
+ */
+noreturn void abort(void)
+{
+	HFTEST_LOG("FAIL");
+	arch_power_off();
+}
+
+static void run_test(hftest_test_fn set_up, hftest_test_fn test,
+		     hftest_test_fn tear_down, const struct fdt_header *fdt)
+{
+	/* Prepare the context. */
+	struct hftest_context *ctx = hftest_get_context();
+	memset_s(ctx, sizeof(*ctx), 0, sizeof(*ctx));
+	ctx->abort = abort;
+	ctx->fdt = fdt;
+
+	/* Run any set up functions. */
+	if (set_up) {
+		set_up();
+		if (ctx->failures) {
+			abort();
+		}
+	}
+
+	/* Run the test. */
+	test();
+	if (ctx->failures) {
+		abort();
+	}
+
+	/* Run any tear down functions. */
+	if (tear_down) {
+		tear_down();
+		if (ctx->failures) {
+			abort();
+		}
+	}
+
+	HFTEST_LOG("FINISHED");
+}
+
+/**
+ * Runs the given test case.
+ */
+void hftest_run(struct memiter suite_name, struct memiter test_name,
+		const struct fdt_header *fdt)
+{
+	size_t i;
+	hftest_test_fn suite_set_up = NULL;
+	hftest_test_fn suite_tear_down = NULL;
+
+	for (i = 0; i < hftest_count; ++i) {
+		struct hftest_test *test = &hftest_list[i];
+
+		/* Check if this test is part of the suite we want. */
+		if (memiter_iseq(&suite_name, test->suite)) {
+			switch (test->kind) {
+			/*
+			 * The first entries in the suite are the set up and
+			 * tear down functions.
+			 */
+			case HFTEST_KIND_SET_UP:
+				suite_set_up = test->fn;
+				break;
+			case HFTEST_KIND_TEAR_DOWN:
+				suite_tear_down = test->fn;
+				break;
+			/* Find the test. */
+			case HFTEST_KIND_TEST:
+				if (memiter_iseq(&test_name, test->name)) {
+					run_test(suite_set_up, test->fn,
+						 suite_tear_down, fdt);
+					return;
+				}
+				break;
+			default:
+				/* Ignore other kinds. */
+				break;
+			}
+		}
+	}
+
+	HFTEST_LOG("Unable to find requested tests.");
+}
+
+/**
+ * Writes out usage information.
+ */
+void hftest_help(void)
+{
+	HFTEST_LOG("usage:");
+	HFTEST_LOG("");
+	HFTEST_LOG("  help");
+	HFTEST_LOG("");
+	HFTEST_LOG("    Show this help.");
+	HFTEST_LOG("");
+	HFTEST_LOG("  json");
+	HFTEST_LOG("");
+	HFTEST_LOG(
+		"    Print a directory of test suites and tests in "
+		"JSON "
+		"format.");
+	HFTEST_LOG("");
+	HFTEST_LOG("  run <suite> <test>");
+	HFTEST_LOG("");
+	HFTEST_LOG("    Run the named test from the named test suite.");
+}
+
+static uintptr_t vcpu_index_to_id(size_t index)
+{
+	/* For now we use indices as IDs for vCPUs. */
+	return index;
+}
+
+/**
+ * Get the ID of the CPU with the given index.
+ */
+uintptr_t hftest_get_cpu_id(size_t index)
+{
+	struct boot_params params;
+	struct fdt_node n;
+	const struct fdt_header *fdt = hftest_get_context()->fdt;
+
+	if (fdt == NULL) {
+		/*
+		 * We must be in a service VM, so apply the mapping that Hafnium
+		 * uses for vCPU IDs.
+		 */
+		return vcpu_index_to_id(index);
+	}
+
+	/* Find physical CPU ID from FDT. */
+	if (!fdt_root_node(&n, fdt)) {
+		FAIL("FDT failed validation.");
+	}
+	if (!fdt_find_child(&n, "")) {
+		FAIL("Unable to find FDT root node.");
+	}
+	fdt_find_cpus(&n, params.cpu_ids, &params.cpu_count);
+
+	return params.cpu_ids[index];
+}
diff --git a/test/hftest/hftest.py b/test/hftest/hftest.py
new file mode 100755
index 0000000..1c88953
--- /dev/null
+++ b/test/hftest/hftest.py
@@ -0,0 +1,593 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script which drives invocation of tests and parsing their output to produce
+a results report.
+"""
+
+from __future__ import print_function
+
+import xml.etree.ElementTree as ET
+
+import argparse
+import collections
+import datetime
+import json
+import os
+import re
+import subprocess
+import sys
+
+HFTEST_LOG_PREFIX = "[hftest] "
+HFTEST_LOG_FAILURE_PREFIX = "Failure:"
+HFTEST_LOG_FINISHED = "FINISHED"
+
+HF_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))))
+DTC_SCRIPT = os.path.join(HF_ROOT, "build", "image", "dtc.py")
+FVP_BINARY = os.path.join(
+    os.path.dirname(HF_ROOT), "fvp", "Base_RevC_AEMv8A_pkg", "models",
+    "Linux64_GCC-4.9", "FVP_Base_RevC-2xAEMv8A")
+FVP_PREBUILTS_ROOT = os.path.join(
+    HF_ROOT, "prebuilts", "linux-aarch64", "arm-trusted-firmware", "fvp")
+FVP_PREBUILT_DTS = os.path.join(
+    FVP_PREBUILTS_ROOT, "fvp-base-gicv3-psci-1t.dts")
+FVP_PREBUILT_BL31 = os.path.join(FVP_PREBUILTS_ROOT, "bl31.bin")
+
+def read_file(path):
+    with open(path, "r") as f:
+        return f.read()
+
+def write_file(path, to_write, append=False):
+    with open(path, "a" if append else "w") as f:
+        f.write(to_write)
+
+def append_file(path, to_write):
+    write_file(path, to_write, append=True)
+
+def join_if_not_None(*args):
+    return " ".join(filter(lambda x: x, args))
+
+class ArtifactsManager:
+    """Class which manages folder with test artifacts."""
+
+    def __init__(self, log_dir):
+        self.created_files = []
+        self.log_dir = log_dir
+
+        # Create directory.
+        try:
+            os.makedirs(self.log_dir)
+        except OSError:
+            if not os.path.isdir(self.log_dir):
+                raise
+        print("Logs saved under", log_dir)
+
+        # Create files expected by the Sponge test result parser.
+        self.sponge_log_path = self.create_file("sponge_log", ".log")
+        self.sponge_xml_path = self.create_file("sponge_log", ".xml")
+
+    def gen_file_path(self, basename, extension):
+        """Generate path to a file in the log directory."""
+        return os.path.join(self.log_dir, basename + extension)
+
+    def create_file(self, basename, extension):
+        """Create and touch a new file in the log folder. Ensure that no other
+        file of the same name was created by this instance of ArtifactsManager.
+        """
+        # Determine the path of the file.
+        path = self.gen_file_path(basename, extension)
+
+        # Check that the path is unique.
+        assert(path not in self.created_files)
+        self.created_files += [ path ]
+
+        # Touch file.
+        with open(path, "w") as f:
+            pass
+
+        return path
+
+    def get_file(self, basename, extension):
+        """Return path to a file in the log folder. Assert that it was created
+        by this instance of ArtifactsManager."""
+        path = self.gen_file_path(basename, extension)
+        assert(path in self.created_files)
+        return path
+
+
+# Tuple holding the arguments common to all driver constructors.
+# This is to avoid having to pass arguments from subclasses to superclasses.
+DriverArgs = collections.namedtuple("DriverArgs", [
+        "artifacts",
+        "kernel",
+        "initrd",
+        "manifest",
+        "vm_args",
+        "cpu"
+    ])
+
+
+# State shared between the common Driver class and its subclasses during
+# a single invocation of the target platform.
+class DriverRunState:
+    def __init__(self, log_path):
+        self.log_path = log_path
+        self.ret_code = 0
+
+    def set_ret_code(self, ret_code):
+        self.ret_code = ret_code
+
+class DriverRunException(Exception):
+    """Exception thrown if subprocess invoked by a driver returned non-zero
+    status code. Used to fast-exit from a driver command sequence."""
+    pass
+
+
+class Driver:
+    """Parent class of drivers for all testable platforms."""
+
+    def __init__(self, args):
+        self.args = args
+
+    def get_run_log(self, run_name):
+        """Return path to the main log of a given test run."""
+        return self.args.artifacts.get_file(run_name, ".log")
+
+    def start_run(self, run_name):
+        """Hook called by Driver subclasses before they invoke the target
+        platform."""
+        return DriverRunState(self.args.artifacts.create_file(run_name, ".log"))
+
+    def exec_logged(self, run_state, exec_args):
+        """Run a subprocess on behalf of a Driver subclass and append its
+        stdout and stderr to the main log."""
+        assert(run_state.ret_code == 0)
+        with open(run_state.log_path, "a") as f:
+            f.write("$ {}\r\n".format(" ".join(exec_args)))
+            f.flush()
+            ret_code = subprocess.call(exec_args, stdout=f, stderr=f)
+            if ret_code != 0:
+                run_state.set_ret_code(ret_code)
+                raise DriverRunException()
+
+    def finish_run(self, run_state):
+        """Hook called by Driver subclasses after they finished running the
+        target platform. `ret_code` argument is the return code of the main
+        command run by the driver. A corresponding log message is printed."""
+        # Decode return code and add a message to the log.
+        with open(run_state.log_path, "a") as f:
+            if run_state.ret_code == 124:
+                f.write("\r\n{}{} timed out\r\n".format(
+                    HFTEST_LOG_PREFIX, HFTEST_LOG_FAILURE_PREFIX))
+            elif run_state.ret_code != 0:
+                f.write("\r\n{}{} process return code {}\r\n".format(
+                    HFTEST_LOG_PREFIX, HFTEST_LOG_FAILURE_PREFIX,
+                    run_state.ret_code))
+
+        # Append log of this run to full test log.
+        log_content = read_file(run_state.log_path)
+        append_file(
+            self.args.artifacts.sponge_log_path,
+            log_content + "\r\n\r\n")
+        return log_content
+
+    def overlay_dtb(self, run_state, base_dtb, overlay_dtb, out_dtb):
+        """Overlay `overlay_dtb` over `base_dtb` into `out_dtb`."""
+        dtc_args = [
+            DTC_SCRIPT, "overlay",
+            out_dtb, base_dtb, overlay_dtb,
+        ]
+        self.exec_logged(run_state, dtc_args)
+
+
+class QemuDriver(Driver):
+    """Driver which runs tests in QEMU."""
+
+    def __init__(self, args):
+        Driver.__init__(self, args)
+
+    def gen_exec_args(self, test_args, is_long_running, dtb_path=None,
+            dumpdtb_path=None):
+        """Generate command line arguments for QEMU."""
+        time_limit = "120s" if is_long_running else "10s"
+        # If no CPU configuration is selected, then test against the maximum
+        # configuration, "max", supported by QEMU.
+        cpu = self.args.cpu or "max"
+        exec_args = [
+            "timeout", "--foreground", time_limit,
+            "./prebuilts/linux-x64/qemu/qemu-system-aarch64",
+            "-machine", "virt,virtualization=on,gic_version=3",
+            "-cpu", cpu, "-smp", "4", "-m", "64M",
+            "-nographic", "-nodefaults", "-serial", "stdio",
+            "-d", "unimp", "-kernel", self.args.kernel,
+        ]
+
+        if dtb_path:
+            exec_args += ["-dtb", dtb_path]
+
+        if dumpdtb_path:
+            exec_args += ["-machine", "dumpdtb=" + dumpdtb_path]
+
+        if self.args.initrd:
+            exec_args += ["-initrd", self.args.initrd]
+
+        vm_args = join_if_not_None(self.args.vm_args, test_args)
+        if vm_args:
+            exec_args += ["-append", vm_args]
+
+        return exec_args
+
+    def dump_dtb(self, run_state, test_args, path):
+        dumpdtb_args = self.gen_exec_args(test_args, False, dumpdtb_path=path)
+        self.exec_logged(run_state, dumpdtb_args)
+
+    def run(self, run_name, test_args, is_long_running):
+        """Run test given by `test_args` in QEMU."""
+        run_state = self.start_run(run_name)
+
+        try:
+            dtb_path = None
+
+            # If manifest DTBO specified, dump DTB from QEMU and overlay them.
+            if self.args.manifest:
+                base_dtb_path = self.args.artifacts.create_file(
+                    run_name, ".base.dtb")
+                dtb_path = self.args.artifacts.create_file(run_name, ".dtb")
+                self.dump_dtb(run_state, test_args, base_dtb_path)
+                self.overlay_dtb(
+                    run_state, base_dtb_path, self.args.manifest, dtb_path)
+
+            # Execute test in QEMU..
+            exec_args = self.gen_exec_args(test_args, is_long_running,
+                    dtb_path=dtb_path)
+            self.exec_logged(run_state, exec_args)
+        except DriverRunException:
+            pass
+
+        return self.finish_run(run_state)
+
+
+class FvpDriver(Driver):
+    """Driver which runs tests in Arm FVP emulator."""
+
+    def __init__(self, args):
+        if args.cpu:
+            raise ValueError("FVP emulator does not support the --cpu option.")
+        Driver.__init__(self, args)
+
+    def gen_dts(self, dts_path, test_args, initrd_start, initrd_end):
+        """Create a DeviceTree source which will be compiled into a DTB and
+        passed to FVP for a test run."""
+        vm_args = join_if_not_None(self.args.vm_args, test_args)
+        write_file(dts_path, read_file(FVP_PREBUILT_DTS))
+        append_file(dts_path, """
+                / {{
+                    chosen {{
+                        bootargs = "{}";
+                        stdout-path = "serial0:115200n8";
+                        linux,initrd-start = <{}>;
+                        linux,initrd-end = <{}>;
+                    }};
+                }};
+            """.format(vm_args, initrd_start, initrd_end))
+
+    def gen_fvp_args(
+            self, is_long_running, initrd_start, uart0_log_path, uart1_log_path,
+            dtb_path):
+        """Generate command line arguments for FVP."""
+        time_limit = "80s" if is_long_running else "40s"
+        fvp_args = [
+            "timeout", "--foreground", time_limit,
+            FVP_BINARY,
+            "-C", "pctl.startup=0.0.0.0",
+            "-C", "bp.secure_memory=0",
+            "-C", "cluster0.NUM_CORES=4",
+            "-C", "cluster1.NUM_CORES=4",
+            "-C", "cache_state_modelled=0",
+            "-C", "bp.vis.disable_visualisation=true",
+            "-C", "bp.vis.rate_limit-enable=false",
+            "-C", "bp.terminal_0.start_telnet=false",
+            "-C", "bp.terminal_1.start_telnet=false",
+            "-C", "bp.terminal_2.start_telnet=false",
+            "-C", "bp.terminal_3.start_telnet=false",
+            "-C", "bp.pl011_uart0.untimed_fifos=1",
+            "-C", "bp.pl011_uart0.unbuffered_output=1",
+            "-C", "bp.pl011_uart0.out_file=" + uart0_log_path,
+            "-C", "bp.pl011_uart1.out_file=" + uart1_log_path,
+            "-C", "cluster0.cpu0.RVBAR=0x04020000",
+            "-C", "cluster0.cpu1.RVBAR=0x04020000",
+            "-C", "cluster0.cpu2.RVBAR=0x04020000",
+            "-C", "cluster0.cpu3.RVBAR=0x04020000",
+            "-C", "cluster1.cpu0.RVBAR=0x04020000",
+            "-C", "cluster1.cpu1.RVBAR=0x04020000",
+            "-C", "cluster1.cpu2.RVBAR=0x04020000",
+            "-C", "cluster1.cpu3.RVBAR=0x04020000",
+            "--data", "cluster0.cpu0=" + FVP_PREBUILT_BL31 + "@0x04020000",
+            "--data", "cluster0.cpu0=" + dtb_path + "@0x82000000",
+            "--data", "cluster0.cpu0=" + self.args.kernel + "@0x80000000",
+            "-C", "bp.ve_sysregs.mmbSiteDefault=0",
+            "-C", "bp.ve_sysregs.exit_on_shutdown=1",
+        ]
+
+        if self.args.initrd:
+            fvp_args += [
+                "--data",
+                "cluster0.cpu0={}@{}".format(
+                    self.args.initrd, hex(initrd_start))
+            ]
+
+        return fvp_args
+
+    def run(self, run_name, test_args, is_long_running):
+        run_state = self.start_run(run_name)
+
+        base_dts_path = self.args.artifacts.create_file(run_name, ".base.dts")
+        base_dtb_path = self.args.artifacts.create_file(run_name, ".base.dtb")
+        dtb_path = self.args.artifacts.create_file(run_name, ".dtb")
+        uart0_log_path = self.args.artifacts.create_file(run_name, ".uart0.log")
+        uart1_log_path = self.args.artifacts.create_file(run_name, ".uart1.log")
+
+        initrd_start = 0x84000000
+        if self.args.initrd:
+            initrd_end = initrd_start + os.path.getsize(self.args.initrd)
+        else:
+            initrd_end = 0x85000000  # Default value
+
+        try:
+            # Create a DT to pass to FVP.
+            self.gen_dts(base_dts_path, test_args, initrd_start, initrd_end)
+
+            # Compile DTS to DTB.
+            dtc_args = [
+                DTC_SCRIPT, "compile", "-i", base_dts_path, "-o", base_dtb_path,
+            ]
+            self.exec_logged(run_state, dtc_args)
+
+            # If manifest DTBO specified, overlay it.
+            if self.args.manifest:
+                self.overlay_dtb(
+                    run_state, base_dtb_path, self.args.manifest, dtb_path)
+            else:
+                dtb_path = base_dtb_path
+
+            # Run FVP.
+            fvp_args = self.gen_fvp_args(
+                is_long_running, initrd_start, uart0_log_path, uart1_log_path,
+                dtb_path)
+            self.exec_logged(run_state, fvp_args)
+        except DriverRunException:
+            pass
+
+        # Append UART0 output to main log.
+        append_file(run_state.log_path, read_file(uart0_log_path))
+        return self.finish_run(run_state)
+
+
+# Tuple used to return information about the results of running a set of tests.
+TestRunnerResult = collections.namedtuple("TestRunnerResult", [
+        "tests_run",
+        "tests_failed",
+    ])
+
+
+class TestRunner:
+    """Class which communicates with a test platform to obtain a list of
+    available tests and driving their execution."""
+
+    def __init__(self, artifacts, driver, image_name, suite_regex, test_regex,
+            skip_long_running_tests):
+        self.artifacts = artifacts
+        self.driver = driver
+        self.image_name = image_name
+        self.skip_long_running_tests = skip_long_running_tests
+
+        self.suite_re = re.compile(suite_regex or ".*")
+        self.test_re = re.compile(test_regex or ".*")
+
+    def extract_hftest_lines(self, raw):
+        """Extract hftest-specific lines from a raw output from an invocation
+        of the test platform."""
+        lines = []
+        for line in raw.splitlines():
+            if line.startswith("VM "):
+                line = line[len("VM 0: "):]
+            if line.startswith(HFTEST_LOG_PREFIX):
+                lines.append(line[len(HFTEST_LOG_PREFIX):])
+        return lines
+
+    def get_test_json(self):
+        """Invoke the test platform and request a JSON of available test and
+        test suites."""
+        out = self.driver.run("json", "json", False)
+        hf_out = "\n".join(self.extract_hftest_lines(out))
+        try:
+            return json.loads(hf_out)
+        except ValueError as e:
+            print(out)
+            raise e
+
+    def collect_results(self, fn, it, xml_node):
+        """Run `fn` on every entry in `it` and collect their TestRunnerResults.
+        Insert "tests" and "failures" nodes to `xml_node`."""
+        tests_run = 0
+        tests_failed = 0
+        for i in it:
+            sub_result = fn(i)
+            assert(sub_result.tests_run >= sub_result.tests_failed)
+            tests_run += sub_result.tests_run
+            tests_failed += sub_result.tests_failed
+
+        xml_node.set("tests", str(tests_run))
+        xml_node.set("failures", str(tests_failed))
+        return TestRunnerResult(tests_run, tests_failed)
+
+    def is_passed_test(self, test_out):
+        """Parse the output of a test and return True if it passed."""
+        return \
+            len(test_out) > 0 and \
+            test_out[-1] == HFTEST_LOG_FINISHED and \
+            not any(l.startswith(HFTEST_LOG_FAILURE_PREFIX) for l in test_out)
+
+    def get_log_name(self, suite, test):
+        """Returns a string with a generated log name for the test."""
+        log_name = ""
+
+        cpu = self.driver.args.cpu
+        if cpu:
+            log_name += cpu + "."
+
+        log_name += suite["name"] + "." + test["name"]
+
+        return log_name
+
+    def run_test(self, suite, test, suite_xml):
+        """Invoke the test platform and request to run a given `test` in given
+        `suite`. Create a new XML node with results under `suite_xml`.
+        Test only invoked if it matches the regex given to constructor."""
+        if not self.test_re.match(test["name"]):
+            return TestRunnerResult(tests_run=0, tests_failed=0)
+
+        if self.skip_long_running_tests and test["is_long_running"]:
+            print("      SKIP", test["name"])
+            return TestRunnerResult(tests_run=0, tests_failed=0)
+
+        print("      RUN", test["name"])
+        log_name = self.get_log_name(suite, test)
+
+        test_xml = ET.SubElement(suite_xml, "testcase")
+        test_xml.set("name", test["name"])
+        test_xml.set("classname", suite["name"])
+        test_xml.set("status", "run")
+
+        out = self.extract_hftest_lines(self.driver.run(
+            log_name, "run {} {}".format(suite["name"], test["name"]),
+            test["is_long_running"]))
+
+        if self.is_passed_test(out):
+            print("        PASS")
+            return TestRunnerResult(tests_run=1, tests_failed=0)
+        else:
+            print("[x]     FAIL --", self.driver.get_run_log(log_name))
+            failure_xml = ET.SubElement(test_xml, "failure")
+            # TODO: set a meaningful message and put log in CDATA
+            failure_xml.set("message", "Test failed")
+            return TestRunnerResult(tests_run=1, tests_failed=1)
+
+    def run_suite(self, suite, xml):
+        """Invoke the test platform and request to run all matching tests in
+        `suite`. Create new XML nodes with results under `xml`.
+        Suite skipped if it does not match the regex given to constructor."""
+        if not self.suite_re.match(suite["name"]):
+            return TestRunnerResult(tests_run=0, tests_failed=0)
+
+        print("    SUITE", suite["name"])
+        suite_xml = ET.SubElement(xml, "testsuite")
+        suite_xml.set("name", suite["name"])
+
+        return self.collect_results(
+            lambda test: self.run_test(suite, test, suite_xml),
+            suite["tests"],
+            suite_xml)
+
+    def run_tests(self):
+        """Run all suites and tests matching regexes given to constructor.
+        Write results to sponge log XML. Return the number of run and failed
+        tests."""
+
+        test_spec = self.get_test_json()
+        timestamp = datetime.datetime.now().replace(microsecond=0).isoformat()
+
+        xml = ET.Element("testsuites")
+        xml.set("name", self.image_name)
+        xml.set("timestamp", timestamp)
+
+        result = self.collect_results(
+            lambda suite: self.run_suite(suite, xml),
+            test_spec["suites"],
+            xml)
+
+        # Write XML to file.
+        with open(self.artifacts.sponge_xml_path, "w") as f:
+            ET.ElementTree(xml).write(f, encoding='utf-8', xml_declaration=True)
+
+        if result.tests_failed > 0:
+            print("[x] FAIL:", result.tests_failed, "of", result.tests_run,
+                    "tests failed")
+        elif result.tests_run > 0:
+            print("    PASS: all", result.tests_run, "tests passed")
+
+        return result
+
+
+def Main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("image")
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--log", required=True)
+    parser.add_argument("--out_initrd")
+    parser.add_argument("--initrd")
+    parser.add_argument("--suite")
+    parser.add_argument("--test")
+    parser.add_argument("--vm_args")
+    parser.add_argument("--fvp", action="store_true")
+    parser.add_argument("--skip-long-running-tests", action="store_true")
+    parser.add_argument("--cpu",
+        help="Selects the CPU configuration for the run environment.")
+    args = parser.parse_args()
+
+    # Resolve some paths.
+    image = os.path.join(args.out, args.image + ".bin")
+    initrd = None
+    manifest = None
+    image_name = args.image
+    if args.initrd:
+        initrd_dir = os.path.join(args.out_initrd, "obj", args.initrd)
+        initrd = os.path.join(initrd_dir, "initrd.img")
+        manifest = os.path.join(initrd_dir, "manifest.dtbo")
+        image_name += "_" + args.initrd
+    vm_args = args.vm_args or ""
+
+    # Create class which will manage all test artifacts.
+    artifacts = ArtifactsManager(os.path.join(args.log, image_name))
+
+    # Create a driver for the platform we want to test on.
+    driver_args = DriverArgs(artifacts, image, initrd, manifest, vm_args,
+        args.cpu)
+    if args.fvp:
+        driver = FvpDriver(driver_args)
+    else:
+        driver = QemuDriver(driver_args)
+
+    # Create class which will drive test execution.
+    runner = TestRunner(artifacts, driver, image_name, args.suite, args.test,
+        args.skip_long_running_tests)
+
+    # Run tests.
+    runner_result = runner.run_tests()
+
+    # Print error message if no tests were run as this is probably unexpected.
+    # Return suitable error code.
+    if runner_result.tests_run == 0:
+        print("Error: no tests match")
+        return 10
+    elif runner_result.tests_failed > 0:
+        return 1
+    else:
+        return 0
+
+if __name__ == "__main__":
+    sys.exit(Main())
diff --git a/test/hftest/hftest_common.h b/test/hftest/hftest_common.h
new file mode 100644
index 0000000..fba4a54
--- /dev/null
+++ b/test/hftest/hftest_common.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/fdt.h"
+#include "hf/memiter.h"
+
+#include "test/hftest_impl.h"
+
+void hftest_use_registered_list(void);
+void hftest_use_list(struct hftest_test list[], size_t count);
+
+void hftest_json(void);
+void hftest_run(struct memiter suite_name, struct memiter test_name,
+		const struct fdt_header *fdt);
+void hftest_help(void);
diff --git a/test/hftest/linux_main.c b/test/hftest/linux_main.c
new file mode 100644
index 0000000..2ebe91b
--- /dev/null
+++ b/test/hftest/linux_main.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "hf/memiter.h"
+
+#include "hftest_common.h"
+#include "test/hftest.h"
+#include <sys/reboot.h>
+
+void test_main(int argc, const char *argv[])
+{
+	const char *command;
+
+	if (argc < 2) {
+		HFTEST_LOG("Unable to parse command.");
+		return;
+	}
+	command = argv[1];
+
+	hftest_use_registered_list();
+
+	if (strcmp(command, "json") == 0) {
+		hftest_json();
+		return;
+	}
+
+	if (strcmp(command, "run") == 0) {
+		struct memiter suite_name;
+		struct memiter test_name;
+
+		if (argc != 4) {
+			HFTEST_LOG("Unable to parse test.");
+			return;
+		}
+
+		memiter_init(&suite_name, argv[2], strnlen_s(argv[2], 64));
+		memiter_init(&test_name, argv[3], strnlen_s(argv[3], 64));
+		hftest_run(suite_name, test_name, NULL);
+		return;
+	}
+
+	hftest_help();
+}
+
+int main(int argc, const char *argv[])
+{
+	test_main(argc, argv);
+	reboot(RB_POWER_OFF);
+	return 0;
+}
diff --git a/test/hftest/mm.c b/test/hftest/mm.c
new file mode 100644
index 0000000..7c78ab9
--- /dev/null
+++ b/test/hftest/mm.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/mm.h"
+
+#include "test/hftest.h"
+
+/* Number of pages reserved for page tables. Increase if necessary. */
+#define PTABLE_PAGES 3
+
+alignas(alignof(struct mm_page_table)) static char ptable_buf
+	[sizeof(struct mm_page_table) * PTABLE_PAGES];
+
+static struct mpool ppool;
+static struct mm_ptable ptable;
+
+static struct mm_stage1_locked get_stage1_locked(void)
+{
+	return (struct mm_stage1_locked){.ptable = &ptable};
+}
+
+bool hftest_mm_init(void)
+{
+	struct mm_stage1_locked stage1_locked;
+
+	mpool_init(&ppool, sizeof(struct mm_page_table));
+	if (!mpool_add_chunk(&ppool, ptable_buf, sizeof(ptable_buf))) {
+		HFTEST_FAIL(true, "Failed to add buffer to page-table pool.");
+	}
+
+	if (!mm_ptable_init(&ptable, MM_FLAG_STAGE1, &ppool)) {
+		HFTEST_FAIL(true, "Unable to allocate memory for page table.");
+	}
+
+	stage1_locked = get_stage1_locked();
+	mm_identity_map(stage1_locked, pa_init(0),
+			pa_init(mm_ptable_addr_space_end(MM_FLAG_STAGE1)),
+			MM_MODE_R | MM_MODE_W | MM_MODE_X, &ppool);
+
+	if (!arch_vm_mm_init()) {
+		return false;
+	}
+
+	arch_vm_mm_enable(ptable.root);
+
+	return true;
+}
+
+void hftest_mm_identity_map(const void *base, size_t size, uint32_t mode)
+{
+	struct mm_stage1_locked stage1_locked = get_stage1_locked();
+	paddr_t start = pa_from_va(va_from_ptr(base));
+	paddr_t end = pa_add(start, size);
+
+	if (mm_identity_map(stage1_locked, start, end, mode, &ppool) != base) {
+		FAIL("Could not add new page table mapping. Try increasing "
+		     "size of the page table buffer.");
+	}
+}
+
+void hftest_mm_vcpu_init(void)
+{
+	arch_vm_mm_enable(ptable.root);
+}
diff --git a/test/hftest/power_mgmt.c b/test/hftest/power_mgmt.c
new file mode 100644
index 0000000..4c651a6
--- /dev/null
+++ b/test/hftest/power_mgmt.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include "hf/arch/mm.h"
+
+#include "hf/spinlock.h"
+
+#include "test/hftest.h"
+
+struct cpu_start_state {
+	void (*entry)(uintptr_t arg);
+	uintreg_t arg;
+	struct spinlock lock;
+};
+
+static noreturn void cpu_entry(uintptr_t arg)
+{
+	struct cpu_start_state *s = (struct cpu_start_state *)arg;
+	struct cpu_start_state s_copy;
+
+	/*
+	 * Initialize memory and enable caching. Must be the first thing we do.
+	 */
+	hftest_mm_vcpu_init();
+
+	/* Make a copy of the cpu_start_state struct. */
+	s_copy = *s;
+
+	/* Inform cpu_start() that the state struct memory can now be freed. */
+	sl_unlock(&s->lock);
+
+	/* Call the given entry function with the given argument. */
+	s_copy.entry(s_copy.arg);
+
+	/* If the entry function returns, turn off the CPU. */
+	arch_cpu_stop();
+}
+
+bool hftest_cpu_start(uintptr_t id, void *stack, size_t stack_size,
+		      void (*entry)(uintptr_t arg), uintptr_t arg)
+{
+	struct cpu_start_state s;
+	struct arch_cpu_start_state s_arch;
+
+	/*
+	 * Config for arch_cpu_start() which will start a new CPU and
+	 * immediately jump to cpu_entry(). This function must guarantee that
+	 * the state struct is not be freed until cpu_entry() is called.
+	 */
+	s_arch.initial_sp = (uintptr_t)stack + stack_size;
+	s_arch.entry = cpu_entry;
+	s_arch.arg = (uintptr_t)&s;
+
+	/*
+	 * Flush the `cpu_start_state` struct because the new CPU will be
+	 * started without caching enabled and will need the data early on.
+	 * Write back is all that is really needed so flushing will definitely
+	 * get the job done.
+	 */
+	arch_mm_flush_dcache(&s_arch, sizeof(s_arch));
+
+	if ((s_arch.initial_sp % STACK_ALIGN) != 0) {
+		HFTEST_FAIL(true,
+			    "Stack pointer of new vCPU not properly aligned.");
+	}
+
+	/*
+	 * Config for cpu_entry(). Its job is to initialize memory and call the
+	 * provided entry point with the provided argument.
+	 */
+	s.entry = entry;
+	s.arg = arg;
+	sl_init(&s.lock);
+
+	/*
+	 * Lock the cpu_start_state struct which will be unlocked once
+	 * cpu_entry() does not need its content anymore. This simultaneously
+	 * protects the arch_cpu_start_state struct which must not be freed
+	 * before cpu_entry() is called.
+	 */
+	sl_lock(&s.lock);
+
+	/* Try to start the given CPU. */
+	if (!arch_cpu_start(id, &s_arch)) {
+		return false;
+	}
+
+	/*
+	 * Wait until cpu_entry() unlocks the cpu_start_state lock before
+	 * freeing stack memory.
+	 */
+	sl_lock(&s.lock);
+	return true;
+}
diff --git a/test/hftest/service.c b/test/hftest/service.c
new file mode 100644
index 0000000..afbf6f6
--- /dev/null
+++ b/test/hftest/service.c
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/memiter.h"
+#include "hf/mm.h"
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+
+alignas(4096) uint8_t kstack[4096];
+
+HFTEST_ENABLE();
+
+extern struct hftest_test hftest_begin[];
+extern struct hftest_test hftest_end[];
+
+static alignas(HF_MAILBOX_SIZE) uint8_t send[HF_MAILBOX_SIZE];
+static alignas(HF_MAILBOX_SIZE) uint8_t recv[HF_MAILBOX_SIZE];
+
+static hf_ipaddr_t send_addr = (hf_ipaddr_t)send;
+static hf_ipaddr_t recv_addr = (hf_ipaddr_t)recv;
+
+static struct hftest_context global_context;
+
+struct hftest_context *hftest_get_context(void)
+{
+	return &global_context;
+}
+
+/** Find the service with the name passed in the arguments. */
+static hftest_test_fn find_service(struct memiter *args)
+{
+	struct memiter service_name;
+	struct hftest_test *test;
+
+	if (!memiter_parse_str(args, &service_name)) {
+		return NULL;
+	}
+
+	for (test = hftest_begin; test < hftest_end; ++test) {
+		if (test->kind == HFTEST_KIND_SERVICE &&
+		    memiter_iseq(&service_name, test->name)) {
+			return test->fn;
+		}
+	}
+
+	return NULL;
+}
+
+noreturn void abort(void)
+{
+	HFTEST_LOG("Service contained failures.");
+	/* Cause a fault, as a secondary can't power down the machine. */
+	*((volatile uint8_t *)1) = 1;
+
+	/* This should never be reached, but to make the compiler happy... */
+	for (;;) {
+	}
+}
+
+noreturn void kmain(size_t memory_size)
+{
+	struct memiter args;
+	hftest_test_fn service;
+	struct hftest_context *ctx;
+	struct spci_value ret;
+
+	/*
+	 * Initialize the stage-1 MMU and identity-map the entire address space.
+	 */
+	if (!hftest_mm_init()) {
+		HFTEST_LOG_FAILURE();
+		HFTEST_LOG(HFTEST_LOG_INDENT "Memory initialization failed");
+		for (;;) {
+			/* Hang if memory init failed. */
+		}
+	}
+
+	/* Prepare the context. */
+
+	/* Set up the mailbox. */
+	spci_rxtx_map(send_addr, recv_addr);
+
+	/* Receive the name of the service to run. */
+	ret = spci_msg_wait();
+	ASSERT_EQ(ret.func, SPCI_MSG_SEND_32);
+	memiter_init(&args, recv, spci_msg_send_size(ret));
+	service = find_service(&args);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Check the service was found. */
+	if (service == NULL) {
+		HFTEST_LOG_FAILURE();
+		HFTEST_LOG(HFTEST_LOG_INDENT
+			   "Unable to find requested service");
+		for (;;) {
+			/* Hang if the service was unknown. */
+		}
+	}
+
+	/* Clean the context. */
+	ctx = hftest_get_context();
+	memset_s(ctx, sizeof(*ctx), 0, sizeof(*ctx));
+	ctx->abort = abort;
+	ctx->send = send;
+	ctx->recv = recv;
+	ctx->memory_size = memory_size;
+
+	/* Pause so the next time cycles are given the service will be run. */
+	spci_yield();
+
+	/* Let the service run. */
+	service();
+
+	/* Cleanly handle it if the service returns. */
+	if (ctx->failures) {
+		abort();
+	}
+
+	for (;;) {
+		/* Hang if the service returns. */
+	}
+}
diff --git a/test/hftest/standalone_main.c b/test/hftest/standalone_main.c
new file mode 100644
index 0000000..586b3b3
--- /dev/null
+++ b/test/hftest/standalone_main.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/fdt.h"
+#include "hf/memiter.h"
+#include "hf/mm.h"
+
+#include "hftest_common.h"
+#include "test/hftest.h"
+
+alignas(4096) uint8_t kstack[4096];
+
+extern struct hftest_test hftest_begin[];
+extern struct hftest_test hftest_end[];
+
+void kmain(const struct fdt_header *fdt)
+{
+	struct fdt_node n;
+	const char *bootargs;
+	uint32_t bootargs_size;
+	struct memiter bootargs_iter;
+	struct memiter command;
+
+	/*
+	 * Initialize the stage-1 MMU and identity-map the entire address space.
+	 */
+	if ((VM_TOOLCHAIN == 1) && !hftest_mm_init()) {
+		HFTEST_LOG("Memory initialization failed.");
+		return;
+	}
+
+	/*
+	 * Install the exception handler with no IRQ callback for now, so that
+	 * exceptions are logged.
+	 */
+	exception_setup(NULL, NULL);
+
+	hftest_use_list(hftest_begin, hftest_end - hftest_begin);
+
+	if (!fdt_root_node(&n, fdt)) {
+		HFTEST_LOG("FDT failed validation.");
+		return;
+	}
+
+	if (!fdt_find_child(&n, "")) {
+		HFTEST_LOG("Unable to find root node in FDT.");
+		return;
+	}
+
+	if (!fdt_find_child(&n, "chosen")) {
+		HFTEST_LOG("Unable to find 'chosen' node in FDT.");
+		return;
+	}
+
+	if (!fdt_read_property(&n, "bootargs", &bootargs, &bootargs_size)) {
+		HFTEST_LOG("Unable to read bootargs.");
+		return;
+	}
+
+	/* Remove null terminator. */
+	memiter_init(&bootargs_iter, bootargs, bootargs_size - 1);
+
+	if (!memiter_parse_str(&bootargs_iter, &command)) {
+		HFTEST_LOG("Unable to parse command.");
+		return;
+	}
+
+	if (memiter_iseq(&command, "json")) {
+		hftest_json();
+		return;
+	}
+
+	if (memiter_iseq(&command, "run")) {
+		struct memiter suite_name;
+		struct memiter test_name;
+
+		if (!memiter_parse_str(&bootargs_iter, &suite_name)) {
+			HFTEST_LOG("Unable to parse test suite.");
+			return;
+		}
+
+		if (!memiter_parse_str(&bootargs_iter, &test_name)) {
+			HFTEST_LOG("Unable to parse test.");
+			return;
+		}
+		hftest_run(suite_name, test_name, fdt);
+		return;
+	}
+
+	hftest_help();
+}
diff --git a/test/inc/test/hftest.h b/test/inc/test/hftest.h
new file mode 100644
index 0000000..bcd22d8
--- /dev/null
+++ b/test/inc/test/hftest.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hf/dlog.h"
+
+/*
+ * Define a set up function to be run before every test in a test suite.
+ */
+#define SET_UP(suite) HFTEST_SET_UP(suite)
+
+/*
+ * Define a tear down function to be run after every test in a test suite.
+ */
+#define TEAR_DOWN(suite) HFTEST_TEAR_DOWN(suite)
+
+/*
+ * Define a test as part of a test suite.
+ */
+#define TEST(suite, test) HFTEST_TEST(suite, test, false)
+
+/*
+ * Define a test as part of a test suite and mark it long-running.
+ */
+#define TEST_LONG_RUNNING(suite, test) HFTEST_TEST(suite, test, true)
+
+/*
+ * Define a test service.
+ */
+#define TEST_SERVICE(service) HFTEST_TEST_SERVICE(service)
+
+/* Assertions. */
+#define ASSERT_EQ(x, y) HFTEST_ASSERT_OP(x, y, ==, true)
+#define ASSERT_NE(x, y) HFTEST_ASSERT_OP(x, y, !=, true)
+#define ASSERT_LE(x, y) HFTEST_ASSERT_OP(x, y, <=, true)
+#define ASSERT_LT(x, y) HFTEST_ASSERT_OP(x, y, <, true)
+#define ASSERT_GE(x, y) HFTEST_ASSERT_OP(x, y, >=, true)
+#define ASSERT_GT(x, y) HFTEST_ASSERT_OP(x, y, >, true)
+
+#define ASSERT_TRUE(x) ASSERT_EQ(x, true)
+#define ASSERT_FALSE(x) ASSERT_EQ(x, false)
+
+#define EXPECT_EQ(x, y) HFTEST_ASSERT_OP(x, y, ==, false)
+#define EXPECT_NE(x, y) HFTEST_ASSERT_OP(x, y, !=, false)
+#define EXPECT_LE(x, y) HFTEST_ASSERT_OP(x, y, <=, false)
+#define EXPECT_LT(x, y) HFTEST_ASSERT_OP(x, y, <, false)
+#define EXPECT_GE(x, y) HFTEST_ASSERT_OP(x, y, >=, false)
+#define EXPECT_GT(x, y) HFTEST_ASSERT_OP(x, y, >, false)
+
+#define EXPECT_TRUE(x) EXPECT_EQ(x, true)
+#define EXPECT_FALSE(x) EXPECT_EQ(x, false)
+
+#define FAIL(...) HFTEST_FAIL(true, __VA_ARGS__)
+
+/* Service utilities. */
+#define SERVICE_NAME_MAX_LENGTH 64
+#define SERVICE_SELECT(vm_id, service, send_buffer) \
+	HFTEST_SERVICE_SELECT(vm_id, service, send_buffer)
+
+#define SERVICE_SEND_BUFFER() HFTEST_SERVICE_SEND_BUFFER()
+#define SERVICE_RECV_BUFFER() HFTEST_SERVICE_RECV_BUFFER()
+#define SERVICE_MEMORY_SIZE() HFTEST_SERVICE_MEMORY_SIZE()
+
+/*
+ * This must be used exactly once in a test image to signal to the linker that
+ * the .hftest section is allowed to be included in the generated image.
+ */
+#define HFTEST_ENABLE() int hftest_enable
+
+/*
+ * Prefixed to log lines from tests for easy filtering in the console.
+ */
+#define HFTEST_LOG_PREFIX "[hftest] "
+
+/*
+ * Indentation used e.g. to give the reason for an assertion failure.
+ */
+#define HFTEST_LOG_INDENT "    "
+
+/** Initializes stage-1 MMU for tests running in a VM. */
+bool hftest_mm_init(void);
+
+/** Adds stage-1 identity mapping for pages covering bytes [base, base+size). */
+void hftest_mm_identity_map(const void *base, size_t size, uint32_t mode);
+
+void hftest_mm_vcpu_init(void);
+
+/**
+ * Starts the CPU with the given ID. It will start at the provided entry point
+ * with the provided argument. It is a wrapper around the generic cpu_start()
+ * and takes care of MMU initialization.
+ */
+bool hftest_cpu_start(uintptr_t id, void *stack, size_t stack_size,
+		      void (*entry)(uintptr_t arg), uintptr_t arg);
+
+uintptr_t hftest_get_cpu_id(size_t index);
+
+/* Above this point is the public API. Now include the implementation. */
+#include "hftest_impl.h"
diff --git a/test/inc/test/hftest_impl.h b/test/inc/test/hftest_impl.h
new file mode 100644
index 0000000..ed2fa91
--- /dev/null
+++ b/test/inc/test/hftest_impl.h
@@ -0,0 +1,310 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdnoreturn.h>
+
+#include "hf/fdt.h"
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/spci.h"
+
+#define HFTEST_MAX_TESTS 50
+
+/*
+ * Log with the HFTEST_LOG_PREFIX and a new line. The zero is added so there is
+ * always at least one variadic argument.
+ */
+#define HFTEST_LOG(...) HFTEST_LOG_IMPL(__VA_ARGS__, 0)
+#define HFTEST_LOG_IMPL(format, ...) \
+	dlog("%s" format "\n", HFTEST_LOG_PREFIX, __VA_ARGS__)
+
+/* Helper to wrap the argument in quotes. */
+#define HFTEST_STR(str) #str
+
+/*
+ * Sections are names such that when the linker sorts them, all entries for the
+ * same test suite are contiguous and the set up and tear down entries come
+ * before the tests. This order simplifies test discovery in the running image.
+ */
+#define HFTEST_SET_UP_SECTION(suite_name) \
+	HFTEST_STR(.hftest.suite.suite_name .1set_up)
+#define HFTEST_TEAR_DOWN_SECTION(suite_name) \
+	HFTEST_STR(.hftest.suite.suite_name .1tear_down)
+#define HFTEST_TEST_SECTION(suite_name, test_name) \
+	HFTEST_STR(.hftest.suite.suite_name .2test.test_name)
+#define HFTEST_SERVICE_SECTION(service_name) \
+	HFTEST_STR(.hftest.service.service_name)
+
+/* Helpers to construct unique identifiers. */
+#define HFTEST_SET_UP_STRUCT(suite_name) hftest_set_up_##suite_name
+#define HFTEST_TEAR_DOWN_STRUCT(suite_name) hftest_tear_down_##suite_name
+#define HFTEST_TEST_STRUCT(suite_name, test_name) \
+	hftest_test_##suite_name##_##test_name
+#define HFTEST_SERVICE_STRUCT(service_name) hftest_service_##service_name
+
+#define HFTEST_SET_UP_FN(suite_name) hftest_set_up_fn_##suite_name
+#define HFTEST_TEAR_DOWN_FN(suite_name) hftest_tear_down_fn_##suite_name
+#define HFTEST_TEST_FN(suite_name, test_name) \
+	hftest_test_fn_##suite_name##_##test_name
+#define HFTEST_SERVICE_FN(service_name) hftest_service_fn_##service_name
+
+#define HFTEST_SET_UP_CONSTRUCTOR(suite_name) hftest_set_up_ctor_##suite_name
+#define HFTEST_TEAR_DOWN_CONSTRUCTOR(suite_name) \
+	hftest_tear_down_ctor_##suite_name
+#define HFTEST_TEST_CONSTRUCTOR(suite_name, test_name) \
+	hftest_test_ctor_##suite_name##_##test_name
+
+/* Register test functions. */
+#define HFTEST_SET_UP(suite_name)                                           \
+	static void HFTEST_SET_UP_FN(suite_name)(void);                     \
+	const struct hftest_test __attribute__((used))                      \
+		__attribute__((section(HFTEST_SET_UP_SECTION(suite_name)))) \
+			HFTEST_SET_UP_STRUCT(suite_name) = {                \
+				.suite = #suite_name,                       \
+				.kind = HFTEST_KIND_SET_UP,                 \
+				.fn = HFTEST_SET_UP_FN(suite_name),         \
+	};                                                                  \
+	static void __attribute__((constructor))                            \
+		HFTEST_SET_UP_CONSTRUCTOR(suite_name)(void)                 \
+	{                                                                   \
+		hftest_register(HFTEST_SET_UP_STRUCT(suite_name));          \
+	}                                                                   \
+	static void HFTEST_SET_UP_FN(suite_name)(void)
+
+#define HFTEST_TEAR_DOWN(suite_name)                                           \
+	static void HFTEST_TEAR_DOWN_FN(suite_name)(void);                     \
+	const struct hftest_test __attribute__((used))                         \
+		__attribute__((section(HFTEST_TEAR_DOWN_SECTION(suite_name)))) \
+			HFTEST_TEAR_DOWN_STRUCT(suite_name) = {                \
+				.suite = #suite_name,                          \
+				.kind = HFTEST_KIND_TEAR_DOWN,                 \
+				.fn = HFTEST_TEAR_DOWN_FN(suite_name),         \
+	};                                                                     \
+	static void __attribute__((constructor))                               \
+		HFTEST_TEAR_DOWN_CONSTRUCTOR(suite_name)(void)                 \
+	{                                                                      \
+		hftest_register(HFTEST_TEAR_DOWN_STRUCT(suite_name));          \
+	}                                                                      \
+	static void HFTEST_TEAR_DOWN_FN(suite_name)(void)
+
+#define HFTEST_TEST(suite_name, test_name, long_running)                    \
+	static void HFTEST_TEST_FN(suite_name, test_name)(void);            \
+	const struct hftest_test __attribute__((used)) __attribute__(       \
+		(section(HFTEST_TEST_SECTION(suite_name, test_name))))      \
+		HFTEST_TEST_STRUCT(suite_name, test_name) = {               \
+			.suite = #suite_name,                               \
+			.kind = HFTEST_KIND_TEST,                           \
+			.name = #test_name,                                 \
+			.is_long_running = long_running,                    \
+			.fn = HFTEST_TEST_FN(suite_name, test_name),        \
+	};                                                                  \
+	static void __attribute__((constructor))                            \
+		HFTEST_TEST_CONSTRUCTOR(suite_name, test_name)(void)        \
+	{                                                                   \
+		hftest_register(HFTEST_TEST_STRUCT(suite_name, test_name)); \
+	}                                                                   \
+	static void HFTEST_TEST_FN(suite_name, test_name)(void)
+
+#define HFTEST_TEST_SERVICE(service_name)                                      \
+	static void HFTEST_SERVICE_FN(service_name)(void);                     \
+	const struct hftest_test __attribute__((used))                         \
+		__attribute__((section(HFTEST_SERVICE_SECTION(service_name)))) \
+			HFTEST_SERVICE_STRUCT(service_name) = {                \
+				.kind = HFTEST_KIND_SERVICE,                   \
+				.name = #service_name,                         \
+				.fn = HFTEST_SERVICE_FN(service_name),         \
+	};                                                                     \
+	static void HFTEST_SERVICE_FN(service_name)(void)
+
+/* Context for tests. */
+struct hftest_context {
+	uint32_t failures;
+	noreturn void (*abort)(void);
+
+	/* These are used in primary VMs. */
+	const struct fdt_header *fdt;
+
+	/* These are used in services. */
+	void *send;
+	void *recv;
+	size_t memory_size;
+};
+
+struct hftest_context *hftest_get_context(void);
+
+/* A test case. */
+typedef void (*hftest_test_fn)(void);
+
+enum hftest_kind {
+	HFTEST_KIND_SET_UP = 0,
+	HFTEST_KIND_TEST = 1,
+	HFTEST_KIND_TEAR_DOWN = 2,
+	HFTEST_KIND_SERVICE = 3,
+};
+
+/**
+ * The .hftest section contains an array of this struct which describes the test
+ * functions contained in the image allowing the image to inspect the tests it
+ * contains.
+ */
+struct hftest_test {
+	const char *suite;
+	enum hftest_kind kind;
+	const char *name;
+	bool is_long_running;
+	hftest_test_fn fn;
+};
+
+/*
+ * This union can store any of the primitive types supported by the assertion
+ * macros.
+ *
+ * It does not include pointers as comparison of pointers is not often needed
+ * and could be a mistake for string comparison. If pointer comparison is needed
+ * and explicit assertion such as ASSERT_PTR_EQ() would be more appropriate.
+ */
+union hftest_any {
+	bool b;
+	char c;
+	signed char sc;
+	unsigned char uc;
+	signed short ss;
+	unsigned short us;
+	signed int si;
+	unsigned int ui;
+	signed long int sli;
+	unsigned long int uli;
+	signed long long int slli;
+	unsigned long long int ulli;
+};
+
+/* _Generic formatting doesn't seem to be supported so doing this manually. */
+/* clang-format off */
+
+/* Select the union member to match the type of the expression. */
+#define hftest_any_get(any, x)                      \
+	_Generic((x),                               \
+		bool:                   (any).b,    \
+		char:                   (any).c,    \
+		signed char:            (any).sc,   \
+		unsigned char:          (any).uc,   \
+		signed short:           (any).ss,   \
+		unsigned short:         (any).us,   \
+		signed int:             (any).si,   \
+		unsigned int:           (any).ui,   \
+		signed long int:        (any).sli,  \
+		unsigned long int:      (any).uli,  \
+		signed long long int:   (any).slli, \
+		unsigned long long int: (any).ulli)
+
+/*
+ * dlog format specifier for types. Note, these aren't the standard specifiers
+ * for the types.
+ */
+#define hftest_dlog_format(x)                 \
+	_Generic((x),                         \
+		bool:                   "%u", \
+		char:                   "%c", \
+		signed char:            "%d", \
+		unsigned char:          "%u", \
+		signed short:           "%d", \
+		unsigned short:         "%u", \
+		signed int:             "%d", \
+		unsigned int:           "%u", \
+		signed long int:        "%d", \
+		unsigned long int:      "%u", \
+		signed long long int:   "%d", \
+		unsigned long long int: "%u")
+
+/* clang-format on */
+
+#define HFTEST_LOG_FAILURE() \
+	dlog(HFTEST_LOG_PREFIX "Failure: %s:%u\n", __FILE__, __LINE__);
+
+#define HFTEST_ASSERT_OP(lhs, rhs, op, fatal)                              \
+	do {                                                               \
+		union hftest_any lhs_value;                                \
+		union hftest_any rhs_value;                                \
+		hftest_any_get(lhs_value, lhs) = (lhs);                    \
+		hftest_any_get(rhs_value, rhs) = (rhs);                    \
+		if (!(hftest_any_get(lhs_value, lhs)                       \
+			      op hftest_any_get(rhs_value, rhs))) {        \
+			struct hftest_context *ctx = hftest_get_context(); \
+			++ctx->failures;                                   \
+			HFTEST_LOG_FAILURE();                              \
+			dlog(HFTEST_LOG_PREFIX HFTEST_LOG_INDENT           \
+			     "%s %s %s (%s=",                              \
+			     #lhs, #op, #rhs, #lhs);                       \
+			dlog(hftest_dlog_format(lhs),                      \
+			     hftest_any_get(lhs_value, lhs));              \
+			dlog(", %s=", #rhs);                               \
+			dlog(hftest_dlog_format(rhs),                      \
+			     hftest_any_get(rhs_value, rhs));              \
+			dlog(")\n");                                       \
+			if (fatal) {                                       \
+				ctx->abort();                              \
+			}                                                  \
+		}                                                          \
+	} while (0)
+
+#define HFTEST_FAIL(fatal, ...)                                        \
+	do {                                                           \
+		struct hftest_context *ctx = hftest_get_context();     \
+		++ctx->failures;                                       \
+		HFTEST_LOG_FAILURE();                                  \
+		dlog(HFTEST_LOG_PREFIX HFTEST_LOG_INDENT __VA_ARGS__); \
+		dlog("\n");                                            \
+		if (fatal) {                                           \
+			ctx->abort();                                  \
+		}                                                      \
+	} while (0)
+
+/**
+ * Select the service to run in a service VM.
+ */
+#define HFTEST_SERVICE_SELECT(vm_id, service, send_buffer)                    \
+	do {                                                                  \
+		struct spci_value run_res;                                    \
+		uint32_t msg_length =                                         \
+			strnlen_s(service, SERVICE_NAME_MAX_LENGTH);          \
+                                                                              \
+		/*                                                            \
+		 * Let the service configure its mailbox and wait for a       \
+		 * message.                                                   \
+		 */                                                           \
+		run_res = spci_run(vm_id, 0);                                 \
+		ASSERT_EQ(run_res.func, SPCI_MSG_WAIT_32);                    \
+		ASSERT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);               \
+                                                                              \
+		/* Send the selected service to run and let it be handled. */ \
+		memcpy_s(send_buffer, SPCI_MSG_PAYLOAD_MAX, service,          \
+			 msg_length);                                         \
+                                                                              \
+		ASSERT_EQ(spci_msg_send(hf_vm_get_id(), vm_id, msg_length, 0) \
+				  .func,                                      \
+			  SPCI_SUCCESS_32);                                   \
+		run_res = spci_run(vm_id, 0);                                 \
+		ASSERT_EQ(run_res.func, SPCI_YIELD_32);                       \
+	} while (0)
+
+#define HFTEST_SERVICE_SEND_BUFFER() hftest_get_context()->send
+#define HFTEST_SERVICE_RECV_BUFFER() hftest_get_context()->recv
+#define HFTEST_SERVICE_MEMORY_SIZE() hftest_get_context()->memory_size
+
+void hftest_register(struct hftest_test test);
diff --git a/test/inc/test/vmapi/exception_handler.h b/test/inc/test/vmapi/exception_handler.h
new file mode 100644
index 0000000..07ef312
--- /dev/null
+++ b/test/inc/test/vmapi/exception_handler.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "vmapi/hf/spci.h"
+
+bool exception_handler_skip_instruction(void);
+
+bool exception_handler_yield(void);
+
+int exception_handler_get_num(void);
+
+void exception_handler_reset(void);
+
+void exception_handler_send_exception_count(void);
+
+int exception_handler_receive_exception_count(
+	const struct spci_value *send_res,
+	const struct spci_memory_region *recv_buf);
diff --git a/test/inc/test/vmapi/spci.h b/test/inc/test/vmapi/spci.h
new file mode 100644
index 0000000..3ecebf6
--- /dev/null
+++ b/test/inc/test/vmapi/spci.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "vmapi/hf/spci.h"
+
+#define EXPECT_SPCI_ERROR(value, spci_error)          \
+	do {                                          \
+		EXPECT_EQ(value.func, SPCI_ERROR_32); \
+		EXPECT_EQ(value.arg2, spci_error);    \
+	} while (0)
+
+struct mailbox_buffers {
+	void *send;
+	void *recv;
+};
+
+struct mailbox_buffers set_up_mailbox(void);
diff --git a/test/linux/BUILD.gn b/test/linux/BUILD.gn
new file mode 100644
index 0000000..0be5630
--- /dev/null
+++ b/test/linux/BUILD.gn
@@ -0,0 +1,96 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+import("//build/toolchain/platform.gni")
+
+executable("test_binary") {
+  include_dirs = [
+    "//driver/linux/inc/uapi",
+    "//third_party/linux/include/uapi",
+  ]
+
+  testonly = true
+  sources = [
+    "linux.c",
+  ]
+  deps = [
+    "//test/hftest:hftest_linux",
+  ]
+  output_name = "test_binary"
+}
+
+vm_kernel("socket_vm1") {
+  testonly = true
+
+  deps = [
+    ":hftest_secondary_vm_socket",
+  ]
+}
+
+linux_initrd("linux_test_initrd") {
+  testonly = true
+
+  # Always use the aarch64_linux_clang toolchain to build test_binary
+  test_binary_target = ":test_binary(//build/toolchain:aarch64_linux_clang)"
+  sources = [
+    get_label_info(test_binary_target, "root_out_dir") + "/test_binary",
+    get_label_info("//driver/linux", "target_out_dir") + "/hafnium.ko",
+  ]
+  deps = [
+    "//driver/linux",
+    test_binary_target,
+  ]
+}
+
+initrd("linux_test") {
+  testonly = true
+
+  manifest = "manifest.dts"
+  primary_name = "vmlinuz"
+  primary_vm = "//third_party/linux:linux__prebuilt"
+  primary_initrd = ":linux_test_initrd"
+  secondary_vms = [ [
+        "socket0",
+        ":socket_vm1",
+      ] ]
+}
+
+group("linux") {
+  testonly = true
+
+  deps = [
+    ":linux_test",
+  ]
+}
+
+# Testing framework for a secondary VM with socket.
+source_set("hftest_secondary_vm_socket") {
+  testonly = true
+
+  configs += [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "hftest_socket.c",
+  ]
+
+  deps = [
+    "//src:dlog",
+    "//src:panic",
+    "//src:std",
+    "//src/arch/${plat_arch}:entry",
+    "//src/arch/${plat_arch}/hftest:entry",
+    "//src/arch/${plat_arch}/hftest:power_mgmt",
+  ]
+}
diff --git a/test/linux/hftest_socket.c b/test/linux/hftest_socket.c
new file mode 100644
index 0000000..b25691f
--- /dev/null
+++ b/test/linux/hftest_socket.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/memiter.h"
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+#include "vmapi/hf/transport.h"
+
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+alignas(4096) uint8_t kstack[4096];
+
+static alignas(HF_MAILBOX_SIZE) uint8_t send[HF_MAILBOX_SIZE];
+static alignas(HF_MAILBOX_SIZE) uint8_t recv[HF_MAILBOX_SIZE];
+
+static hf_ipaddr_t send_addr = (hf_ipaddr_t)send;
+static hf_ipaddr_t recv_addr = (hf_ipaddr_t)recv;
+
+static struct hftest_context global_context;
+
+struct hftest_context *hftest_get_context(void)
+{
+	return &global_context;
+}
+
+noreturn void abort(void)
+{
+	HFTEST_LOG("Service contained failures.");
+	/* Cause a fault, as a secondary can't power down the machine. */
+	*((volatile uint8_t *)1) = 1;
+
+	/* This should never be reached, but to make the compiler happy... */
+	for (;;) {
+	}
+}
+
+static void swap(uint64_t *a, uint64_t *b)
+{
+	uint64_t t = *a;
+	*a = *b;
+	*b = t;
+}
+
+noreturn void kmain(size_t memory_size)
+{
+	struct hftest_context *ctx;
+
+	/* Prepare the context. */
+
+	/* Set up the mailbox. */
+	spci_rxtx_map(send_addr, recv_addr);
+
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+
+	/* Clean the context. */
+	ctx = hftest_get_context();
+	memset_s(ctx, sizeof(*ctx), 0, sizeof(*ctx));
+	ctx->abort = abort;
+	ctx->send = send;
+	ctx->recv = recv;
+	ctx->memory_size = memory_size;
+
+	for (;;) {
+		struct spci_value ret;
+
+		/* Receive the packet. */
+		ret = spci_msg_wait();
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		EXPECT_LE(spci_msg_send_size(ret), SPCI_MSG_PAYLOAD_MAX);
+
+		/* Echo the message back to the sender. */
+		memcpy_s(send, SPCI_MSG_PAYLOAD_MAX, recv,
+			 spci_msg_send_size(ret));
+
+		/* Swap the socket's source and destination ports */
+		struct hf_msg_hdr *hdr = (struct hf_msg_hdr *)send;
+		swap(&(hdr->src_port), &(hdr->dst_port));
+
+		/* Swap the destination and source ids. */
+		spci_vm_id_t dst_id = spci_msg_send_sender(ret);
+		spci_vm_id_t src_id = spci_msg_send_receiver(ret);
+
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		EXPECT_EQ(spci_msg_send(src_id, dst_id, spci_msg_send_size(ret),
+					0)
+				  .func,
+			  SPCI_SUCCESS_32);
+	}
+}
diff --git a/test/linux/linux.c b/test/linux/linux.c
new file mode 100644
index 0000000..d98d412
--- /dev/null
+++ b/test/linux/linux.c
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "hf/dlog.h"
+#include "hf/socket.h"
+
+#include "test/hftest.h"
+#include <sys/socket.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+
+#define MAX_BUF_SIZE 256
+
+static int finit_module(int fd, const char *param_values, int flags)
+{
+	return (int)syscall(SYS_finit_module, fd, param_values, flags);
+}
+
+static int delete_module(const char *name, int flags)
+{
+	return (int)syscall(SYS_delete_module, name, flags);
+}
+
+static void insmod_hafnium(void)
+{
+	int module_file = open("/hafnium.ko", O_RDONLY);
+	if (module_file < 0) {
+		FAIL("Failed to load Hafnium kernel module from /hafnium.ko");
+		return;
+	}
+	EXPECT_EQ(finit_module(module_file, "", 0), 0);
+	close(module_file);
+}
+
+static void rmmod_hafnium(void)
+{
+	EXPECT_EQ(delete_module("hafnium", 0), 0);
+}
+
+/**
+ * Loads and unloads the Hafnium kernel module.
+ */
+TEST(linux, load_hafnium)
+{
+	insmod_hafnium();
+	rmmod_hafnium();
+}
+
+/**
+ * Uses the kernel module to send a socket message from the primary VM to a
+ * secondary VM and echoes it back to the primary.
+ */
+TEST(linux, socket_echo_hafnium)
+{
+	spci_vm_id_t vm_id = HF_VM_ID_OFFSET + 1;
+	int port = 10;
+	int socket_id;
+	struct hf_sockaddr addr;
+	const char send_buf[] = "The quick brown fox jumps over the lazy dogs.";
+	size_t send_len = strlen(send_buf);
+	char resp_buf[MAX_BUF_SIZE];
+	ssize_t recv_len;
+
+	ASSERT_LT(send_len, MAX_BUF_SIZE);
+
+	insmod_hafnium();
+
+	/* Create Hafnium socket. */
+	socket_id = socket(PF_HF, SOCK_DGRAM, 0);
+	if (socket_id == -1) {
+		FAIL("Socket creation failed: %s", strerror(errno));
+		return;
+	}
+	HFTEST_LOG("Socket created successfully.");
+
+	/* Connect to requested VM & port. */
+	addr.family = PF_HF;
+	addr.vm_id = vm_id;
+	addr.port = port;
+	if (connect(socket_id, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
+		FAIL("Socket connection failed: %s", strerror(errno));
+		return;
+	}
+	HFTEST_LOG("Socket to secondary VM %d connected on port %d.", vm_id,
+		   port);
+
+	/*
+	 * Send a message to the secondary VM.
+	 * Enable the confirm flag to try again in case port is busy.
+	 */
+	if (send(socket_id, send_buf, send_len, MSG_CONFIRM) < 0) {
+		FAIL("Socket send() failed: %s", strerror(errno));
+		return;
+	}
+	HFTEST_LOG("Packet with length %d sent.", send_len);
+
+	/* Receive a response, which should be an echo of the sent packet. */
+	recv_len = recv(socket_id, resp_buf, sizeof(resp_buf) - 1, 0);
+
+	if (recv_len == -1) {
+		FAIL("Socket recv() failed: %s", strerror(errno));
+		return;
+	}
+	HFTEST_LOG("Packet with length %d received.", recv_len);
+
+	EXPECT_EQ(recv_len, send_len);
+	EXPECT_EQ(memcmp(send_buf, resp_buf, send_len), 0);
+
+	EXPECT_EQ(close(socket_id), 0);
+	rmmod_hafnium();
+}
diff --git a/test/linux/manifest.dts b/test/linux/manifest.dts
new file mode 100644
index 0000000..29e2075
--- /dev/null
+++ b/test/linux/manifest.dts
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+		vm1 {
+			debug_name = "linux_test";
+			kernel_filename = "vmlinuz";
+			ramdisk_filename = "initrd.img";
+		};
+
+		vm2 {
+			debug_name = "socket0";
+			vcpu_count = <1>;
+			mem_size = <0x100000>;
+			kernel_filename = "socket0";
+		};
+	};
+};
diff --git a/test/vmapi/BUILD.gn b/test/vmapi/BUILD.gn
new file mode 100644
index 0000000..c981e76
--- /dev/null
+++ b/test/vmapi/BUILD.gn
@@ -0,0 +1,27 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+import("//build/toolchain/platform.gni")
+
+group("vmapi") {
+  testonly = true
+
+  deps = [
+    "arch/${plat_arch}:arch",
+    "common:common",
+    "primary_only:primary_only_test",
+    "primary_with_secondaries:primary_with_secondaries_test",
+  ]
+}
diff --git a/test/vmapi/arch/aarch64/BUILD.gn b/test/vmapi/arch/aarch64/BUILD.gn
new file mode 100644
index 0000000..8441677
--- /dev/null
+++ b/test/vmapi/arch/aarch64/BUILD.gn
@@ -0,0 +1,48 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+
+group("arch") {
+  testonly = true
+
+  deps = [
+    ":aarch64_test",
+  ]
+}
+
+# Tests specific to aarch64.
+vm_kernel("aarch64_test_vm") {
+  testonly = true
+  public_configs = [ "//src/arch/aarch64:config" ]
+
+  sources = [
+    "arch_features.c",
+    "smc_whitelist.c",
+    "smccc.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64:arch",
+    "//test/hftest:hftest_primary_vm",
+  ]
+}
+
+initrd("aarch64_test") {
+  testonly = true
+
+  manifest = "manifest.dts"
+  primary_name = "aarch64_test"
+  primary_vm = ":aarch64_test_vm"
+}
diff --git a/test/vmapi/arch/aarch64/arch_features.c b/test/vmapi/arch/aarch64/arch_features.c
new file mode 100644
index 0000000..66f29db
--- /dev/null
+++ b/test/vmapi/arch/aarch64/arch_features.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "test/hftest.h"
+
+/**
+ * Test that encoding a system register using the implementation defined syntax
+ * maps to the same register defined by name.
+ */
+TEST(arch_features, read_write_msr_impdef)
+{
+	uintreg_t value = 0xa;
+	write_msr(S3_3_C9_C13_0, value);
+	EXPECT_EQ(read_msr(S3_3_C9_C13_0), value);
+	EXPECT_EQ(read_msr(PMCCNTR_EL0), value);
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/BUILD.gn b/test/vmapi/arch/aarch64/gicv3/BUILD.gn
new file mode 100644
index 0000000..57ceb99
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/BUILD.gn
@@ -0,0 +1,52 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+import("//build/toolchain/platform.gni")
+
+config("config") {
+  include_dirs = [ "inc" ]
+}
+
+# Tests specific to GICv3.
+vm_kernel("gicv3_test_vm") {
+  testonly = true
+  public_configs = [ ":config" ]
+
+  sources = [
+    "busy_secondary.c",
+    "gicv3.c",
+    "interrupts.c",
+    "timer_secondary.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64:arch",
+    "//src/arch/aarch64/hftest:interrupts",
+    "//src/arch/aarch64/hftest:interrupts_gicv3",
+    "//test/hftest:hftest_primary_vm",
+  ]
+}
+
+initrd("gicv3_test") {
+  testonly = true
+
+  manifest = "manifest.dts"
+  primary_name = "gicv3_test"
+  primary_vm = ":gicv3_test_vm"
+  secondary_vms = [ [
+        "services1",
+        "services:gicv3_service_vm1",
+      ] ]
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/busy_secondary.c b/test/vmapi/arch/aarch64/gicv3/busy_secondary.c
new file mode 100644
index 0000000..8e39a2e
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/busy_secondary.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts_gicv3.h"
+
+#include "hf/dlog.h"
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "gicv3.h"
+#include "msr.h"
+#include "test/hftest.h"
+
+/**
+ * Converts a number of nanoseconds to the equivalent number of timer ticks.
+ */
+static uint64_t ns_to_ticks(uint64_t ns)
+{
+	return ns * read_msr(cntfrq_el0) / NANOS_PER_UNIT;
+}
+
+SET_UP(busy_secondary)
+{
+	system_setup();
+	EXPECT_EQ(spci_rxtx_map(send_page_addr, recv_page_addr).func,
+		  SPCI_SUCCESS_32);
+	SERVICE_SELECT(SERVICE_VM1, "busy", send_buffer);
+}
+
+TEST(busy_secondary, virtual_timer)
+{
+	const char message[] = "loop";
+	struct spci_value run_res;
+
+	interrupt_enable(VIRTUAL_TIMER_IRQ, true);
+	interrupt_set_priority(VIRTUAL_TIMER_IRQ, 0x80);
+	interrupt_set_edge_triggered(VIRTUAL_TIMER_IRQ, true);
+	/*
+	 * Hypervisor timer IRQ is needed for Hafnium to return control to the
+	 * primary if the (emulated) virtual timer fires while the secondary is
+	 * running.
+	 */
+	interrupt_enable(HYPERVISOR_TIMER_IRQ, true);
+	interrupt_set_priority(HYPERVISOR_TIMER_IRQ, 0x80);
+	interrupt_set_edge_triggered(HYPERVISOR_TIMER_IRQ, true);
+	interrupt_set_priority_mask(0xff);
+	arch_irq_enable();
+
+	/* Let the secondary get started and wait for our message. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Check that no interrupts are active or pending to start with. */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+
+	dlog("Starting timer\n");
+	/* Set virtual timer for 1 mS and enable. */
+	write_msr(CNTV_TVAL_EL0, ns_to_ticks(1000000));
+	write_msr(CNTV_CTL_EL0, 0x00000001);
+
+	/* Let secondary start looping. */
+	dlog("Telling secondary to loop.\n");
+	memcpy_s(send_buffer, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_INTERRUPT_32);
+
+	dlog("Waiting for interrupt\n");
+	while (last_interrupt_id == 0) {
+		EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+		EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+	}
+
+	/* Check that we got the interrupt. */
+	dlog("Checking for interrupt\n");
+	EXPECT_EQ(last_interrupt_id, VIRTUAL_TIMER_IRQ);
+	/* Check timer status. */
+	EXPECT_EQ(read_msr(CNTV_CTL_EL0), 0x00000005);
+
+	/* There should again be no pending or active interrupts. */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+}
+
+TEST(busy_secondary, physical_timer)
+{
+	const char message[] = "loop";
+	struct spci_value run_res;
+
+	interrupt_enable(PHYSICAL_TIMER_IRQ, true);
+	interrupt_set_priority(PHYSICAL_TIMER_IRQ, 0x80);
+	interrupt_set_edge_triggered(PHYSICAL_TIMER_IRQ, true);
+	interrupt_set_priority_mask(0xff);
+	arch_irq_enable();
+
+	/* Let the secondary get started and wait for our message. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Check that no interrupts are active or pending to start with. */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+
+	dlog("Starting timer\n");
+	/* Set physical timer for 1 ms and enable. */
+	write_msr(CNTP_TVAL_EL0, ns_to_ticks(1000000));
+	write_msr(CNTP_CTL_EL0, 0x00000001);
+
+	/* Let secondary start looping. */
+	dlog("Telling secondary to loop.\n");
+	memcpy_s(send_buffer, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_INTERRUPT_32);
+
+	dlog("Waiting for interrupt\n");
+	while (last_interrupt_id == 0) {
+		EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+		EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+	}
+
+	/* Check that we got the interrupt. */
+	dlog("Checking for interrupt\n");
+	EXPECT_EQ(last_interrupt_id, PHYSICAL_TIMER_IRQ);
+	/* Check timer status. */
+	EXPECT_EQ(read_msr(CNTP_CTL_EL0), 0x00000005);
+
+	/* There should again be no pending or active interrupts. */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/gicv3.c b/test/vmapi/arch/aarch64/gicv3/gicv3.c
new file mode 100644
index 0000000..13c4565
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/gicv3.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "gicv3.h"
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts.h"
+#include "hf/arch/vm/interrupts_gicv3.h"
+
+#include "hf/dlog.h"
+#include "hf/mm.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "test/hftest.h"
+
+alignas(PAGE_SIZE) uint8_t send_page[PAGE_SIZE];
+alignas(PAGE_SIZE) uint8_t recv_page[PAGE_SIZE];
+
+hf_ipaddr_t send_page_addr = (hf_ipaddr_t)send_page;
+hf_ipaddr_t recv_page_addr = (hf_ipaddr_t)recv_page;
+
+void *send_buffer = send_page;
+void *recv_buffer = recv_page;
+
+volatile uint32_t last_interrupt_id = 0;
+
+static void irq(void)
+{
+	uint32_t interrupt_id = interrupt_get_and_acknowledge();
+	dlog("primary IRQ %d from current\n", interrupt_id);
+	last_interrupt_id = interrupt_id;
+	interrupt_end(interrupt_id);
+	dlog("primary IRQ %d ended\n", interrupt_id);
+}
+
+void system_setup()
+{
+	const uint32_t mode = MM_MODE_R | MM_MODE_W | MM_MODE_D;
+	hftest_mm_identity_map((void *)GICD_BASE, PAGE_SIZE, mode);
+	hftest_mm_identity_map((void *)GICR_BASE, PAGE_SIZE, mode);
+	hftest_mm_identity_map((void *)SGI_BASE, PAGE_SIZE, mode);
+
+	exception_setup(irq, NULL);
+	interrupt_gic_setup();
+}
+
+/* Check that system registers are configured as we expect on startup. */
+TEST(system, system_registers_enabled)
+{
+	/* Check that system register interface to GICv3 is enabled. */
+	uint32_t expected_sre =
+		1U << 2 | /* Disable IRQ bypass. */
+		1U << 1 | /* Disable FIQ bypass. */
+		1U << 0;  /* Enable system register interface to GICv3. */
+	EXPECT_EQ(read_msr(ICC_SRE_EL1), expected_sre);
+}
+
+TEST(system, system_setup)
+{
+	system_setup();
+
+	/* Should have affinity routing enabled, group 1 interrupts enabled,
+	 * group 0 disabled. */
+	EXPECT_EQ(io_read32(GICD_CTLR) & 0x13, 0x12);
+	EXPECT_EQ(read_msr(ICC_CTLR_EL1) & 0xff, 0);
+}
+
+/*
+ * Check that an attempt by a secondary VM to access a GICv3 system register is
+ * trapped.
+ */
+TEST(system, icc_ctlr_access_trapped_secondary)
+{
+	struct spci_value run_res;
+
+	EXPECT_EQ(spci_rxtx_map(send_page_addr, recv_page_addr).func,
+		  SPCI_SUCCESS_32);
+	SERVICE_SELECT(SERVICE_VM1, "access_systemreg_ctlr", send_buffer);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/*
+ * Check that an attempt by a secondary VM to write ICC_SRE_EL1 is trapped or
+ * ignored.
+ */
+TEST(system, icc_sre_write_trapped_secondary)
+{
+	struct spci_value run_res;
+
+	EXPECT_EQ(spci_rxtx_map(send_page_addr, recv_page_addr).func,
+		  SPCI_SUCCESS_32);
+	SERVICE_SELECT(SERVICE_VM1, "write_systemreg_sre", send_buffer);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/inc/gicv3.h b/test/vmapi/arch/aarch64/gicv3/inc/gicv3.h
new file mode 100644
index 0000000..5fe9a52
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/inc/gicv3.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/addr.h"
+#include "hf/mm.h"
+#include "hf/types.h"
+
+#define PPI_IRQ_BASE 16
+#define PHYSICAL_TIMER_IRQ (PPI_IRQ_BASE + 14)
+#define VIRTUAL_TIMER_IRQ (PPI_IRQ_BASE + 11)
+#define HYPERVISOR_TIMER_IRQ (PPI_IRQ_BASE + 10)
+
+#define NANOS_PER_UNIT 1000000000
+
+#define SERVICE_VM1 (HF_VM_ID_OFFSET + 1)
+
+extern alignas(PAGE_SIZE) uint8_t send_page[PAGE_SIZE];
+extern alignas(PAGE_SIZE) uint8_t recv_page[PAGE_SIZE];
+
+extern hf_ipaddr_t send_page_addr;
+extern hf_ipaddr_t recv_page_addr;
+
+extern void *send_buffer;
+extern void *recv_buffer;
+
+extern volatile uint32_t last_interrupt_id;
+
+void system_setup();
diff --git a/test/vmapi/arch/aarch64/gicv3/interrupts.c b/test/vmapi/arch/aarch64/gicv3/interrupts.c
new file mode 100644
index 0000000..3472663
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/interrupts.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts_gicv3.h"
+
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "gicv3.h"
+#include "test/hftest.h"
+
+SET_UP(interrupts)
+{
+	system_setup();
+}
+
+TEST(interrupts, enable_sgi)
+{
+	/* Interrupt IDs 0 to 15 are SGIs. */
+	uint8_t intid = 3;
+	interrupt_set_priority_mask(0xff);
+	interrupt_set_priority(intid, 0x80);
+	arch_irq_enable();
+	interrupt_enable_all(true);
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+
+	/* Send ourselves the SGI. */
+	last_interrupt_id = 0xffffffff;
+	dlog("sending SGI\n");
+	interrupt_send_sgi(intid, false, 0, 0, 0, 1);
+	dlog("sent SGI\n");
+
+	/* Check that we got it, and we are back to not active or pending. */
+	EXPECT_EQ(last_interrupt_id, intid);
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+}
+
+TEST(interrupts, disable_sgi)
+{
+	/* Interrupt IDs 0 to 15 are SGIs. */
+	uint8_t intid = 3;
+	interrupt_enable_all(true);
+	interrupt_enable(intid, false);
+	interrupt_set_priority_mask(0xff);
+	interrupt_set_priority(intid, 0x80);
+	arch_irq_enable();
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+
+	/* Send ourselves the SGI. */
+	last_interrupt_id = 0xffffffff;
+	dlog("sending SGI\n");
+	interrupt_send_sgi(intid, false, 0, 0, 0, 1);
+	dlog("sent SGI\n");
+
+	/* Check that we didn't get it, but it is pending (and not active). */
+	EXPECT_EQ(last_interrupt_id, 0xffffffff);
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0x1 << intid);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+}
+
+TEST(interrupts, physical_timer)
+{
+	interrupt_enable(PHYSICAL_TIMER_IRQ, true);
+	interrupt_set_priority(PHYSICAL_TIMER_IRQ, 0x80);
+	interrupt_set_edge_triggered(PHYSICAL_TIMER_IRQ, true);
+	interrupt_set_priority_mask(0xff);
+	arch_irq_enable();
+
+	/*
+	 * Check that no (SGI or PPI) interrupts are active or pending to start
+	 * with.
+	 */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+
+	dlog("Starting timer\n");
+	/* Set physical timer for 1 tick. */
+	write_msr(CNTP_TVAL_EL0, 1);
+	/* Enable it. */
+	write_msr(CNTP_CTL_EL0, 0x00000001);
+
+	dlog("waiting for interrupt\n");
+	while (last_interrupt_id == 0) {
+		EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+		EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+	}
+
+	/* Check that we got the interrupt. */
+	dlog("Checking for interrupt\n");
+	EXPECT_EQ(last_interrupt_id, PHYSICAL_TIMER_IRQ);
+	/* Check timer status. */
+	EXPECT_EQ(read_msr(CNTP_CTL_EL0), 0x00000005);
+
+	/* There should again be no pending or active interrupts. */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+}
+
+TEST(interrupts, virtual_timer)
+{
+	interrupt_enable(VIRTUAL_TIMER_IRQ, true);
+	interrupt_set_priority(VIRTUAL_TIMER_IRQ, 0x80);
+	interrupt_set_edge_triggered(VIRTUAL_TIMER_IRQ, true);
+	interrupt_set_priority_mask(0xff);
+	arch_irq_enable();
+
+	/* Check that no interrupts are active or pending to start with. */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+
+	dlog("Starting timer\n");
+	/* Set virtual timer for 1 tick. */
+	write_msr(CNTV_TVAL_EL0, 1);
+	/* Enable it. */
+	write_msr(CNTV_CTL_EL0, 0x00000001);
+
+	dlog("Waiting for interrupt\n");
+	while (last_interrupt_id == 0) {
+		EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+		EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+		EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+	}
+
+	/* Check that we got the interrupt. */
+	dlog("Checking for interrupt\n");
+	EXPECT_EQ(last_interrupt_id, VIRTUAL_TIMER_IRQ);
+	/* Check timer status. */
+	EXPECT_EQ(read_msr(CNTV_CTL_EL0), 0x00000005);
+
+	/* There should again be no pending or active interrupts. */
+	EXPECT_EQ(io_read32_array(GICD_ISPENDR, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISPENDR0), 0);
+	EXPECT_EQ(io_read32_array(GICD_ISACTIVER, 0), 0);
+	EXPECT_EQ(io_read32(GICR_ISACTIVER0), 0);
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/manifest.dts b/test/vmapi/arch/aarch64/gicv3/manifest.dts
new file mode 100644
index 0000000..ad500e3
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/manifest.dts
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+		vm1 {
+			debug_name = "gicv3_test";
+			kernel_filename = "gicv3_test";
+		};
+
+		vm2 {
+			debug_name = "services1";
+			vcpu_count = <1>;
+			mem_size = <0x100000>;
+			kernel_filename = "services1";
+		};
+	};
+};
diff --git a/test/vmapi/arch/aarch64/gicv3/services/BUILD.gn b/test/vmapi/arch/aarch64/gicv3/services/BUILD.gn
new file mode 100644
index 0000000..6c2f019
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/services/BUILD.gn
@@ -0,0 +1,87 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+
+source_set("common") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+  sources = [
+    "common.c",
+  ]
+}
+
+# Service which loops forever.
+source_set("busy") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "busy.c",
+  ]
+
+  deps = [
+    ":common",
+  ]
+}
+
+# Service which uses timers.
+source_set("timer") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "timer.c",
+  ]
+
+  deps = [
+    ":common",
+    "//src/arch/aarch64:arch",
+    "//src/arch/aarch64/hftest:interrupts",
+  ]
+}
+
+# Service which tries to access GICv3 system registers.
+source_set("systemreg") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "systemreg.c",
+  ]
+
+  deps = [
+    ":common",
+    "//src/arch/aarch64:arch",
+  ]
+
+  include_dirs = [
+    "//test/vmapi/common",
+    "//inc/vmapi/hf",
+  ]
+}
+
+# Group services together into VMs.
+
+vm_kernel("gicv3_service_vm1") {
+  testonly = true
+
+  deps = [
+    ":busy",
+    ":systemreg",
+    ":timer",
+    "//test/hftest:hftest_secondary_vm",
+    "//test/vmapi/common:common",
+  ]
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/services/busy.c b/test/vmapi/arch/aarch64/gicv3/services/busy.c
new file mode 100644
index 0000000..fece9f6
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/services/busy.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts_gicv3.h"
+
+#include "hf/dlog.h"
+
+#include "vmapi/hf/call.h"
+
+#include "common.h"
+#include "test/hftest.h"
+
+/*
+ * Secondary VM that loops forever after receiving a message.
+ */
+
+TEST_SERVICE(busy)
+{
+	dlog("Secondary waiting for message...\n");
+	mailbox_receive_retry();
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	dlog("Secondary received message, looping forever.\n");
+	for (;;) {
+	}
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/services/common.c b/test/vmapi/arch/aarch64/gicv3/services/common.c
new file mode 100644
index 0000000..ba18d39
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/services/common.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+
+/**
+ * Try to receive a message from the mailbox, blocking if necessary, and
+ * retrying if interrupted.
+ */
+struct spci_value mailbox_receive_retry(void)
+{
+	struct spci_value received;
+
+	do {
+		received = spci_msg_wait();
+	} while (received.func == SPCI_ERROR_32 &&
+		 received.arg2 == SPCI_INTERRUPTED);
+
+	return received;
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/services/common.h b/test/vmapi/arch/aarch64/gicv3/services/common.h
new file mode 100644
index 0000000..ced8baf
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/services/common.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "vmapi/hf/spci.h"
+
+struct spci_value mailbox_receive_retry(void);
diff --git a/test/vmapi/arch/aarch64/gicv3/services/systemreg.c b/test/vmapi/arch/aarch64/gicv3/services/systemreg.c
new file mode 100644
index 0000000..78ddc0f
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/services/systemreg.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/events.h"
+#include "hf/arch/vm/interrupts.h"
+#include "hf/arch/vm/interrupts_gicv3.h"
+#include "hf/arch/vm/timer.h"
+
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "common.h"
+#include "test/hftest.h"
+#include "test/vmapi/exception_handler.h"
+
+/*
+ * Secondary VM that tries to access GICv3 system registers.
+ */
+
+TEST_SERVICE(access_systemreg_ctlr)
+{
+	exception_setup(NULL, exception_handler_skip_instruction);
+
+	/* Reading ICC_CTLR_EL1 should trap the VM. */
+	read_msr(ICC_CTLR_EL1);
+
+	/* Writing ICC_CTLR_EL1 should trap the VM. */
+	write_msr(ICC_CTLR_EL1, 0);
+
+	EXPECT_EQ(exception_handler_get_num(), 2);
+
+	/* Yield after catching the exceptions. */
+	spci_yield();
+}
+
+TEST_SERVICE(write_systemreg_sre)
+{
+	ASSERT_EQ(read_msr(ICC_SRE_EL1), 0x7);
+	/* Writing ICC_SRE_EL1 should trap the VM or be ignored. */
+	write_msr(ICC_SRE_EL1, 0x0);
+	ASSERT_EQ(read_msr(ICC_SRE_EL1), 0x7);
+	write_msr(ICC_SRE_EL1, 0xffffffff);
+	ASSERT_EQ(read_msr(ICC_SRE_EL1), 0x7);
+	spci_yield();
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/services/timer.c b/test/vmapi/arch/aarch64/gicv3/services/timer.c
new file mode 100644
index 0000000..f62aa80
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/services/timer.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/timer.h"
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/events.h"
+#include "hf/arch/vm/interrupts.h"
+#include "hf/arch/vm/interrupts_gicv3.h"
+
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "common.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+/*
+ * Secondary VM that sets timers in response to messages, and sends messages
+ * back when they fire.
+ */
+
+static volatile bool timer_fired = false;
+
+static void irq_current(void)
+{
+	uint32_t interrupt_id = hf_interrupt_get();
+	char buffer[] = "Got IRQ xx.";
+	int size = sizeof(buffer);
+	dlog("secondary IRQ %d from current\n", interrupt_id);
+	if (interrupt_id == HF_VIRTUAL_TIMER_INTID) {
+		timer_fired = true;
+	}
+	buffer[8] = '0' + interrupt_id / 10;
+	buffer[9] = '0' + interrupt_id % 10;
+	memcpy_s(SERVICE_SEND_BUFFER(), SPCI_MSG_PAYLOAD_MAX, buffer, size);
+	spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, size, 0);
+	dlog("secondary IRQ %d ended\n", interrupt_id);
+	event_send_local();
+}
+
+TEST_SERVICE(timer)
+{
+	exception_setup(irq_current, NULL);
+	hf_interrupt_enable(HF_VIRTUAL_TIMER_INTID, true);
+	arch_irq_enable();
+
+	for (;;) {
+		uint8_t *message = (uint8_t *)SERVICE_RECV_BUFFER();
+		bool wfi;
+		bool wfe;
+		bool receive;
+		bool disable_interrupts;
+		uint32_t ticks;
+		struct spci_value ret = mailbox_receive_retry();
+
+		if (spci_msg_send_sender(ret) != HF_PRIMARY_VM_ID ||
+		    spci_msg_send_size(ret) != sizeof("**** xxxxxxx")) {
+			FAIL("Got unexpected message from VM %d, size %d.\n",
+			     spci_msg_send_sender(ret),
+			     spci_msg_send_size(ret));
+		}
+
+		/*
+		 * Start a timer to send the message back: enable it and
+		 * set it for the requested number of ticks.
+		 */
+		wfi = memcmp(message, "WFI ", 4) == 0;
+		wfe = memcmp(message, "WFE ", 4) == 0;
+		receive = memcmp(message, "RECV", 4) == 0;
+		disable_interrupts = wfi || receive;
+		ticks = (message[5] - '0') * 1000000 +
+			(message[6] - '0') * 100000 +
+			(message[7] - '0') * 10000 + (message[8] - '0') * 1000 +
+			(message[9] - '0') * 100 + (message[10] - '0') * 10 +
+			(message[11] - '0');
+
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+		dlog("Starting timer for %d ticks.\n", ticks);
+
+		if (disable_interrupts) {
+			arch_irq_disable();
+		}
+
+		timer_set(ticks);
+		timer_start();
+		dlog("Waiting for timer...\n");
+
+		/* Wait for the timer interrupt. */
+		if (wfi) {
+			interrupt_wait();
+		} else if (wfe) {
+			while (!timer_fired) {
+				event_wait();
+			}
+		} else if (receive) {
+			struct spci_value res = spci_msg_wait();
+
+			EXPECT_SPCI_ERROR(res, SPCI_INTERRUPTED);
+		} else {
+			/* Busy wait until the timer fires. */
+			while (!timer_fired) {
+			}
+		}
+
+		if (disable_interrupts) {
+			arch_irq_enable();
+		}
+
+		EXPECT_TRUE(timer_fired);
+		timer_fired = false;
+		dlog("Done waiting.\n");
+	}
+}
diff --git a/test/vmapi/arch/aarch64/gicv3/timer_secondary.c b/test/vmapi/arch/aarch64/gicv3/timer_secondary.c
new file mode 100644
index 0000000..1e5c107
--- /dev/null
+++ b/test/vmapi/arch/aarch64/gicv3/timer_secondary.c
@@ -0,0 +1,286 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts_gicv3.h"
+
+#include "hf/abi.h"
+#include "hf/call.h"
+#include "hf/spci.h"
+
+#include "gicv3.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+SET_UP(timer_secondary)
+{
+	system_setup();
+
+	EXPECT_EQ(spci_rxtx_map(send_page_addr, recv_page_addr).func,
+		  SPCI_SUCCESS_32);
+	SERVICE_SELECT(SERVICE_VM1, "timer", send_buffer);
+
+	interrupt_enable(VIRTUAL_TIMER_IRQ, true);
+	interrupt_set_edge_triggered(VIRTUAL_TIMER_IRQ, true);
+	interrupt_set_priority_mask(0xff);
+	arch_irq_enable();
+}
+
+TEAR_DOWN(timer_secondary)
+{
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+}
+
+static void timer_busywait_secondary()
+{
+	const char message[] = "loop 0099999";
+	const char expected_response[] = "Got IRQ 03.";
+	struct spci_value run_res;
+
+	/* Let the secondary get started and wait for our message. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Send the message for the secondary to set a timer. */
+	memcpy_s(send_buffer, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+
+	/*
+	 * Let the secondary handle the message and set the timer. It will loop
+	 * until the hardware interrupt fires, at which point we'll get and
+	 * ignore the interrupt, and see a SPCI_YIELD return code.
+	 */
+	dlog("running secondary after sending timer message.\n");
+	last_interrupt_id = 0;
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_INTERRUPT_32);
+	dlog("secondary yielded after receiving timer message\n");
+	EXPECT_EQ(last_interrupt_id, VIRTUAL_TIMER_IRQ);
+
+	/*
+	 * Now that the timer has expired, when we call spci_run again Hafnium
+	 * should inject a virtual timer interrupt into the secondary, which
+	 * should get it and respond.
+	 */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(recv_buffer, expected_response,
+			 sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Send a message to the interruptible VM, which will start a timer to interrupt
+ * itself to send a response back.
+ */
+TEST(timer_secondary, busywait)
+{
+	/*
+	 * Run the test twice in a row, to check that the state doesn't get
+	 * messed up.
+	 */
+	timer_busywait_secondary();
+	timer_busywait_secondary();
+}
+
+static void timer_secondary(const char message[], uint64_t expected_code)
+{
+	const char expected_response[] = "Got IRQ 03.";
+	size_t message_length = strnlen_s(message, 64) + 1;
+	struct spci_value run_res;
+
+	/* Let the secondary get started and wait for our message. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Send the message for the secondary to set a timer. */
+	memcpy_s(send_buffer, SPCI_MSG_PAYLOAD_MAX, message, message_length);
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, message_length, 0)
+			.func,
+		SPCI_SUCCESS_32);
+
+	/* Let the secondary handle the message and set the timer. */
+	last_interrupt_id = 0;
+	run_res = spci_run(SERVICE_VM1, 0);
+
+	/*
+	 * There's a race for whether the secondary manages to block and switch
+	 * to the primary before the hardware timer fires, so we need to handle
+	 * three cases:
+	 * 1. The (hardware) timer fires immediately, we get SPCI_INTERRUPT.
+	 * 2. The secondary blocks and switches back, we get expected_code until
+	 *   the timer fires.
+	 *  2a. The timer then expires while we are in the primary, so Hafnium
+	 *   can inject the timer interrupt the next time we call spci_run.
+	 *  2b. The timer fires while the secondary is running, so we get
+	 *   SPCI_INTERRUPT as in case 1.
+	 */
+
+	if (run_res.func != expected_code &&
+	    run_res.func != SPCI_INTERRUPT_32) {
+		FAIL("Expected run to return SPCI_INTERRUPT or %#x, but "
+		     "got %#x",
+		     expected_code, run_res.func);
+	}
+
+	/* Loop until the timer fires. */
+	while (run_res.func == expected_code) {
+		/*
+		 * This case happens if the secondary manages to block and
+		 * switch to the primary before the timer fires.
+		 */
+		dlog("Primary looping until timer fires\n");
+		if (expected_code == HF_SPCI_RUN_WAIT_FOR_INTERRUPT ||
+		    expected_code == SPCI_MSG_WAIT_32) {
+			EXPECT_NE(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+			dlog("%d ns remaining\n", run_res.arg2);
+		}
+		run_res = spci_run(SERVICE_VM1, 0);
+	}
+	dlog("Primary done looping\n");
+
+	if (run_res.func == SPCI_INTERRUPT_32) {
+		/*
+		 * This case happens if the (hardware) timer fires before the
+		 * secondary blocks and switches to the primary, either
+		 * immediately after setting the timer or during the loop above.
+		 * Then we get the interrupt to the primary, ignore it, and see
+		 * a SPCI_INTERRUPT code from the spci_run call, so we should
+		 * call it again for the timer interrupt to be injected
+		 * automatically by Hafnium.
+		 */
+		EXPECT_EQ(last_interrupt_id, VIRTUAL_TIMER_IRQ);
+		dlog("Preempted by timer interrupt, running again\n");
+		run_res = spci_run(SERVICE_VM1, 0);
+	}
+
+	/* Once we wake it up it should get the timer interrupt and respond. */
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(recv_buffer, expected_response,
+			 sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Send a message to the interruptible VM, which will start a timer to interrupt
+ * itself to send a response back. This test is run with both long and short
+ * timer lengths, to try to cover both cases of the race for whether the timer
+ * fires before or after the secondary VM blocks and switches back to the
+ * primary.
+ */
+TEST(timer_secondary, wfi_short)
+{
+	/*
+	 * Run the test twice in a row, to check that the state doesn't get
+	 * messed up.
+	 */
+	timer_secondary("WFI  0000001", HF_SPCI_RUN_WAIT_FOR_INTERRUPT);
+	timer_secondary("WFI  0000001", HF_SPCI_RUN_WAIT_FOR_INTERRUPT);
+}
+
+TEST(timer_secondary, wfi_long)
+{
+	/*
+	 * Run the test twice in a row, to check that the state doesn't get
+	 * messed up.
+	 */
+	timer_secondary("WFI  0099999", HF_SPCI_RUN_WAIT_FOR_INTERRUPT);
+	timer_secondary("WFI  0099999", HF_SPCI_RUN_WAIT_FOR_INTERRUPT);
+}
+
+TEST(timer_secondary, wfe_short)
+{
+	/*
+	 * Run the test twice in a row, to check that the state doesn't get
+	 * messed up.
+	 */
+	timer_secondary("WFE  0000001", SPCI_YIELD_32);
+	timer_secondary("WFE  0000001", SPCI_YIELD_32);
+}
+
+TEST(timer_secondary, wfe_long)
+{
+	/*
+	 * Run the test twice in a row, to check that the state doesn't get
+	 * messed up.
+	 */
+	timer_secondary("WFE  0099999", SPCI_YIELD_32);
+	timer_secondary("WFE  0099999", SPCI_YIELD_32);
+}
+
+TEST(timer_secondary, receive_short)
+{
+	/*
+	 * Run the test twice in a row, to check that the state doesn't get
+	 * messed up.
+	 */
+	timer_secondary("RECV 0000001", SPCI_MSG_WAIT_32);
+	timer_secondary("RECV 0000001", SPCI_MSG_WAIT_32);
+}
+
+TEST(timer_secondary, receive_long)
+{
+	/*
+	 * Run the test twice in a row, to check that the state doesn't get
+	 * messed up.
+	 */
+	timer_secondary("RECV 0099999", SPCI_MSG_WAIT_32);
+	timer_secondary("RECV 0099999", SPCI_MSG_WAIT_32);
+}
+
+/**
+ * Set the timer for a very long time, and expect that it doesn't fire.
+ */
+TEST(timer_secondary, wfi_very_long)
+{
+	const char message[] = "WFI  9999999";
+	size_t message_length = strnlen_s(message, 64) + 1;
+	struct spci_value run_res;
+
+	/* Let the secondary get started and wait for our message. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Send the message for the secondary to set a timer. */
+	memcpy_s(send_buffer, SPCI_MSG_PAYLOAD_MAX, message, message_length);
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, message_length, 0)
+			.func,
+		SPCI_SUCCESS_32);
+
+	/*
+	 * Let the secondary handle the message and set the timer.
+	 */
+	last_interrupt_id = 0;
+	for (int i = 0; i < 20; ++i) {
+		run_res = spci_run(SERVICE_VM1, 0);
+		EXPECT_EQ(run_res.func, HF_SPCI_RUN_WAIT_FOR_INTERRUPT);
+		dlog("Primary looping until timer fires; %d ns "
+		     "remaining\n",
+		     run_res.arg2);
+	}
+}
diff --git a/test/vmapi/arch/aarch64/manifest.dts b/test/vmapi/arch/aarch64/manifest.dts
new file mode 100644
index 0000000..2d56429
--- /dev/null
+++ b/test/vmapi/arch/aarch64/manifest.dts
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+		vm1 {
+			debug_name = "aarch64_test";
+			kernel_filename = "aarch64_test";
+
+			smc_whitelist = <
+				0x30000001
+				0x30000002
+				0x30000003
+				0x30000004
+				>;
+		};
+	};
+};
diff --git a/test/vmapi/arch/aarch64/smc_whitelist.c b/test/vmapi/arch/aarch64/smc_whitelist.c
new file mode 100644
index 0000000..fe0c664
--- /dev/null
+++ b/test/vmapi/arch/aarch64/smc_whitelist.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "vmapi/hf/call.h"
+
+#include "smc.h"
+#include "test/hftest.h"
+
+TEST(smc_whitelist, not_whitelisted_unknown)
+{
+	const uint32_t non_whitelisted_ta_call = 0x3000f00d;
+	struct spci_value smc_res = smc_forward(
+		non_whitelisted_ta_call, 0x1111111111111111, 0x2222222222222222,
+		0x3333333333333333, 0x4444444444444444, 0x5555555555555555,
+		0x6666666666666666, 0x77777777);
+
+	EXPECT_EQ(smc_res.func, SMCCC_ERROR_UNKNOWN);
+	EXPECT_EQ(smc_res.arg1, UINT64_C(0x1111111111111111));
+	EXPECT_EQ(smc_res.arg2, UINT64_C(0x2222222222222222));
+	EXPECT_EQ(smc_res.arg3, UINT64_C(0x3333333333333333));
+	EXPECT_EQ(smc_res.arg4, UINT64_C(0x4444444444444444));
+	EXPECT_EQ(smc_res.arg5, UINT64_C(0x5555555555555555));
+	EXPECT_EQ(smc_res.arg6, UINT64_C(0x6666666666666666));
+	EXPECT_EQ(smc_res.arg7, UINT64_C(0x77777777));
+}
diff --git a/test/vmapi/arch/aarch64/smccc.c b/test/vmapi/arch/aarch64/smccc.c
new file mode 100644
index 0000000..d42cc16
--- /dev/null
+++ b/test/vmapi/arch/aarch64/smccc.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+
+#include "vmapi/hf/call.h"
+#include "vmapi/hf/spci.h"
+
+#include "smc.h"
+#include "test/hftest.h"
+
+static struct spci_value hvc(uint32_t func, uint64_t arg0, uint64_t arg1,
+			     uint64_t arg2, uint64_t arg3, uint64_t arg4,
+			     uint64_t arg5, uint32_t caller_id)
+{
+	register uint64_t r0 __asm__("x0") = func;
+	register uint64_t r1 __asm__("x1") = arg0;
+	register uint64_t r2 __asm__("x2") = arg1;
+	register uint64_t r3 __asm__("x3") = arg2;
+	register uint64_t r4 __asm__("x4") = arg3;
+	register uint64_t r5 __asm__("x5") = arg4;
+	register uint64_t r6 __asm__("x6") = arg5;
+	register uint64_t r7 __asm__("x7") = caller_id;
+
+	__asm__ volatile(
+		"hvc #0"
+		: /* Output registers, also used as inputs ('+' constraint). */
+		"+r"(r0), "+r"(r1), "+r"(r2), "+r"(r3), "+r"(r4), "+r"(r5),
+		"+r"(r6), "+r"(r7));
+
+	return (struct spci_value){.func = r0,
+				   .arg1 = r1,
+				   .arg2 = r2,
+				   .arg3 = r3,
+				   .arg4 = r4,
+				   .arg5 = r5,
+				   .arg6 = r6,
+				   .arg7 = r7};
+}
+
+TEST(smccc, hf_debug_log_smc_zero_or_unchanged)
+{
+	struct spci_value smc_res =
+		smc_forward(HF_DEBUG_LOG, '\n', 0x2222222222222222,
+			    0x3333333333333333, 0x4444444444444444,
+			    0x5555555555555555, 0x6666666666666666, 0x77777777);
+
+	EXPECT_EQ(smc_res.func, 0);
+	EXPECT_EQ(smc_res.arg1, '\n');
+	EXPECT_EQ(smc_res.arg2, UINT64_C(0x2222222222222222));
+	EXPECT_EQ(smc_res.arg3, UINT64_C(0x3333333333333333));
+	EXPECT_EQ(smc_res.arg4, UINT64_C(0x4444444444444444));
+	EXPECT_EQ(smc_res.arg5, UINT64_C(0x5555555555555555));
+	EXPECT_EQ(smc_res.arg6, UINT64_C(0x6666666666666666));
+	EXPECT_EQ(smc_res.arg7, UINT64_C(0x77777777));
+}
+
+TEST(smccc, hf_debug_log_hvc_zero_or_unchanged)
+{
+	struct spci_value smc_res =
+		hvc(HF_DEBUG_LOG, '\n', 0x2222222222222222, 0x3333333333333333,
+		    0x4444444444444444, 0x5555555555555555, 0x6666666666666666,
+		    0x77777777);
+
+	EXPECT_EQ(smc_res.func, 0);
+	EXPECT_EQ(smc_res.arg1, '\n');
+	EXPECT_EQ(smc_res.arg2, UINT64_C(0x2222222222222222));
+	EXPECT_EQ(smc_res.arg3, UINT64_C(0x3333333333333333));
+	EXPECT_EQ(smc_res.arg4, UINT64_C(0x4444444444444444));
+	EXPECT_EQ(smc_res.arg5, UINT64_C(0x5555555555555555));
+	EXPECT_EQ(smc_res.arg6, UINT64_C(0x6666666666666666));
+	EXPECT_EQ(smc_res.arg7, UINT64_C(0x77777777));
+}
+
+/**
+ * Checks that calling SPCI_FEATURES via an SMC works as expected.
+ * The spci_features helper function uses an HVC, but an SMC should also work.
+ */
+TEST(smccc, spci_features_smc)
+{
+	struct spci_value ret;
+
+	ret = smc32(SPCI_FEATURES_32, SPCI_VERSION_32, 0, 0, 0, 0, 0, 0);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+	EXPECT_EQ(ret.arg1, 0);
+	EXPECT_EQ(ret.arg2, 0);
+	EXPECT_EQ(ret.arg3, 0);
+	EXPECT_EQ(ret.arg4, 0);
+	EXPECT_EQ(ret.arg5, 0);
+	EXPECT_EQ(ret.arg6, 0);
+	EXPECT_EQ(ret.arg7, 0);
+}
diff --git a/test/vmapi/common/BUILD.gn b/test/vmapi/common/BUILD.gn
new file mode 100644
index 0000000..48be642
--- /dev/null
+++ b/test/vmapi/common/BUILD.gn
@@ -0,0 +1,24 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+
+source_set("common") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+  sources = [
+    "exception_handler.c",
+    "spci.c",
+  ]
+}
diff --git a/test/vmapi/common/exception_handler.c b/test/vmapi/common/exception_handler.c
new file mode 100644
index 0000000..c58a5b8
--- /dev/null
+++ b/test/vmapi/common/exception_handler.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/dlog.h"
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "test/hftest.h"
+
+/**
+ * Tracks the number of times the exception handler has been invoked.
+ */
+static int exception_handler_exception_count = 0;
+
+/**
+ * Sends the number of exceptions handled to the Primary VM.
+ */
+void exception_handler_send_exception_count(void)
+{
+	void *send_buf = SERVICE_SEND_BUFFER();
+
+	dlog("Sending exception_count %d to primary VM\n",
+	     exception_handler_exception_count);
+	memcpy_s(send_buf, SPCI_MSG_PAYLOAD_MAX,
+		 (const void *)&exception_handler_exception_count,
+		 sizeof(exception_handler_exception_count));
+	EXPECT_EQ(spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID,
+				sizeof(exception_handler_exception_count), 0)
+			  .func,
+		  SPCI_SUCCESS_32);
+}
+
+/**
+ * Receives the number of exceptions handled.
+ */
+int exception_handler_receive_exception_count(
+	const struct spci_value *send_res,
+	const struct spci_memory_region *recv_buf)
+{
+	int exception_count = *((const int *)recv_buf);
+
+	EXPECT_EQ(send_res->func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(*send_res), sizeof(exception_count));
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	return exception_count;
+}
+
+/**
+ * EL1 exception handler to use in unit test VMs.
+ * Skips the instruction that triggered the exception.
+ */
+bool exception_handler_skip_instruction(void)
+{
+	dlog("%s function is triggered!\n", __func__);
+	++exception_handler_exception_count;
+
+	/* Skip instruction that triggered the exception. */
+	uint64_t next_pc = read_msr(elr_el1);
+	next_pc += 4UL;
+	write_msr(elr_el1, next_pc);
+
+	/* Indicate that elr_el1 should not be restored. */
+	return true;
+}
+
+/**
+ * EL1 exception handler to use in unit test VMs.
+ * Yields control back to the hypervisor and sends the number of exceptions.
+ */
+bool exception_handler_yield(void)
+{
+	dlog("%s function is triggered!\n", __func__);
+	++exception_handler_exception_count;
+
+	exception_handler_send_exception_count();
+
+	/* Indicate that elr_el1 should not be restored. */
+	return true;
+}
+
+/**
+ * Returns the number of times the instruction handler was invoked.
+ */
+int exception_handler_get_num(void)
+{
+	return exception_handler_exception_count;
+}
+
+/**
+ * Resets the number of exceptions counter;
+ */
+void exception_handler_reset(void)
+{
+	exception_handler_exception_count = 0;
+}
diff --git a/test/vmapi/common/spci.c b/test/vmapi/common/spci.c
new file mode 100644
index 0000000..a175cea
--- /dev/null
+++ b/test/vmapi/common/spci.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/spci.h"
+
+#include "hf/mm.h"
+#include "hf/static_assert.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+static alignas(PAGE_SIZE) uint8_t send_page[PAGE_SIZE];
+static alignas(PAGE_SIZE) uint8_t recv_page[PAGE_SIZE];
+static_assert(sizeof(send_page) == PAGE_SIZE, "Send page is not a page.");
+static_assert(sizeof(recv_page) == PAGE_SIZE, "Recv page is not a page.");
+
+static hf_ipaddr_t send_page_addr = (hf_ipaddr_t)send_page;
+static hf_ipaddr_t recv_page_addr = (hf_ipaddr_t)recv_page;
+
+struct mailbox_buffers set_up_mailbox(void)
+{
+	ASSERT_EQ(spci_rxtx_map(send_page_addr, recv_page_addr).func,
+		  SPCI_SUCCESS_32);
+	return (struct mailbox_buffers){
+		.send = send_page,
+		.recv = recv_page,
+	};
+}
diff --git a/test/vmapi/primary_only/BUILD.gn b/test/vmapi/primary_only/BUILD.gn
new file mode 100644
index 0000000..cdb35b9
--- /dev/null
+++ b/test/vmapi/primary_only/BUILD.gn
@@ -0,0 +1,36 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+
+# Tests with no secondary VMs.
+vm_kernel("primary_only_test_vm") {
+  testonly = true
+
+  sources = [
+    "faults.c",
+    "primary_only.c",
+  ]
+
+  deps = [
+    "//test/hftest:hftest_primary_vm",
+  ]
+}
+
+initrd("primary_only_test") {
+  testonly = true
+  manifest = "manifest.dts"
+  primary_name = "primary_only_test"
+  primary_vm = ":primary_only_test_vm"
+}
diff --git a/test/vmapi/primary_only/faults.c b/test/vmapi/primary_only/faults.c
new file mode 100644
index 0000000..34b7e15
--- /dev/null
+++ b/test/vmapi/primary_only/faults.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include "hf/mm.h"
+#include "hf/spinlock.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+
+alignas(PAGE_SIZE) static char tx[PAGE_SIZE];
+alignas(PAGE_SIZE) static char rx[PAGE_SIZE];
+
+struct state {
+	volatile bool done;
+	struct spinlock lock;
+};
+
+/**
+ * Releases the lock passed in, then spins reading the rx buffer.
+ */
+static void rx_reader(uintptr_t arg)
+{
+	struct state *s = (struct state *)arg;
+	sl_unlock(&s->lock);
+
+	while (!s->done) {
+		*(volatile char *)(&rx[0]);
+	}
+
+	sl_unlock(&s->lock);
+}
+
+/**
+ * Forces a spurious fault and check that Hafnium recovers from it.
+ */
+TEST(faults, spurious_due_to_configure)
+{
+	struct state s;
+	alignas(4096) static uint8_t other_stack[4096];
+
+	sl_init(&s.lock);
+	s.done = false;
+
+	/* Start secondary CPU while holding lock. */
+	sl_lock(&s.lock);
+	EXPECT_EQ(
+		hftest_cpu_start(hftest_get_cpu_id(1), other_stack,
+				 sizeof(other_stack), rx_reader, (uintptr_t)&s),
+		true);
+
+	/* Wait for CPU to release the lock. */
+	sl_lock(&s.lock);
+
+	/* Configure the VM's buffers. */
+	EXPECT_EQ(spci_rxtx_map((hf_ipaddr_t)&tx[0], (hf_ipaddr_t)&rx[0]).func,
+		  SPCI_SUCCESS_32);
+
+	/* Tell other CPU to stop and wait for it. */
+	s.done = true;
+	sl_lock(&s.lock);
+}
diff --git a/test/vmapi/primary_only/manifest.dts b/test/vmapi/primary_only/manifest.dts
new file mode 100644
index 0000000..f83e057
--- /dev/null
+++ b/test/vmapi/primary_only/manifest.dts
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+		vm1 {
+			debug_name = "primary_only_test";
+			kernel_filename = "primary_only_test";
+		};
+	};
+};
diff --git a/test/vmapi/primary_only/primary_only.c b/test/vmapi/primary_only/primary_only.c
new file mode 100644
index 0000000..ab26f1d
--- /dev/null
+++ b/test/vmapi/primary_only/primary_only.c
@@ -0,0 +1,257 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include "hf/spinlock.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+/*
+ * TODO: Some of these tests are duplicated between 'primary_only' and
+ * 'primary_with_secondaries'. Move them to a common place consider running
+ * them inside secondary VMs too.
+ */
+
+/**
+ * Confirms the primary VM has the primary ID.
+ */
+TEST(hf_vm_get_id, primary_has_primary_id)
+{
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+}
+
+/**
+ * Confirm there is only the primary VM.
+ */
+TEST(hf_vm_get_count, no_secondary_vms)
+{
+	EXPECT_EQ(hf_vm_get_count(), 1);
+}
+
+/**
+ * Confirm the primary has at least one vCPU.
+ */
+TEST(hf_vcpu_get_count, primary_has_at_least_one)
+{
+	EXPECT_GE(hf_vcpu_get_count(HF_PRIMARY_VM_ID), 0);
+}
+
+/**
+ * Confirm an error is returned when getting the vCPU count of a non-existent
+ * VM.
+ */
+TEST(hf_vcpu_get_count, no_secondary_vms)
+{
+	EXPECT_EQ(hf_vcpu_get_count(HF_VM_ID_OFFSET + 1), 0);
+}
+
+/**
+ * Confirm an error is returned when getting the vCPU count for a reserved ID.
+ */
+TEST(hf_vcpu_get_count, reserved_vm_id)
+{
+	spci_vm_id_t id;
+
+	for (id = 0; id < HF_VM_ID_OFFSET; ++id) {
+		EXPECT_EQ(hf_vcpu_get_count(id), 0);
+	}
+}
+
+/**
+ * Confirm an error is returned when getting the vCPU count of a VM with an ID
+ * that is likely to be far outside the resource limit.
+ */
+TEST(hf_vcpu_get_count, large_invalid_vm_id)
+{
+	EXPECT_EQ(hf_vcpu_get_count(0xffff), 0);
+}
+
+/**
+ * Confirm it is an error when running a vCPU from the primary VM.
+ */
+TEST(spci_run, cannot_run_primary)
+{
+	struct spci_value res = spci_run(HF_PRIMARY_VM_ID, 0);
+	EXPECT_SPCI_ERROR(res, SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Confirm it is an error when running a vCPU from a non-existent secondary VM.
+ */
+TEST(spci_run, cannot_run_absent_secondary)
+{
+	struct spci_value res = spci_run(1, 0);
+	EXPECT_SPCI_ERROR(res, SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Yielding from the primary is a noop.
+ */
+TEST(spci_yield, yield_is_noop_for_primary)
+{
+	EXPECT_EQ(spci_yield().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Releases the lock passed in.
+ */
+static void vm_cpu_entry(uintptr_t arg)
+{
+	struct spinlock *lock = (struct spinlock *)arg;
+
+	dlog("Second CPU started.\n");
+	sl_unlock(lock);
+}
+
+/**
+ * Confirm a new CPU can be started to execute in parallel.
+ */
+TEST(cpus, start)
+{
+	struct spinlock lock = SPINLOCK_INIT;
+	alignas(4096) static uint8_t other_stack[4096];
+
+	/* Start secondary while holding lock. */
+	sl_lock(&lock);
+	EXPECT_EQ(hftest_cpu_start(hftest_get_cpu_id(1), other_stack,
+				   sizeof(other_stack), vm_cpu_entry,
+				   (uintptr_t)&lock),
+		  true);
+
+	/* Wait for CPU to release the lock. */
+	sl_lock(&lock);
+}
+
+/** Ensures that the Hafnium SPCI version is reported as expected. */
+TEST(spci, spci_version)
+{
+	const int32_t major_revision = 0;
+	const int32_t major_revision_offset = 16;
+	const int32_t minor_revision = 9;
+	const int32_t current_version =
+		(major_revision << major_revision_offset) | minor_revision;
+
+	struct spci_value ret = spci_version();
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+	EXPECT_EQ(ret.arg2, current_version);
+}
+
+/** Ensures that SPCI_FEATURES is reporting the expected interfaces. */
+TEST(spci, spci_features)
+{
+	struct spci_value ret;
+
+	ret = spci_features(SPCI_ERROR_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_SUCCESS_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_VERSION_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_FEATURES_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_ID_GET_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_YIELD_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_MSG_SEND_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_MSG_POLL_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_MSG_WAIT_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+
+	ret = spci_features(SPCI_YIELD_32);
+	EXPECT_EQ(ret.func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Ensures that SPCI_FEATURES returns not supported for a bogus FID or
+ * currently non-implemented interfaces.
+ */
+TEST(spci, spci_features_not_supported)
+{
+	struct spci_value ret;
+
+	ret = spci_features(0);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(0x84000000);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_INTERRUPT_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_RX_RELEASE_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_RXTX_MAP_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_RXTX_UNMAP_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_PARTITION_INFO_GET_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_RUN_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_MSG_SEND_DIRECT_RESP_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_MSG_SEND_DIRECT_REQ_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_MSG_SEND_DIRECT_REQ_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+
+	ret = spci_features(SPCI_MSG_SEND_DIRECT_RESP_32);
+	EXPECT_SPCI_ERROR(ret, SPCI_NOT_SUPPORTED);
+}
+
+/**
+ * Test that floating-point operations work in the primary VM.
+ */
+TEST(fp, fp)
+{
+	/*
+	 * Get some numbers that the compiler can't tell are constants, so it
+	 * can't optimise them away.
+	 */
+	double a = hf_vm_get_count();
+	double b = hf_vcpu_get_count(HF_PRIMARY_VM_ID);
+	double result = a * b;
+	dlog("VM count: %d\n", hf_vm_get_count());
+	dlog("vCPU count: %d\n", hf_vcpu_get_count(HF_PRIMARY_VM_ID));
+	dlog("result: %d\n", (int)result);
+	EXPECT_TRUE(a == 1.0);
+	EXPECT_TRUE(b == 8.0);
+	EXPECT_TRUE(result == 8.0);
+}
diff --git a/test/vmapi/primary_with_secondaries/BUILD.gn b/test/vmapi/primary_with_secondaries/BUILD.gn
new file mode 100644
index 0000000..ee3ab92
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/BUILD.gn
@@ -0,0 +1,74 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+
+config("config") {
+  include_dirs = [ "inc" ]
+}
+
+# Tests with secondary VMs.
+vm_kernel("primary_with_secondaries_test_vm") {
+  testonly = true
+  public_configs = [
+    ":config",
+    "//src/arch/aarch64:config",
+  ]
+
+  sources = [
+    "boot.c",
+    "debug_el1.c",
+    "floating_point.c",
+    "interrupts.c",
+    "mailbox.c",
+    "memory_sharing.c",
+    "no_services.c",
+    "perfmon.c",
+    "run_race.c",
+    "smp.c",
+    "spci.c",
+    "sysregs.c",
+    "unmapped.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64/hftest:registers",
+    "//test/hftest:hftest_primary_vm",
+    "//test/vmapi/common",
+    "//vmlib",
+  ]
+}
+
+initrd("primary_with_secondaries_test") {
+  testonly = true
+
+  manifest = "manifest.dts"
+
+  primary_name = "primary_with_secondaries_test"
+  primary_vm = ":primary_with_secondaries_test_vm"
+  secondary_vms = [
+    [
+      "services1",
+      "services:service_vm1",
+    ],
+    [
+      "services2",
+      "services:service_vm2",
+    ],
+    [
+      "services3",
+      "services:service_vm3",
+    ],
+  ]
+}
diff --git a/test/vmapi/primary_with_secondaries/boot.c b/test/vmapi/primary_with_secondaries/boot.c
new file mode 100644
index 0000000..e098538
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/boot.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/dlog.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/exception_handler.h"
+#include "test/vmapi/spci.h"
+
+/**
+ * The VM gets its memory size on boot, and can access it all.
+ */
+TEST(boot, memory_size)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "boot_memory", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Accessing memory outside the given range traps the VM and yields.
+ */
+TEST(boot, beyond_memory_size)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "boot_memory_overrun", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Accessing memory before the start of the image traps the VM and yields.
+ */
+TEST(boot, memory_before_image)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "boot_memory_underrun", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
diff --git a/test/vmapi/primary_with_secondaries/debug_el1.c b/test/vmapi/primary_with_secondaries/debug_el1.c
new file mode 100644
index 0000000..d7d3df1
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/debug_el1.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "primary_with_secondary.h"
+#include "sysregs.h"
+#include "test/vmapi/spci.h"
+
+/**
+ * QEMU does not properly handle the trapping of certain system register
+ * accesses. This was fixed in a custom local build that we could use. If not
+ * using that build, limit testing to the subset QEMU handles correctly.
+ */
+#define CUSTOM_QEMU_BUILD() 0
+
+TEST(debug_el1, secondary_basic)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "debug_el1_secondary_basic", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Attempts to access debug registers for read, without validating their value.
+ */
+TEST(debug_el1, primary_basic)
+{
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+
+	if (CUSTOM_QEMU_BUILD()) {
+		TRY_READ(DBGAUTHSTATUS_EL1);
+		TRY_READ(DBGCLAIMCLR_EL1);
+		TRY_READ(DBGCLAIMSET_EL1);
+		TRY_READ(DBGPRCR_EL1);
+		TRY_READ(OSDTRRX_EL1);
+		TRY_READ(OSDTRTX_EL1);
+		TRY_READ(OSECCR_EL1);
+
+		TRY_READ(DBGBCR2_EL1);
+		TRY_READ(DBGBCR3_EL1);
+		TRY_READ(DBGBCR4_EL1);
+		TRY_READ(DBGBCR5_EL1);
+		TRY_READ(DBGBVR2_EL1);
+		TRY_READ(DBGBVR3_EL1);
+		TRY_READ(DBGBVR4_EL1);
+		TRY_READ(DBGBVR5_EL1);
+		TRY_READ(DBGWCR2_EL1);
+		TRY_READ(DBGWCR3_EL1);
+		TRY_READ(DBGWVR2_EL1);
+		TRY_READ(DBGWVR3_EL1);
+	}
+
+	/* The following is the subset currently supported by QEMU. */
+	TRY_READ(MDCCINT_EL1);
+	TRY_READ(MDRAR_EL1);
+	TRY_READ(MDSCR_EL1);
+	TRY_READ(OSDLR_EL1);
+	TRY_READ(OSLSR_EL1);
+
+	TRY_READ(DBGBCR0_EL1);
+	TRY_READ(DBGBCR1_EL1);
+	TRY_READ(DBGBVR0_EL1);
+	TRY_READ(DBGBVR1_EL1);
+	TRY_READ(DBGWCR0_EL1);
+	TRY_READ(DBGWCR1_EL1);
+	TRY_READ(DBGWVR0_EL1);
+	TRY_READ(DBGWVR1_EL1);
+}
+
+/**
+ * Tests a few debug registers for read and write, and checks that the expected
+ * value is written/read.
+ */
+TEST(debug_el1, primary_read_write)
+{
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+
+	TRY_WRITE_READ(DBGBCR0_EL1, 0x2);
+	TRY_WRITE_READ(DBGBVR0_EL1, 0xf0);
+}
diff --git a/test/vmapi/primary_with_secondaries/floating_point.c b/test/vmapi/primary_with_secondaries/floating_point.c
new file mode 100644
index 0000000..1904d68
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/floating_point.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/std.h"
+#include "hf/arch/vm/registers.h"
+
+#include "hf/spci.h"
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+/**
+ * Test that floating point registers are saved and restored by
+ * filling them with one value here and a different value in the
+ * service.
+ */
+TEST(floating_point, fp_fill)
+{
+	const double first = 1.2;
+	const double second = -2.3;
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	fill_fp_registers(first);
+	SERVICE_SELECT(SERVICE_VM1, "fp_fill", mb.send);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+	EXPECT_EQ(check_fp_register(first), true);
+
+	fill_fp_registers(second);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+	EXPECT_EQ(check_fp_register(second), true);
+}
+
+/**
+ * Test that the floating point control register is restored correctly
+ * on full context switch when needed by changing it in the service.
+ */
+TEST(floating_point, fp_fpcr)
+{
+	uintreg_t value = 0;
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	EXPECT_EQ(read_msr(fpcr), value);
+
+	SERVICE_SELECT(SERVICE_VM1, "fp_fpcr", mb.send);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+	EXPECT_EQ(read_msr(fpcr), value);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+	EXPECT_EQ(read_msr(fpcr), value);
+}
diff --git a/test/vmapi/primary_with_secondaries/inc/primary_with_secondary.h b/test/vmapi/primary_with_secondaries/inc/primary_with_secondary.h
new file mode 100644
index 0000000..ad8dcc8
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/inc/primary_with_secondary.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#define SERVICE_VM1 (HF_VM_ID_OFFSET + 1)
+#define SERVICE_VM2 (HF_VM_ID_OFFSET + 2)
+#define SERVICE_VM3 (HF_VM_ID_OFFSET + 3)
+
+#define SELF_INTERRUPT_ID 5
+#define EXTERNAL_INTERRUPT_ID_A 7
+#define EXTERNAL_INTERRUPT_ID_B 8
+#define EXTERNAL_INTERRUPT_ID_C 9
diff --git a/test/vmapi/primary_with_secondaries/interrupts.c b/test/vmapi/primary_with_secondaries/interrupts.c
new file mode 100644
index 0000000..dce89ab
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/interrupts.c
@@ -0,0 +1,301 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+TEAR_DOWN(interrupts)
+{
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+}
+
+/**
+ * Send a message to the interruptible VM, which will interrupt itself to send a
+ * response back.
+ */
+TEST(interrupts, interrupt_self)
+{
+	const char message[] = "Ping";
+	const char expected_response[] = "Got IRQ 05.";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "interruptible", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Set the message, echo it and wait for a response. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Inject an interrupt to the interrupt VM, which will send a message back.
+ * Repeat this twice to make sure it doesn't get into a bad state after the
+ * first one.
+ */
+TEST(interrupts, inject_interrupt_twice)
+{
+	const char expected_response[] = "Got IRQ 07.";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "interruptible", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Inject the interrupt and wait for a message. */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_A);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Inject the interrupt again, and wait for the same message. */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_A);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Inject two different interrupts to the interrupt VM, which will send a
+ * message back each time.
+ */
+TEST(interrupts, inject_two_interrupts)
+{
+	const char expected_response[] = "Got IRQ 07.";
+	const char expected_response_2[] = "Got IRQ 08.";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "interruptible", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Inject the interrupt and wait for a message. */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_A);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Inject a different interrupt and wait for a different message. */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_B);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response_2));
+	EXPECT_EQ(memcmp(mb.recv, expected_response_2,
+			 sizeof(expected_response_2)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Inject an interrupt then send a message to the interrupt VM, which will send
+ * a message back each time. This is to test that interrupt injection doesn't
+ * interfere with message reception.
+ */
+TEST(interrupts, inject_interrupt_message)
+{
+	const char expected_response[] = "Got IRQ 07.";
+	const char message[] = "Ping";
+	const char expected_response_2[] = "Got IRQ 05.";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "interruptible", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Inject the interrupt and wait for a message. */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_A);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Now send a message to the secondary. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response_2));
+	EXPECT_EQ(memcmp(mb.recv, expected_response_2,
+			 sizeof(expected_response_2)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Inject an interrupt which the target VM has not enabled, and then send a
+ * message telling it to enable that interrupt ID. It should then (and only
+ * then) send a message back.
+ */
+TEST(interrupts, inject_interrupt_disabled)
+{
+	const char expected_response[] = "Got IRQ 09.";
+	const char message[] = "Enable interrupt C";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "interruptible", mb.send);
+
+	/* Inject the interrupt and expect not to get a message. */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_C);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/*
+	 * Now send a message to the secondary to enable the interrupt ID, and
+	 * expect the response from the interrupt we sent before.
+	 */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * If a secondary VM has an enabled and pending interrupt, even if interrupts
+ * are disabled globally via PSTATE, then hf_mailbox_receive should not block
+ * even if `block` is true.
+ */
+TEST(interrupts, pending_interrupt_no_blocking_receive)
+{
+	const char expected_response[] = "Done waiting";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "receive_block", mb.send);
+
+	/*
+	 * Inject the interrupt and run the VM. It should disable interrupts
+	 * globally, enable the specific interrupt, and then send us a message
+	 * back after failing to receive a message a few times.
+	 */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_A);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * If a secondary VM has an enabled and pending interrupt, even if interrupts
+ * are disabled globally via PSTATE, then WFI should be treated as a no-op and
+ * not return to the primary.
+ */
+TEST(interrupts, pending_interrupt_wfi_not_trapped)
+{
+	const char expected_response[] = "Done waiting";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "wfi", mb.send);
+
+	/*
+	 * Inject the interrupt and run the VM. It should disable interrupts
+	 * globally, enable the specific interrupt, and then send us a message
+	 * back after running WFI a few times.
+	 */
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_A);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response));
+	EXPECT_EQ(memcmp(mb.recv, expected_response, sizeof(expected_response)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/*
+ * Deliver an interrupt and a message to the same vCPU and check that both are
+ * delivered the next time the vCPU is run.
+ */
+TEST(interrupts, deliver_interrupt_and_message)
+{
+	const char message[] = "I\'ll see you again.";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "interruptible_echo", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	hf_interrupt_inject(SERVICE_VM1, 0, EXTERNAL_INTERRUPT_ID_A);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(message));
+	EXPECT_EQ(memcmp(mb.recv, message, sizeof(message)), 0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
diff --git a/test/vmapi/primary_with_secondaries/mailbox.c b/test/vmapi/primary_with_secondaries/mailbox.c
new file mode 100644
index 0000000..b96c595
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/mailbox.c
@@ -0,0 +1,365 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+/**
+ * Reverses the order of the elements in the given array.
+ */
+static void reverse(char *s, size_t len)
+{
+	size_t i;
+
+	for (i = 0; i < len / 2; i++) {
+		char t = s[i];
+		s[i] = s[len - 1 - i];
+		s[len - 1 - i] = t;
+	}
+}
+
+/**
+ * Finds the next lexicographic permutation of the given array, if there is one.
+ */
+static void next_permutation(char *s, size_t len)
+{
+	size_t i;
+	size_t j;
+
+	for (i = len - 2; i < len; i--) {
+		const char t = s[i];
+		if (t >= s[i + 1]) {
+			continue;
+		}
+
+		for (j = len - 1; t >= s[j]; j--) {
+		}
+
+		s[i] = s[j];
+		s[j] = t;
+		reverse(s + i + 1, len - i - 1);
+		return;
+	}
+}
+
+TEAR_DOWN(mailbox)
+{
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+}
+
+/**
+ * Clearing an empty mailbox is an error.
+ */
+TEST(mailbox, clear_empty)
+{
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+}
+
+/**
+ * Send and receive the same message from the echo VM.
+ */
+TEST(mailbox, echo)
+{
+	const char message[] = "Echo this back to me!";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "echo", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Set the message, echo it and check it didn't change. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(message));
+	EXPECT_EQ(memcmp(mb.recv, message, sizeof(message)), 0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Repeatedly send a message and receive it back from the echo VM.
+ */
+TEST(mailbox, repeated_echo)
+{
+	char message[] = "Echo this back to me!";
+	struct spci_value run_res;
+	uint8_t i;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "echo", mb.send);
+
+	for (i = 0; i < 100; i++) {
+		/* Run secondary until it reaches the wait for messages. */
+		run_res = spci_run(SERVICE_VM1, 0);
+		EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+		EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+		/* Set the message, echo it and check it didn't change. */
+		next_permutation(message, sizeof(message) - 1);
+		memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message,
+			 sizeof(message));
+		EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1,
+					sizeof(message), 0)
+				  .func,
+			  SPCI_SUCCESS_32);
+		run_res = spci_run(SERVICE_VM1, 0);
+		EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+		EXPECT_EQ(spci_msg_send_size(run_res), sizeof(message));
+		EXPECT_EQ(memcmp(mb.recv, message, sizeof(message)), 0);
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	}
+}
+
+/**
+ * Send a message to relay_a which will forward it to relay_b where it will be
+ * sent back here.
+ */
+TEST(mailbox, relay)
+{
+	const char message[] = "Send this round the relay!";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "relay", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "relay", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+	run_res = spci_run(SERVICE_VM2, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/*
+	 * Build the message chain so the message is sent from here to
+	 * SERVICE_VM1, then to SERVICE_VM2 and finally back to here.
+	 */
+	{
+		spci_vm_id_t *chain = (spci_vm_id_t *)mb.send;
+		*chain++ = htole32(SERVICE_VM2);
+		*chain++ = htole32(HF_PRIMARY_VM_ID);
+		memcpy_s(chain,
+			 SPCI_MSG_PAYLOAD_MAX - (2 * sizeof(spci_vm_id_t)),
+			 message, sizeof(message));
+
+		EXPECT_EQ(
+			spci_msg_send(
+				HF_PRIMARY_VM_ID, SERVICE_VM1,
+				sizeof(message) + (2 * sizeof(spci_vm_id_t)), 0)
+				.func,
+			SPCI_SUCCESS_32);
+	}
+
+	/* Let SERVICE_VM1 forward the message. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_receiver(run_res), SERVICE_VM2);
+	EXPECT_EQ(spci_msg_send_size(run_res), 0);
+
+	/* Let SERVICE_VM2 forward the message. */
+	run_res = spci_run(SERVICE_VM2, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+
+	/* Ensure the message is intact. */
+	EXPECT_EQ(spci_msg_send_receiver(run_res), HF_PRIMARY_VM_ID);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(message));
+	EXPECT_EQ(memcmp(mb.recv, message, sizeof(message)), 0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Send a message before the secondary VM is configured, but do not register
+ * for notification. Ensure we're not notified.
+ */
+TEST(mailbox, no_primary_to_secondary_notification_on_configure)
+{
+	struct spci_value run_res;
+
+	set_up_mailbox();
+
+	EXPECT_SPCI_ERROR(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, 0, 0),
+			  SPCI_BUSY);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, 0, 0).func,
+		  SPCI_SUCCESS_32);
+}
+
+/**
+ * Send a message before the secondary VM is configured, and receive a
+ * notification when it configures.
+ */
+TEST(mailbox, secondary_to_primary_notification_on_configure)
+{
+	struct spci_value run_res;
+
+	set_up_mailbox();
+
+	EXPECT_SPCI_ERROR(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, 0,
+					SPCI_MSG_SEND_NOTIFY),
+			  SPCI_BUSY);
+
+	/*
+	 * Run first VM for it to configure itself. It should result in
+	 * notifications having to be issued.
+	 */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_RX_RELEASE_32);
+
+	/* A single waiter is returned. */
+	EXPECT_EQ(hf_mailbox_waiter_get(SERVICE_VM1), HF_PRIMARY_VM_ID);
+	EXPECT_EQ(hf_mailbox_waiter_get(SERVICE_VM1), -1);
+
+	/* Send should now succeed. */
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, 0, 0).func,
+		  SPCI_SUCCESS_32);
+}
+
+/**
+ * Causes secondary VM to send two messages to primary VM. The second message
+ * will reach the mailbox while it's not writable. Checks that notifications are
+ * properly delivered when mailbox is cleared.
+ */
+TEST(mailbox, primary_to_secondary)
+{
+	char message[] = "not ready echo";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "echo_with_notification", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Send a message to echo service, and get response back. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(message));
+	EXPECT_EQ(memcmp(mb.recv, message, sizeof(message)), 0);
+
+	/* Let secondary VM continue running so that it will wait again. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Without clearing our mailbox, send message again. */
+	reverse(message, strnlen_s(message, sizeof(message)));
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+
+	/* Message should be dropped since the mailbox was not cleared. */
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, HF_SPCI_RUN_WAIT_FOR_INTERRUPT);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Clear the mailbox. We expect to be told there are pending waiters. */
+	EXPECT_EQ(spci_rx_release().func, SPCI_RX_RELEASE_32);
+
+	/* Retrieve a single waiter. */
+	EXPECT_EQ(hf_mailbox_waiter_get(HF_PRIMARY_VM_ID), SERVICE_VM1);
+	EXPECT_EQ(hf_mailbox_waiter_get(HF_PRIMARY_VM_ID), -1);
+
+	/*
+	 * Inject interrupt into VM and let it run again. We should receive
+	 * the echoed message.
+	 */
+	EXPECT_EQ(
+		hf_interrupt_inject(SERVICE_VM1, 0, HF_MAILBOX_WRITABLE_INTID),
+		1);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(message));
+	EXPECT_EQ(memcmp(mb.recv, message, sizeof(message)), 0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * Sends two messages to secondary VM without letting it run, so second message
+ * won't go through. Ensure that a notification is delivered when secondary VM
+ * clears the mailbox.
+ */
+TEST(mailbox, secondary_to_primary_notification)
+{
+	const char message[] = "not ready echo";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "echo_with_notification", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+
+	/* Send a message to echo service twice. The second should fail. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+	EXPECT_SPCI_ERROR(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1,
+					sizeof(message), SPCI_MSG_SEND_NOTIFY),
+			  SPCI_BUSY);
+
+	/* Receive a reply for the first message. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(message));
+	EXPECT_EQ(memcmp(mb.recv, message, sizeof(message)), 0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Run VM again so that it clears its mailbox. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_RX_RELEASE_32);
+
+	/* Retrieve a single waiter. */
+	EXPECT_EQ(hf_mailbox_waiter_get(SERVICE_VM1), HF_PRIMARY_VM_ID);
+	EXPECT_EQ(hf_mailbox_waiter_get(SERVICE_VM1), -1);
+
+	/* Send should now succeed. */
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+}
diff --git a/test/vmapi/primary_with_secondaries/manifest.dts b/test/vmapi/primary_with_secondaries/manifest.dts
new file mode 100644
index 0000000..3d9a73a
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/manifest.dts
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	hypervisor {
+		compatible = "hafnium,hafnium";
+		vm1 {
+			debug_name = "primary_with_secondaries_test";
+			kernel_filename = "primary_with_secondaries_test";
+		};
+
+		vm2 {
+			debug_name = "services1";
+			vcpu_count = <1>;
+			mem_size = <0x100000>;
+			kernel_filename = "services1";
+		};
+
+		vm3 {
+			debug_name = "services2";
+			vcpu_count = <1>;
+			mem_size = <0x100000>;
+			kernel_filename = "services2";
+		};
+
+		vm4 {
+			debug_name = "services3";
+			vcpu_count = <2>;
+			mem_size = <0x100000>;
+			kernel_filename = "services3";
+		};
+	};
+};
diff --git a/test/vmapi/primary_with_secondaries/memory_sharing.c b/test/vmapi/primary_with_secondaries/memory_sharing.c
new file mode 100644
index 0000000..fd742f0
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/memory_sharing.c
@@ -0,0 +1,1938 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+
+#include "hf/mm.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/exception_handler.h"
+#include "test/vmapi/spci.h"
+
+alignas(PAGE_SIZE) static uint8_t pages[4 * PAGE_SIZE];
+
+/**
+ * Helper function to test sending memory in the different configurations.
+ */
+static void check_cannot_send_memory(
+	struct mailbox_buffers mb, uint32_t mode,
+	struct spci_memory_region_constituent constituents[],
+	int constituent_count, int32_t avoid_vm)
+
+{
+	enum spci_memory_access access[] = {SPCI_MEMORY_RO_NX, SPCI_MEMORY_RO_X,
+					    SPCI_MEMORY_RW_NX,
+					    SPCI_MEMORY_RW_X};
+	enum spci_memory_cacheability cacheability[] = {
+		SPCI_MEMORY_CACHE_NON_CACHEABLE,
+		SPCI_MEMORY_CACHE_WRITE_THROUGH, SPCI_MEMORY_CACHE_WRITE_BACK};
+	enum spci_memory_cacheability device[] = {
+		SPCI_MEMORY_DEV_NGNRNE, SPCI_MEMORY_DEV_NGNRE,
+		SPCI_MEMORY_DEV_NGRE, SPCI_MEMORY_DEV_GRE};
+	enum spci_memory_shareability shareability[] = {
+		SPCI_MEMORY_SHARE_NON_SHAREABLE, SPCI_MEMORY_RESERVED,
+		SPCI_MEMORY_OUTER_SHAREABLE, SPCI_MEMORY_INNER_SHAREABLE};
+	uint32_t vms[] = {HF_PRIMARY_VM_ID, SERVICE_VM1, SERVICE_VM2};
+
+	size_t i = 0;
+	size_t j = 0;
+	size_t k = 0;
+	size_t l = 0;
+
+	for (i = 0; i < ARRAY_SIZE(vms); ++i) {
+		/* Optionally skip one VM as the send would succeed. */
+		if (vms[i] == avoid_vm) {
+			continue;
+		}
+		for (j = 0; j < ARRAY_SIZE(access); ++j) {
+			for (k = 0; k < ARRAY_SIZE(shareability); ++k) {
+				for (l = 0; l < ARRAY_SIZE(cacheability); ++l) {
+					uint32_t msg_size =
+						spci_memory_region_init(
+							mb.send,
+							HF_PRIMARY_VM_ID,
+							vms[i], constituents,
+							constituent_count, 0, 0,
+							access[j],
+							SPCI_MEMORY_NORMAL_MEM,
+							cacheability[l],
+							shareability[k]);
+					EXPECT_SPCI_ERROR(
+						spci_msg_send(HF_PRIMARY_VM_ID,
+							      vms[i], msg_size,
+							      mode),
+						SPCI_INVALID_PARAMETERS);
+				}
+				for (l = 0; l < ARRAY_SIZE(device); ++l) {
+					uint32_t msg_size =
+						spci_memory_region_init(
+							mb.send,
+							HF_PRIMARY_VM_ID,
+							vms[i], constituents,
+							constituent_count, 0, 0,
+							access[j],
+							SPCI_MEMORY_DEVICE_MEM,
+							device[l],
+							shareability[k]);
+					EXPECT_SPCI_ERROR(
+						spci_msg_send(HF_PRIMARY_VM_ID,
+							      vms[i], msg_size,
+							      mode),
+						SPCI_INVALID_PARAMETERS);
+				}
+			}
+		}
+	}
+}
+
+/**
+ * Helper function to test lending memory in the different configurations.
+ */
+static void check_cannot_lend_memory(
+	struct mailbox_buffers mb,
+	struct spci_memory_region_constituent constituents[],
+	int constituent_count, int32_t avoid_vm)
+
+{
+	check_cannot_send_memory(mb, SPCI_MSG_SEND_LEGACY_MEMORY_LEND,
+				 constituents, constituent_count, avoid_vm);
+}
+
+/**
+ * Helper function to test sharing memory in the different configurations.
+ */
+static void check_cannot_share_memory(
+	struct mailbox_buffers mb,
+	struct spci_memory_region_constituent constituents[],
+	int constituent_count, int32_t avoid_vm)
+
+{
+	check_cannot_send_memory(mb, SPCI_MSG_SEND_LEGACY_MEMORY_SHARE,
+				 constituents, constituent_count, avoid_vm);
+}
+
+/**
+ * Tries donating memory in available modes with different VMs and asserts that
+ * it will fail to all except the supplied VM ID as this would succeed if it
+ * is the only borrower.
+ */
+static void check_cannot_donate_memory(
+	struct mailbox_buffers mb,
+	struct spci_memory_region_constituent constituents[],
+	int constituent_count, int32_t avoid_vm)
+{
+	uint32_t vms[] = {HF_PRIMARY_VM_ID, SERVICE_VM1, SERVICE_VM2};
+
+	size_t i;
+	for (i = 0; i < ARRAY_SIZE(vms); ++i) {
+		uint32_t msg_size;
+		/* Optionally skip one VM as the donate would succeed. */
+		if (vms[i] == avoid_vm) {
+			continue;
+		}
+		msg_size = spci_memory_region_init(
+			mb.send, HF_PRIMARY_VM_ID, vms[i], constituents,
+			constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(HF_PRIMARY_VM_ID, vms[i], msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+			SPCI_INVALID_PARAMETERS);
+	}
+}
+
+/**
+ * Tries relinquishing memory with different VMs and asserts that
+ * it will fail.
+ */
+static void check_cannot_relinquish_memory(
+	struct mailbox_buffers mb,
+	struct spci_memory_region_constituent constituents[],
+	int constituent_count)
+{
+	uint32_t vms[] = {HF_PRIMARY_VM_ID, SERVICE_VM1, SERVICE_VM2};
+
+	size_t i;
+	size_t j;
+	for (i = 0; i < ARRAY_SIZE(vms); ++i) {
+		for (j = 0; j < ARRAY_SIZE(vms); ++j) {
+			uint32_t msg_size = spci_memory_region_init(
+				mb.send, vms[j], vms[i], constituents,
+				constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+				SPCI_MEMORY_NORMAL_MEM,
+				SPCI_MEMORY_CACHE_WRITE_BACK,
+				SPCI_MEMORY_OUTER_SHAREABLE);
+			EXPECT_SPCI_ERROR(
+				spci_msg_send(
+					vms[j], vms[i], msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH),
+				SPCI_INVALID_PARAMETERS);
+		}
+	}
+}
+
+TEAR_DOWN(memory_sharing)
+{
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+}
+
+/**
+ * Sharing memory concurrently gives both VMs access to the memory so it can be
+ * used for communication.
+ */
+TEST(memory_sharing, concurrent)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	SERVICE_SELECT(SERVICE_VM1, "memory_increment", mb.send);
+
+	memset_s(ptr, sizeof(pages), 'a', PAGE_SIZE);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, SPCI_MEMORY_REGION_FLAG_CLEAR,
+		SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+		SPCI_MEMORY_CACHE_WRITE_BACK, SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		pages[i] = i;
+	}
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		uint8_t value = i + 1;
+
+		EXPECT_EQ(pages[i], value);
+	}
+}
+
+/**
+ * Memory shared concurrently can be returned to the owner.
+ */
+TEST(memory_sharing, share_concurrently_and_get_back)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish", mb.send);
+
+	/* Dirty the memory before sharing it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be returned. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'c');
+	}
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Device address space cannot be shared, only normal memory.
+ */
+TEST(memory_sharing, cannot_share_device_memory)
+{
+	struct mailbox_buffers mb = set_up_mailbox();
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = PAGE_SIZE, .page_count = 1},
+	};
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_return", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_return", mb.send);
+
+	check_cannot_lend_memory(mb, constituents, ARRAY_SIZE(constituents),
+				 -1);
+	check_cannot_share_memory(mb, constituents, ARRAY_SIZE(constituents),
+				  -1);
+	check_cannot_donate_memory(mb, constituents, ARRAY_SIZE(constituents),
+				   -1);
+}
+
+/**
+ * Check that memory can be lent and is accessible by both parties.
+ */
+TEST(memory_sharing, lend_relinquish)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+		{.address = (uint64_t)pages + PAGE_SIZE, .page_count = 2},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+
+	/* Let the memory be returned. */
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Ensure that the secondary VM accessed the region. */
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'c');
+	}
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Check that memory that is donated can't be relinquished.
+ */
+TEST(memory_sharing, donate_relinquish)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_donate_relinquish", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+		{.address = (uint64_t)pages + PAGE_SIZE, .page_count = 2},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/*
+	 * Let the service access the memory, and try and fail to relinquish it.
+	 */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Memory given away can be given back.
+ */
+TEST(memory_sharing, give_and_get_back)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_return", mb.send);
+
+	/* Dirty the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be returned. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'c');
+	}
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Memory that has been lent can be returned to the owner.
+ */
+TEST(memory_sharing, lend_and_get_back)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish", mb.send);
+
+	/* Dirty the memory before lending it. */
+	memset_s(ptr, sizeof(pages), 'c', PAGE_SIZE);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be returned. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'd');
+	}
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * After memory has been returned, it is free to be shared again.
+ */
+TEST(memory_sharing, reshare_after_return)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint32_t msg_size;
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish", mb.send);
+
+	/* Share the memory initially. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be returned. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Share the memory again after it has been returned. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Observe the service doesn't fault when accessing the memory. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+}
+
+/**
+ * After memory has been returned, it is free to be shared with another VM.
+ */
+TEST(memory_sharing, share_elsewhere_after_return)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint32_t msg_size;
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_lend_relinquish", mb.send);
+
+	/* Share the memory initially. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be returned. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Share the memory with a different VM after it has been returned. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * After memory has been given, it is no longer accessible by the sharing VM.
+ */
+TEST(memory_sharing, give_memory_and_lose_access)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	struct spci_memory_region *memory_region;
+	struct spci_memory_region_constituent *constituents;
+	uint8_t *ptr;
+
+	SERVICE_SELECT(SERVICE_VM1, "give_memory_and_fault", mb.send);
+
+	/* Have the memory be given. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(run_res),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+
+	/* Check the memory was cleared. */
+	memory_region = (struct spci_memory_region *)mb.recv;
+	constituents = spci_memory_region_get_constituents(memory_region);
+	ptr = (uint8_t *)constituents[0].address;
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 0);
+	}
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * After memory has been lent, it is no longer accessible by the sharing VM.
+ */
+TEST(memory_sharing, lend_memory_and_lose_access)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	struct spci_memory_region *memory_region;
+	struct spci_memory_region_constituent *constituents;
+	uint8_t *ptr;
+
+	SERVICE_SELECT(SERVICE_VM1, "lend_memory_and_fault", mb.send);
+
+	/* Have the memory be lent. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(run_res),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_LEND);
+
+	/* Check the memory was cleared. */
+	memory_region = (struct spci_memory_region *)mb.recv;
+	constituents = spci_memory_region_get_constituents(memory_region);
+	ptr = (uint8_t *)constituents[0].address;
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 0);
+	}
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Verify past the upper bound of the donated region cannot be accessed.
+ */
+TEST(memory_sharing, donate_check_upper_bounds)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_donate_check_upper_bound", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', 4 * PAGE_SIZE);
+
+	/* Specify non-contiguous memory regions. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+		{.address = (uint64_t)pages + PAGE_SIZE * 2, .page_count = 1},
+	};
+
+	/*
+	 * Specify that we want to test the first constituent of the donated
+	 * memory region. This is utilised by the test service.
+	 */
+	pages[0] = 0;
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+
+	/* Use different memory regions for verifying the second constituent. */
+	constituents[0].address = (uint64_t)pages + PAGE_SIZE * 1;
+	constituents[1].address = (uint64_t)pages + PAGE_SIZE * 3;
+
+	/*
+	 * Specify that we now want to test the second constituent of the
+	 * donated memory region.
+	 */
+	pages[PAGE_SIZE] = 1;
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Verify past the lower bound of the donated region cannot be accessed.
+ */
+TEST(memory_sharing, donate_check_lower_bounds)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_donate_check_lower_bound", mb.send);
+
+	/* Initialise the memory before donating it. */
+	memset_s(ptr, sizeof(pages), 'b', 4 * PAGE_SIZE);
+
+	/* Specify non-contiguous memory regions. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+		{.address = (uint64_t)pages + PAGE_SIZE * 2, .page_count = 1},
+	};
+
+	/*
+	 * Specify that we want to test the first constituent of the donated
+	 * memory region. This is utilised by the test service.
+	 */
+	pages[0] = 0;
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+
+	/* Use different memory regions for verifying the second constituent. */
+	constituents[0].address = (uint64_t)pages + PAGE_SIZE * 1;
+	constituents[1].address = (uint64_t)pages + PAGE_SIZE * 3;
+
+	/*
+	 * Specify that we now want to test the second constituent of the
+	 * donated memory region.
+	 */
+	pages[PAGE_SIZE] = 1;
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	/*
+	 * NOTE: This generates two exceptions, one for the page fault, and one
+	 * for accessing a region past the lower bound.
+	 */
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  2);
+}
+
+/**
+ * After memory has been returned, it is free to be shared with another
+ * VM.
+ */
+TEST(memory_sharing, donate_elsewhere_after_return)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_return", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_return", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', 1 * PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+
+	/* Let the memory be returned. */
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Share the memory with another VM. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Check if memory can be donated between secondary VMs.
+ * Ensure that the memory can no longer be accessed by the first VM.
+ */
+TEST(memory_sharing, donate_vms)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_donate_secondary_and_fault", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_receive", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', 1 * PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	/* Set up VM2 to wait for message. */
+	run_res = spci_run(SERVICE_VM2, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_WAIT_32);
+
+	/* Donate memory. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be sent from VM1 to VM2. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_receiver(run_res), SERVICE_VM2);
+
+	/* Receive memory in VM2. */
+	run_res = spci_run(SERVICE_VM2, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Try to access memory in VM1. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+
+	/* Ensure that memory in VM2 remains the same. */
+	run_res = spci_run(SERVICE_VM2, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Check that memory is unable to be donated to multiple parties.
+ */
+TEST(memory_sharing, donate_twice)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_donate_twice", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_receive", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', 1 * PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	/* Donate memory to VM1. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be received. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Fail to share memory again with any VM. */
+	check_cannot_share_memory(mb, constituents, ARRAY_SIZE(constituents),
+				  -1);
+	check_cannot_lend_memory(mb, constituents, ARRAY_SIZE(constituents),
+				 -1);
+	check_cannot_donate_memory(mb, constituents, ARRAY_SIZE(constituents),
+				   -1);
+	/* Fail to relinquish memory from any VM. */
+	check_cannot_relinquish_memory(mb, constituents,
+				       ARRAY_SIZE(constituents));
+
+	/* Let the memory be sent from VM1 to PRIMARY (returned). */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Check we have access again. */
+	ptr[0] = 'f';
+
+	/* Try and fail to donate memory from VM1 to VM2. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Check cannot donate to self.
+ */
+TEST(memory_sharing, donate_to_self)
+{
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(HF_PRIMARY_VM_ID, HF_PRIMARY_VM_ID, msg_size,
+			      SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+		SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Check cannot lend to self.
+ */
+TEST(memory_sharing, lend_to_self)
+{
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(HF_PRIMARY_VM_ID, HF_PRIMARY_VM_ID, msg_size,
+			      SPCI_MSG_SEND_LEGACY_MEMORY_LEND),
+		SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Check cannot share to self.
+ */
+TEST(memory_sharing, share_to_self)
+{
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(HF_PRIMARY_VM_ID, HF_PRIMARY_VM_ID, msg_size,
+			      SPCI_MSG_SEND_LEGACY_MEMORY_SHARE),
+		SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Check cannot donate from alternative VM.
+ */
+TEST(memory_sharing, donate_invalid_source)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_donate_invalid_source", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_receive", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	/* Try invalid configurations. */
+	msg_size = spci_memory_region_init(
+		mb.send, SERVICE_VM1, HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(spci_msg_send(SERVICE_VM1, HF_PRIMARY_VM_ID, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+			  SPCI_INVALID_PARAMETERS);
+
+	msg_size = spci_memory_region_init(
+		mb.send, SERVICE_VM1, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(spci_msg_send(SERVICE_VM1, SERVICE_VM1, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+			  SPCI_INVALID_PARAMETERS);
+
+	msg_size = spci_memory_region_init(
+		mb.send, SERVICE_VM2, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(spci_msg_send(SERVICE_VM2, SERVICE_VM1, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+			  SPCI_INVALID_PARAMETERS);
+
+	/* Successfully donate to VM1. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Receive and return memory from VM1. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Use VM1 to fail to donate memory from the primary to VM2. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Check that unaligned addresses can not be shared.
+ */
+TEST(memory_sharing, give_and_get_back_unaligned)
+{
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_return", mb.send);
+
+	/* Check for unaligned pages for either constituent. */
+	for (int i = 0; i < PAGE_SIZE; i++) {
+		for (int j = 0; i < PAGE_SIZE; i++) {
+			/* Skip the case they're both aligned. */
+			if (i == 0 && j == 0) {
+				continue;
+			}
+			struct spci_memory_region_constituent constituents[] = {
+				{.address = (uint64_t)pages + i,
+				 .page_count = 1},
+				{.address = (uint64_t)pages + PAGE_SIZE + j,
+				 .page_count = 1},
+			};
+			uint32_t msg_size = spci_memory_region_init(
+				mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1,
+				constituents, ARRAY_SIZE(constituents), 0, 0,
+				SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+				SPCI_MEMORY_CACHE_WRITE_BACK,
+				SPCI_MEMORY_OUTER_SHAREABLE);
+			EXPECT_SPCI_ERROR(
+				spci_msg_send(
+					HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+				SPCI_INVALID_PARAMETERS);
+			msg_size = spci_memory_region_init(
+				mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1,
+				constituents, ARRAY_SIZE(constituents), 0, 0,
+				SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+				SPCI_MEMORY_CACHE_WRITE_BACK,
+				SPCI_MEMORY_OUTER_SHAREABLE);
+			EXPECT_SPCI_ERROR(
+				spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1,
+					      msg_size,
+					      SPCI_MSG_SEND_LEGACY_MEMORY_LEND),
+				SPCI_INVALID_PARAMETERS);
+		}
+	}
+}
+
+/**
+ * Check cannot lend from alternative VM.
+ */
+TEST(memory_sharing, lend_invalid_source)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_lend_invalid_source", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	/* Check cannot swap VM IDs. */
+	msg_size = spci_memory_region_init(
+		mb.send, SERVICE_VM1, HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(spci_msg_send(SERVICE_VM1, HF_PRIMARY_VM_ID, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_LEND),
+			  SPCI_INVALID_PARAMETERS);
+
+	/* Lend memory to VM1. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Receive and return memory from VM1. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Try to lend memory from primary in VM1. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Memory can be lent with executable permissions.
+ * Check RO and RW permissions.
+ */
+TEST(memory_sharing, lend_relinquish_X_RW)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish_RW", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Let service write to and return memory. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Re-initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Memory can be shared with executable permissions.
+ * Check RO and RW permissions.
+ */
+TEST(memory_sharing, share_relinquish_X_RW)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish_RW", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Ensure we still have access. */
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'b');
+		ptr[i]++;
+	}
+
+	/* Let service write to and return memory. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Re-initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Ensure we still have access. */
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'b');
+		ptr[i]++;
+	}
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Memory can be shared without executable permissions.
+ * Check RO and RW permissions.
+ */
+TEST(memory_sharing, share_relinquish_NX_RW)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish_RW", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_NX,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Ensure we still have access. */
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'b');
+	}
+
+	/* Let service write to and return memory. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Re-initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_NX,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Ensure we still have access. */
+	for (int i = 0; i < PAGE_SIZE; ++i) {
+		ASSERT_EQ(ptr[i], 'b');
+		ptr[i]++;
+	}
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Exercise execution permissions for lending memory.
+ */
+TEST(memory_sharing, lend_relinquish_RW_X)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish_X", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 0, PAGE_SIZE);
+
+	uint64_t *ptr2 = (uint64_t *)pages;
+	/* Set memory to contain the RET instruction to attempt to execute. */
+	*ptr2 = 0xD65F03C0;
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Attempt to execute from memory. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_NX,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Exercise execution permissions for lending memory without write access.
+ */
+TEST(memory_sharing, lend_relinquish_RO_X)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish_X", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 0, PAGE_SIZE);
+
+	uint64_t *ptr2 = (uint64_t *)pages;
+	/* Set memory to contain the RET instruction to attempt to execute. */
+	*ptr2 = 0xD65F03C0;
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Attempt to execute from memory. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_NX,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Memory can be lent, but then no part can be donated.
+ */
+TEST(memory_sharing, lend_donate)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish_RW", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_lend_relinquish_RW", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages) * 2, 'b', PAGE_SIZE * 2);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 2},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Ensure we can't donate any sub section of memory to another VM. */
+	constituents[0].page_count = 1;
+	for (int i = 1; i < PAGE_SIZE * 2; i++) {
+		constituents[0].address = (uint64_t)pages + PAGE_SIZE;
+		msg_size = spci_memory_region_init(
+			mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+			ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+			SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Ensure we can donate to the only borrower. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+}
+
+/**
+ * Memory can be shared, but then no part can be donated.
+ */
+TEST(memory_sharing, share_donate)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_relinquish_RW", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_lend_relinquish_RW", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE * 4);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 2},
+		{.address = (uint64_t)pages + PAGE_SIZE * 2, .page_count = 2},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Attempt to share the same area of memory. */
+	check_cannot_share_memory(mb, constituents, ARRAY_SIZE(constituents),
+				  SERVICE_VM1);
+
+	/* Ensure we can't donate any sub section of memory to another VM. */
+	constituents[0].page_count = 1;
+	for (int i = 1; i < PAGE_SIZE * 2; i++) {
+		constituents[0].address = (uint64_t)pages + PAGE_SIZE;
+		msg_size = spci_memory_region_init(
+			mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+			ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+			SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Ensure we can donate to the only borrower. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+}
+
+/**
+ * Memory can be lent, but then no part can be lent again.
+ */
+TEST(memory_sharing, lend_twice)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_twice", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_lend_twice", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages), 'b', PAGE_SIZE * 4);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 2},
+		{.address = (uint64_t)pages + PAGE_SIZE * 3, .page_count = 1},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/* Attempt to lend the same area of memory. */
+	check_cannot_lend_memory(mb, constituents, ARRAY_SIZE(constituents),
+				 -1);
+	/* Attempt to share the same area of memory. */
+	check_cannot_share_memory(mb, constituents, ARRAY_SIZE(constituents),
+				  -1);
+	/* Fail to donate to VM apart from VM1. */
+	check_cannot_donate_memory(mb, constituents, ARRAY_SIZE(constituents),
+				   SERVICE_VM1);
+	/* Fail to relinquish from any VM. */
+	check_cannot_relinquish_memory(mb, constituents,
+				       ARRAY_SIZE(constituents));
+
+	/* Now attempt to share only a portion of the same area of memory. */
+	struct spci_memory_region_constituent constituents_subsection[] = {
+		{.address = (uint64_t)pages + PAGE_SIZE * 3, .page_count = 1},
+	};
+	check_cannot_lend_memory(mb, constituents_subsection,
+				 ARRAY_SIZE(constituents_subsection), -1);
+	check_cannot_donate_memory(mb, constituents_subsection,
+				   ARRAY_SIZE(constituents_subsection),
+				   SERVICE_VM1);
+	check_cannot_relinquish_memory(mb, constituents_subsection,
+				       ARRAY_SIZE(constituents_subsection));
+
+	/* Attempt to lend again with different permissions. */
+	constituents[0].page_count = 1;
+	for (int i = 0; i < 2; i++) {
+		constituents[0].address = (uint64_t)pages + i * PAGE_SIZE;
+		msg_size = spci_memory_region_init(
+			mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+			ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_LEND),
+			SPCI_INVALID_PARAMETERS);
+	}
+}
+
+/**
+ * Memory can be shared, but then no part can be shared again.
+ */
+TEST(memory_sharing, share_twice)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_lend_twice", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_memory_lend_twice", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages) * 2, 'b', PAGE_SIZE * 2);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 2},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Let the memory be accessed. */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+
+	/*
+	 * Attempting to share or lend the same area of memory with any VM
+	 * should fail.
+	 */
+	check_cannot_share_memory(mb, constituents, ARRAY_SIZE(constituents),
+				  -1);
+	check_cannot_lend_memory(mb, constituents, ARRAY_SIZE(constituents),
+				 -1);
+	/* Fail to donate to VM apart from VM1. */
+	check_cannot_donate_memory(mb, constituents, ARRAY_SIZE(constituents),
+				   SERVICE_VM1);
+	/* Fail to relinquish from any VM. */
+	check_cannot_relinquish_memory(mb, constituents,
+				       ARRAY_SIZE(constituents));
+
+	/* Attempt to share again with different permissions. */
+	constituents[0].page_count = 1;
+	for (int i = 0; i < 2; i++) {
+		constituents[0].address = (uint64_t)pages + i * PAGE_SIZE;
+		msg_size = spci_memory_region_init(
+			mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+			ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RO_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_SHARE),
+			SPCI_INVALID_PARAMETERS);
+	}
+}
+
+/**
+ * Memory can be cleared while being shared.
+ */
+TEST(memory_sharing, share_clear)
+{
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+	size_t i;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_memory_return", mb.send);
+
+	/* Initialise the memory before giving it. */
+	memset_s(ptr, sizeof(pages) * 2, 'b', PAGE_SIZE * 2);
+
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 2},
+	};
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, SPCI_MEMORY_REGION_FLAG_CLEAR,
+		SPCI_MEMORY_RO_X, SPCI_MEMORY_NORMAL_MEM,
+		SPCI_MEMORY_CACHE_WRITE_BACK, SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_SHARE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Check that it has been cleared. */
+	for (i = 0; i < PAGE_SIZE * 2; ++i) {
+		ASSERT_EQ(ptr[i], 0);
+	};
+}
+
+/**
+ * SPCI: Verify past the upper bound of the lent region cannot be accessed.
+ */
+TEST(memory_sharing, spci_lend_check_upper_bounds)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_lend_check_upper_bound", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_lend_check_upper_bound", mb.send);
+
+	/* Initialise the memory before lending it. */
+	memset_s(ptr, sizeof(pages), 'b', 4 * PAGE_SIZE);
+
+	/* Specify non-contiguous memory regions. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+		{.address = (uint64_t)pages + PAGE_SIZE * 2, .page_count = 1},
+	};
+
+	/*
+	 * Specify that we want to test the first constituent of the donated
+	 * memory region. This is utilised by the test service.
+	 */
+	pages[0] = 0;
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+
+	/* Use different memory regions for verifying the second constituent. */
+	constituents[0].address = (uint64_t)pages + PAGE_SIZE * 1;
+	constituents[1].address = (uint64_t)pages + PAGE_SIZE * 3;
+
+	/*
+	 * Specify that we now want to test the second constituent of the
+	 * lent memory region.
+	 */
+	pages[PAGE_SIZE] = 1;
+
+	/* Use the secondary VM for this test as the first is now aborted. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM2, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * SPCI: Verify past the lower bound of the lent region cannot be accessed.
+ */
+TEST(memory_sharing, spci_lend_check_lower_bounds)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+	uint8_t *ptr = pages;
+	uint32_t msg_size;
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_lend_check_lower_bound", mb.send);
+	SERVICE_SELECT(SERVICE_VM2, "spci_lend_check_lower_bound", mb.send);
+
+	/* Initialise the memory before lending it. */
+	memset_s(ptr, sizeof(pages), 'b', 4 * PAGE_SIZE);
+
+	/* Specify non-contiguous memory regions. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)pages, .page_count = 1},
+		{.address = (uint64_t)pages + PAGE_SIZE * 2, .page_count = 1},
+	};
+
+	/*
+	 * Specify that we want to test the first constituent of the lent
+	 * memory region. This is utilised by the test service.
+	 */
+	pages[0] = 0;
+
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM1, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+
+	/* Use different memory regions for verifying the second constituent. */
+	constituents[0].address = (uint64_t)pages + PAGE_SIZE * 1;
+	constituents[1].address = (uint64_t)pages + PAGE_SIZE * 3;
+
+	/*
+	 * Specify that we now want to test the second constituent of the
+	 * lent memory region.
+	 */
+	pages[PAGE_SIZE] = 1;
+
+	/* Use the secondary VM for this test as the first is now aborted. */
+	msg_size = spci_memory_region_init(
+		mb.send, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM2, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
diff --git a/test/vmapi/primary_with_secondaries/no_services.c b/test/vmapi/primary_with_secondaries/no_services.c
new file mode 100644
index 0000000..2757df2
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/no_services.c
@@ -0,0 +1,186 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/mm.h"
+#include "hf/static_assert.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+static alignas(PAGE_SIZE) uint8_t send_page[PAGE_SIZE];
+static alignas(PAGE_SIZE) uint8_t recv_page[PAGE_SIZE];
+static_assert(sizeof(send_page) == PAGE_SIZE, "Send page is not a page.");
+static_assert(sizeof(recv_page) == PAGE_SIZE, "Recv page is not a page.");
+
+static hf_ipaddr_t send_page_addr = (hf_ipaddr_t)send_page;
+static hf_ipaddr_t recv_page_addr = (hf_ipaddr_t)recv_page;
+
+/**
+ * Confirms the primary VM has the primary ID.
+ */
+TEST(hf_vm_get_id, primary_has_primary_id)
+{
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+}
+
+/**
+ * Confirm there are 3 secondary VMs as well as this primary VM.
+ */
+TEST(hf_vm_get_count, three_secondary_vms)
+{
+	EXPECT_EQ(hf_vm_get_count(), 4);
+}
+
+/**
+ * Confirm that secondary VM has 1 vCPU.
+ */
+TEST(hf_vcpu_get_count, secondary_has_one_vcpu)
+{
+	EXPECT_EQ(hf_vcpu_get_count(SERVICE_VM1), 1);
+}
+
+/**
+ * Confirm an error is returned when getting the vCPU count for a reserved ID.
+ */
+TEST(hf_vcpu_get_count, reserved_vm_id)
+{
+	spci_vm_id_t id;
+
+	for (id = 0; id < HF_VM_ID_OFFSET; ++id) {
+		EXPECT_EQ(hf_vcpu_get_count(id), 0);
+	}
+}
+
+/**
+ * Confirm it is an error to query how many vCPUs are assigned to a nonexistent
+ * secondary VM.
+ */
+TEST(hf_vcpu_get_count, large_invalid_vm_id)
+{
+	EXPECT_EQ(hf_vcpu_get_count(0xffff), 0);
+}
+
+/**
+ * The primary can't be run by the hypervisor.
+ */
+TEST(spci_run, cannot_run_primary)
+{
+	struct spci_value res = spci_run(HF_PRIMARY_VM_ID, 0);
+	EXPECT_SPCI_ERROR(res, SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Can only run a VM that exists.
+ */
+TEST(spci_run, cannot_run_absent_secondary)
+{
+	struct spci_value res = spci_run(1234, 0);
+	EXPECT_SPCI_ERROR(res, SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Can only run a vCPU that exists.
+ */
+TEST(spci_run, cannot_run_absent_vcpu)
+{
+	struct spci_value res = spci_run(SERVICE_VM1, 1234);
+	EXPECT_SPCI_ERROR(res, SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * The configured send/receive addresses can't be device memory.
+ */
+TEST(spci_rxtx_map, fails_with_device_memory)
+{
+	EXPECT_SPCI_ERROR(spci_rxtx_map(PAGE_SIZE, PAGE_SIZE * 2), SPCI_DENIED);
+}
+
+/**
+ * The configured send/receive addresses can't be unaligned.
+ */
+TEST(spci_rxtx_map, fails_with_unaligned_pointer)
+{
+	uint8_t maybe_aligned[2];
+	hf_ipaddr_t unaligned_addr = (hf_ipaddr_t)&maybe_aligned[1];
+	hf_ipaddr_t aligned_addr = (hf_ipaddr_t)send_page;
+
+	/* Check that the address is unaligned. */
+	ASSERT_EQ(unaligned_addr & 1, 1);
+
+	EXPECT_SPCI_ERROR(spci_rxtx_map(aligned_addr, unaligned_addr),
+			  SPCI_INVALID_PARAMETERS);
+	EXPECT_SPCI_ERROR(spci_rxtx_map(unaligned_addr, aligned_addr),
+			  SPCI_INVALID_PARAMETERS);
+	EXPECT_SPCI_ERROR(spci_rxtx_map(unaligned_addr, unaligned_addr),
+			  SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * The configured send/receive addresses can't be the same page.
+ */
+TEST(spci_rxtx_map, fails_with_same_page)
+{
+	EXPECT_SPCI_ERROR(spci_rxtx_map(send_page_addr, send_page_addr),
+			  SPCI_INVALID_PARAMETERS);
+	EXPECT_SPCI_ERROR(spci_rxtx_map(recv_page_addr, recv_page_addr),
+			  SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * The configuration of the send/receive addresses can only happen once.
+ */
+TEST(spci_rxtx_map, fails_if_already_succeeded)
+{
+	EXPECT_EQ(spci_rxtx_map(send_page_addr, recv_page_addr).func,
+		  SPCI_SUCCESS_32);
+	EXPECT_SPCI_ERROR(spci_rxtx_map(send_page_addr, recv_page_addr),
+			  SPCI_DENIED);
+}
+
+/**
+ * The configuration of the send/receive address is successful with valid
+ * arguments.
+ */
+TEST(spci_rxtx_map, succeeds)
+{
+	EXPECT_EQ(spci_rxtx_map(send_page_addr, recv_page_addr).func,
+		  SPCI_SUCCESS_32);
+}
+
+/**
+ * The primary receives messages from spci_run().
+ */
+TEST(hf_mailbox_receive, cannot_receive_from_primary_blocking)
+{
+	struct spci_value res = spci_msg_wait();
+	EXPECT_NE(res.func, SPCI_SUCCESS_32);
+}
+
+/**
+ * The primary receives messages from spci_run().
+ */
+TEST(hf_mailbox_receive, cannot_receive_from_primary_non_blocking)
+{
+	struct spci_value res = spci_msg_poll();
+	EXPECT_NE(res.func, SPCI_SUCCESS_32);
+}
diff --git a/test/vmapi/primary_with_secondaries/perfmon.c b/test/vmapi/primary_with_secondaries/perfmon.c
new file mode 100644
index 0000000..ad43de9
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/perfmon.c
@@ -0,0 +1,230 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "../../src/arch/aarch64/hypervisor/perfmon.h"
+
+#include "../../src/arch/aarch64/hypervisor/sysregs.h"
+#include "primary_with_secondary.h"
+#include "sysregs.h"
+#include "test/vmapi/spci.h"
+
+TEST(perfmon, secondary_basic)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "perfmon_secondary_basic", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Attempts to access performance monitor registers for read, without validating
+ * their value.
+ */
+TEST(perfmon, primary_basic)
+{
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+
+	TRY_READ(PMCEID0_EL0);
+	TRY_READ(PMCEID1_EL0);
+	TRY_READ(PMCCFILTR_EL0);
+	TRY_READ(PMCR_EL0);
+}
+
+/**
+ * Tests a few performance counter registers for read and write, and checks that
+ * the expected value is written/read.
+ */
+TEST(perfmon, primary_read_write)
+{
+	uintreg_t pmcr_el0 = read_msr(PMCR_EL0);
+	uintreg_t perf_mon_count = GET_PMCR_EL0_N(pmcr_el0);
+
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+
+	/*
+	 * Ensure that there are enough performance counters in the underlying
+	 * uArch for this test to pass.
+	 */
+	EXPECT_GE(perf_mon_count, 4);
+
+	TRY_WRITE_READ(PMCCNTR_EL0, 0xaaaa);
+
+	write_msr(PMINTENCLR_EL1, 0xffff);
+	CHECK_READ(PMINTENSET_EL1, 0);
+
+	/*
+	 * Enable the first and second performance counters.
+	 * Bits set in PMINTENSET_EL1 can be read in PMINTENCLR_EL1.
+	 */
+	write_msr(PMINTENSET_EL1, 0x3);
+	CHECK_READ(PMINTENCLR_EL1, 0x3);
+
+	/*
+	 * Enable the third and fourth performance counters.
+	 * Writes to PMINTENSET_EL1 do not clear already set bits.
+	 */
+	write_msr(PMINTENSET_EL1, 0xc);
+	CHECK_READ(PMINTENCLR_EL1, 0xf);
+}
+
+/**
+ * Attempts to read all performance counters supported by the current CPU
+ * configuration.
+ */
+/* NOLINTNEXTLINE(readability-function-size) */
+TEST(perfmon, primary_counters)
+{
+	uintreg_t pmcr_el0 = read_msr(PMCR_EL0);
+	uintreg_t perf_mon_count = GET_PMCR_EL0_N(pmcr_el0);
+
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+
+	if (perf_mon_count == 0) {
+		return;
+	}
+
+	switch (perf_mon_count - 1) {
+	default:
+		FAIL("More performance monitor registers than supported.");
+	case 30:
+		TRY_READ(PMEVCNTR30_EL0);
+		TRY_WRITE_READ(PMEVTYPER30_EL0, 0x1);
+		/* fallthrough */
+	case 29:
+		TRY_READ(PMEVCNTR29_EL0);
+		TRY_WRITE_READ(PMEVTYPER29_EL0, 0x1);
+		/* fallthrough */
+	case 28:
+		TRY_READ(PMEVCNTR28_EL0);
+		TRY_WRITE_READ(PMEVTYPER28_EL0, 0x1);
+		/* fallthrough */
+	case 27:
+		TRY_READ(PMEVCNTR27_EL0);
+		TRY_WRITE_READ(PMEVTYPER27_EL0, 0x1);
+		/* fallthrough */
+	case 26:
+		TRY_READ(PMEVCNTR26_EL0);
+		TRY_WRITE_READ(PMEVTYPER26_EL0, 0x1);
+		/* fallthrough */
+	case 25:
+		TRY_READ(PMEVCNTR25_EL0);
+		TRY_WRITE_READ(PMEVTYPER25_EL0, 0x1);
+		/* fallthrough */
+	case 24:
+		TRY_READ(PMEVCNTR24_EL0);
+		TRY_WRITE_READ(PMEVTYPER24_EL0, 0x1);
+		/* fallthrough */
+	case 23:
+		TRY_READ(PMEVCNTR23_EL0);
+		TRY_WRITE_READ(PMEVTYPER23_EL0, 0x1);
+		/* fallthrough */
+	case 22:
+		TRY_READ(PMEVCNTR22_EL0);
+		TRY_WRITE_READ(PMEVTYPER22_EL0, 0x1);
+		/* fallthrough */
+	case 21:
+		TRY_READ(PMEVCNTR21_EL0);
+		TRY_WRITE_READ(PMEVTYPER21_EL0, 0x1);
+		/* fallthrough */
+	case 20:
+		TRY_READ(PMEVCNTR20_EL0);
+		TRY_WRITE_READ(PMEVTYPER20_EL0, 0x1);
+		/* fallthrough */
+	case 19:
+		TRY_READ(PMEVCNTR19_EL0);
+		TRY_WRITE_READ(PMEVTYPER19_EL0, 0x1);
+		/* fallthrough */
+	case 18:
+		TRY_READ(PMEVCNTR18_EL0);
+		TRY_WRITE_READ(PMEVTYPER18_EL0, 0x1);
+		/* fallthrough */
+	case 17:
+		TRY_READ(PMEVCNTR17_EL0);
+		TRY_WRITE_READ(PMEVTYPER17_EL0, 0x1);
+		/* fallthrough */
+	case 16:
+		TRY_READ(PMEVCNTR16_EL0);
+		TRY_WRITE_READ(PMEVTYPER16_EL0, 0x1);
+		/* fallthrough */
+	case 15:
+		TRY_READ(PMEVCNTR15_EL0);
+		TRY_WRITE_READ(PMEVTYPER15_EL0, 0x1);
+		/* fallthrough */
+	case 14:
+		TRY_READ(PMEVCNTR14_EL0);
+		TRY_WRITE_READ(PMEVTYPER14_EL0, 0x1);
+		/* fallthrough */
+	case 13:
+		TRY_READ(PMEVCNTR13_EL0);
+		TRY_WRITE_READ(PMEVTYPER13_EL0, 0x1);
+		/* fallthrough */
+	case 12:
+		TRY_READ(PMEVCNTR12_EL0);
+		TRY_WRITE_READ(PMEVTYPER12_EL0, 0x1);
+		/* fallthrough */
+	case 11:
+		TRY_READ(PMEVCNTR11_EL0);
+		TRY_WRITE_READ(PMEVTYPER11_EL0, 0x1);
+		/* fallthrough */
+	case 10:
+		TRY_READ(PMEVCNTR10_EL0);
+		TRY_WRITE_READ(PMEVTYPER10_EL0, 0x1);
+		/* fallthrough */
+	case 9:
+		TRY_READ(PMEVCNTR9_EL0);
+		TRY_WRITE_READ(PMEVTYPER9_EL0, 0x1);
+		/* fallthrough */
+	case 8:
+		TRY_READ(PMEVCNTR8_EL0);
+		TRY_WRITE_READ(PMEVTYPER8_EL0, 0x1);
+		/* fallthrough */
+	case 7:
+		TRY_READ(PMEVCNTR7_EL0);
+		TRY_WRITE_READ(PMEVTYPER7_EL0, 0x1);
+		/* fallthrough */
+	case 6:
+		TRY_READ(PMEVCNTR6_EL0);
+		TRY_WRITE_READ(PMEVTYPER6_EL0, 0x1);
+		/* fallthrough */
+	case 5:
+		TRY_READ(PMEVCNTR5_EL0);
+		TRY_WRITE_READ(PMEVTYPER5_EL0, 0x1);
+		/* fallthrough */
+	case 4:
+		TRY_READ(PMEVCNTR4_EL0);
+		TRY_WRITE_READ(PMEVTYPER4_EL0, 0x1);
+		/* fallthrough */
+	case 3:
+		TRY_READ(PMEVCNTR3_EL0);
+		TRY_WRITE_READ(PMEVTYPER3_EL0, 0x1);
+		/* fallthrough */
+	case 2:
+		TRY_READ(PMEVCNTR2_EL0);
+		TRY_WRITE_READ(PMEVTYPER2_EL0, 0x1);
+		/* fallthrough */
+	case 1:
+		TRY_READ(PMEVCNTR1_EL0);
+		TRY_WRITE_READ(PMEVTYPER1_EL0, 0x1);
+		/* fallthrough */
+	case 0:
+		TRY_READ(PMEVCNTR0_EL0);
+		TRY_WRITE_READ(PMEVTYPER0_EL0, 0x1);
+		break;
+	}
+}
diff --git a/test/vmapi/primary_with_secondaries/run_race.c b/test/vmapi/primary_with_secondaries/run_race.c
new file mode 100644
index 0000000..222f17a
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/run_race.c
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include "hf/mm.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+/**
+ * Iterates trying to run vCPU of the secondary VM. Returns when a message
+ * of non-zero length is received.
+ */
+static bool run_loop(struct mailbox_buffers *mb)
+{
+	struct spci_value run_res;
+	bool ok = false;
+
+	for (;;) {
+		/* Run until it manages to schedule vCPU on this CPU. */
+		do {
+			run_res = spci_run(SERVICE_VM1, 0);
+		} while (run_res.func == SPCI_ERROR_32 &&
+			 run_res.arg2 == SPCI_BUSY);
+
+		/* Break out if we received a message with non-zero length. */
+		if (run_res.func == SPCI_MSG_SEND_32 &&
+		    spci_msg_send_size(run_res) != 0) {
+			break;
+		}
+
+		/* Clear mailbox so that next message can be received. */
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	}
+
+	/* Copies the contents of the received boolean to the return value. */
+	if (spci_msg_send_size(run_res) == sizeof(ok)) {
+		ok = *(bool *)mb->recv;
+	}
+
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	return ok;
+}
+
+/**
+ * This is the entry point of the additional primary VM vCPU. It just calls
+ * the run loop so that two CPUs compete for the chance to run a secondary VM.
+ */
+static void vm_cpu_entry(uintptr_t arg)
+{
+	run_loop((struct mailbox_buffers *)arg);
+}
+
+TEAR_DOWN(vcpu_state)
+{
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+}
+
+/**
+ * This test tries to run the same secondary vCPU from two different physical
+ * CPUs concurrently. The vCPU checks that the state is ok while it bounces
+ * between the physical CPUs.
+ *
+ * Test is marked long-running because our implementation of spin-locks does not
+ * perform well under QEMU.
+ */
+TEST_LONG_RUNNING(vcpu_state, concurrent_save_restore)
+{
+	alignas(4096) static char stack[4096];
+	static struct mailbox_buffers mb;
+
+	mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "check_state", mb.send);
+
+	/* Start second vCPU. */
+	ASSERT_TRUE(hftest_cpu_start(hftest_get_cpu_id(1), stack, sizeof(stack),
+				     vm_cpu_entry, (uintptr_t)&mb));
+
+	/* Run on a loop until the secondary VM is done. */
+	EXPECT_TRUE(run_loop(&mb));
+}
diff --git a/test/vmapi/primary_with_secondaries/services/BUILD.gn b/test/vmapi/primary_with_secondaries/services/BUILD.gn
new file mode 100644
index 0000000..d4a78c2
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/BUILD.gn
@@ -0,0 +1,265 @@
+# Copyright 2018 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/image/image.gni")
+
+# Service to expose race conditions when running a vCPU.
+source_set("check_state") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "check_state.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64/hftest:state",
+  ]
+}
+
+# Service to try to access EL1 debug registers.
+source_set("debug_el1") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "debug_el1.c",
+  ]
+}
+
+# Service to try to access performance monitor registers.
+source_set("perfmon") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "perfmon.c",
+  ]
+}
+
+# Service to listen for messages and echo them back to the sender.
+source_set("echo") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "echo.c",
+  ]
+}
+
+# Echo service that waits for recipient to become writable.
+source_set("echo_with_notification") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "echo_with_notification.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64/hftest:interrupts",
+  ]
+}
+
+# Service for floating point register save/restore checks.
+source_set("floating_point") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "floating_point.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64/hftest:registers",
+  ]
+}
+
+# Services related to memory sharing.
+source_set("memory") {
+  testonly = true
+  public_configs = [
+    "..:config",
+    "//test/hftest:hftest_config",
+  ]
+  deps = [
+    "//test/vmapi/common",
+    "//vmlib",
+  ]
+
+  sources = [
+    "memory.c",
+  ]
+}
+
+# Services related to VMs that access unmapped memory.
+source_set("unmapped") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "unmapped.c",
+  ]
+
+  deps = [
+    "//test/vmapi/common:common",
+  ]
+}
+
+# Services related to the boot process for VMs.
+source_set("boot") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "boot.c",
+  ]
+
+  deps = [
+    "//test/vmapi/common:common",
+  ]
+}
+
+# Service that can be interrupted.
+source_set("interruptible") {
+  testonly = true
+  public_configs = [
+    "..:config",
+    "//test/hftest:hftest_config",
+  ]
+
+  sources = [
+    "interruptible.c",
+    "interruptible_echo.c",
+  ]
+
+  deps = [
+    "//src/arch/aarch64/hftest:interrupts",
+  ]
+}
+
+# Service to check that hf_mailbox_receive can't block when there are pending
+# interrupts.
+source_set("receive_block") {
+  testonly = true
+  public_configs = [
+    "..:config",
+    "//test/hftest:hftest_config",
+  ]
+  sources = [
+    "receive_block.c",
+  ]
+  deps = [
+    "//src/arch/aarch64:arch",
+    "//src/arch/aarch64/hftest:interrupts",
+    "//test/vmapi/common",
+  ]
+}
+
+# Service to listen for messages and forward them on to another.
+source_set("relay") {
+  testonly = true
+  public_configs = [ "//test/hftest:hftest_config" ]
+
+  sources = [
+    "relay.c",
+  ]
+}
+
+# Service to start a second vCPU and send messages from both.
+source_set("smp") {
+  testonly = true
+  public_configs = [
+    "..:config",
+    "//test/hftest:hftest_config",
+  ]
+  sources = [
+    "smp.c",
+  ]
+}
+
+# Service to check that WFI is a no-op when there are pending interrupts.
+source_set("wfi") {
+  testonly = true
+  public_configs = [
+    "..:config",
+    "//test/hftest:hftest_config",
+  ]
+  sources = [
+    "wfi.c",
+  ]
+  deps = [
+    "//src/arch/aarch64/hftest:interrupts",
+  ]
+}
+
+# Service to receive messages in a secondary VM and ensure that the header fields are correctly set.
+source_set("spci_check") {
+  testonly = true
+  public_configs = [
+    "..:config",
+    "//test/hftest:hftest_config",
+  ]
+  deps = [
+    "//test/vmapi/common",
+  ]
+
+  sources = [
+    "spci_check.c",
+  ]
+}
+
+# Group services together into VMs.
+
+vm_kernel("service_vm1") {
+  testonly = true
+
+  deps = [
+    ":boot",
+    ":check_state",
+    ":debug_el1",
+    ":echo",
+    ":echo_with_notification",
+    ":floating_point",
+    ":interruptible",
+    ":memory",
+    ":perfmon",
+    ":receive_block",
+    ":relay",
+    ":spci_check",
+    ":unmapped",
+    ":wfi",
+    "//test/hftest:hftest_secondary_vm",
+  ]
+}
+
+vm_kernel("service_vm2") {
+  testonly = true
+
+  deps = [
+    ":interruptible",
+    ":memory",
+    ":relay",
+    "//test/hftest:hftest_secondary_vm",
+  ]
+}
+
+vm_kernel("service_vm3") {
+  testonly = true
+
+  deps = [
+    ":smp",
+    "//test/hftest:hftest_secondary_vm",
+  ]
+}
diff --git a/test/vmapi/primary_with_secondaries/services/boot.c b/test/vmapi/primary_with_secondaries/services/boot.c
new file mode 100644
index 0000000..a7b9b66
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/boot.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/mm.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+#include "test/vmapi/exception_handler.h"
+
+/*
+ * This must match the size specified for services1 in
+ * //test/vmapi/primary_with_secondaries:primary_with_secondaries_test.
+ */
+#define SECONDARY_MEMORY_SIZE 1048576
+
+extern uint8_t volatile text_begin[];
+
+TEST_SERVICE(boot_memory)
+{
+	uint8_t checksum = 0;
+
+	/* Check that the size passed in by Hafnium is what is expected. */
+	ASSERT_EQ(SERVICE_MEMORY_SIZE(), SECONDARY_MEMORY_SIZE);
+
+	/*
+	 * Check that we can read all memory up to the given size. Calculate a
+	 * basic checksum and check that it is non-zero, as a double-check that
+	 * we are actually reading something.
+	 */
+	for (size_t i = 0; i < SERVICE_MEMORY_SIZE(); ++i) {
+		checksum += text_begin[i];
+	}
+	ASSERT_NE(checksum, 0);
+	dlog("Checksum of all memory is %d\n", checksum);
+
+	spci_yield();
+}
+
+TEST_SERVICE(boot_memory_underrun)
+{
+	exception_setup(NULL, exception_handler_yield);
+	/*
+	 * Try to read memory below the start of the image. This should result
+	 * in the VM trapping and yielding.
+	 */
+	dlog("Read memory below limit: %d\n", text_begin[-1]);
+	FAIL("Managed to read memory below limit");
+}
+
+TEST_SERVICE(boot_memory_overrun)
+{
+	exception_setup(NULL, exception_handler_yield);
+	/*
+	 * Try to read memory above the limit defined by memory_size. This
+	 * should result in the VM trapping and yielding.
+	 */
+	dlog("Read memory above limit: %d\n",
+	     text_begin[SERVICE_MEMORY_SIZE()]);
+	FAIL("Managed to read memory above limit");
+}
diff --git a/test/vmapi/primary_with_secondaries/services/check_state.c b/test/vmapi/primary_with_secondaries/services/check_state.c
new file mode 100644
index 0000000..da27cc6
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/check_state.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/state.h"
+
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+
+void send_with_retry(spci_vm_id_t sender_vm_id, spci_vm_id_t target_vm_id,
+		     uint32_t size)
+{
+	struct spci_value res;
+
+	do {
+		res = spci_msg_send(sender_vm_id, target_vm_id, size, 0);
+	} while (res.func != SPCI_SUCCESS_32);
+}
+
+/**
+ * This service repeatedly takes the following steps: sets the per-CPU pointer
+ * to some value, makes a hypervisor call, check that the value is still what it
+ * was set to.
+ *
+ * This loop helps detect bugs where the hypervisor inadvertently destroys
+ * state.
+ *
+ * At the end of its iterations, the service reports the result to the primary
+ * VM, which then fails or succeeds the test.
+ */
+TEST_SERVICE(check_state)
+{
+	size_t i;
+	bool ok = true;
+	static volatile uintptr_t expected;
+	static volatile uintptr_t actual;
+
+	for (i = 0; i < 100000; i++) {
+		/*
+		 * We store the expected/actual values in volatile static
+		 * variables to avoid relying on registers that may have been
+		 * modified by the hypervisor.
+		 */
+		expected = i;
+		per_cpu_ptr_set(expected);
+		send_with_retry(hf_vm_get_id(), HF_PRIMARY_VM_ID, 0);
+		actual = per_cpu_ptr_get();
+		ok &= expected == actual;
+	}
+
+	/* Send two replies, one for each physical CPU. */
+	memcpy_s(SERVICE_SEND_BUFFER(), SPCI_MSG_PAYLOAD_MAX, &ok, sizeof(ok));
+	send_with_retry(hf_vm_get_id(), HF_PRIMARY_VM_ID, sizeof(ok));
+	send_with_retry(hf_vm_get_id(), HF_PRIMARY_VM_ID, sizeof(ok));
+}
diff --git a/test/vmapi/primary_with_secondaries/services/debug_el1.c b/test/vmapi/primary_with_secondaries/services/debug_el1.c
new file mode 100644
index 0000000..1e09f76
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/debug_el1.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/dlog.h"
+
+#include "../sysregs.h"
+#include "test/vmapi/exception_handler.h"
+
+TEST_SERVICE(debug_el1_secondary_basic)
+{
+	exception_setup(NULL, exception_handler_skip_instruction);
+
+	EXPECT_GT(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+	TRY_READ(MDCCINT_EL1);
+	TRY_READ(DBGBCR0_EL1);
+	TRY_READ(DBGBVR0_EL1);
+	TRY_READ(DBGWCR0_EL1);
+	TRY_READ(DBGWVR0_EL1);
+
+	EXPECT_EQ(exception_handler_get_num(), 5);
+	spci_yield();
+}
diff --git a/test/vmapi/primary_with_secondaries/services/echo.c b/test/vmapi/primary_with_secondaries/services/echo.c
new file mode 100644
index 0000000..b1c99d9
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/echo.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+
+TEST_SERVICE(echo)
+{
+	/* Loop, echo messages back to the sender. */
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		spci_vm_id_t target_vm_id = spci_msg_send_receiver(ret);
+		spci_vm_id_t source_vm_id = spci_msg_send_sender(ret);
+		void *send_buf = SERVICE_SEND_BUFFER();
+		void *recv_buf = SERVICE_RECV_BUFFER();
+
+		ASSERT_EQ(ret.func, SPCI_MSG_SEND_32);
+		memcpy_s(send_buf, SPCI_MSG_PAYLOAD_MAX, recv_buf,
+			 spci_msg_send_size(ret));
+
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		spci_msg_send(target_vm_id, source_vm_id,
+			      spci_msg_send_size(ret), 0);
+	}
+}
diff --git a/test/vmapi/primary_with_secondaries/services/echo_with_notification.c b/test/vmapi/primary_with_secondaries/services/echo_with_notification.c
new file mode 100644
index 0000000..a74391b
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/echo_with_notification.c
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "test/hftest.h"
+
+static void irq(void)
+{
+	hf_interrupt_get();
+}
+
+static void wait_for_vm(uint32_t vmid)
+{
+	for (;;) {
+		int64_t w = hf_mailbox_writable_get();
+		if (w == vmid) {
+			return;
+		}
+
+		if (w == -1) {
+			interrupt_wait();
+			arch_irq_enable();
+			arch_irq_disable();
+		}
+	}
+}
+
+TEST_SERVICE(echo_with_notification)
+{
+	exception_setup(irq, NULL);
+	hf_interrupt_enable(HF_MAILBOX_WRITABLE_INTID, true);
+
+	/* Loop, echo messages back to the sender. */
+	for (;;) {
+		void *send_buf = SERVICE_SEND_BUFFER();
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		struct spci_value ret = spci_msg_wait();
+		spci_vm_id_t target_vm_id = spci_msg_send_receiver(ret);
+		spci_vm_id_t source_vm_id = spci_msg_send_sender(ret);
+
+		memcpy_s(send_buf, SPCI_MSG_PAYLOAD_MAX, recv_buf,
+			 spci_msg_send_size(ret));
+
+		while (spci_msg_send(target_vm_id, source_vm_id,
+				     spci_msg_send_size(ret),
+				     SPCI_MSG_SEND_NOTIFY)
+			       .func != SPCI_SUCCESS_32) {
+			wait_for_vm(source_vm_id);
+		}
+
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	}
+}
diff --git a/test/vmapi/primary_with_secondaries/services/floating_point.c b/test/vmapi/primary_with_secondaries/services/floating_point.c
new file mode 100644
index 0000000..f5c0bcc
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/floating_point.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/std.h"
+#include "hf/arch/vm/registers.h"
+
+#include "hf/spci.h"
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "test/hftest.h"
+
+TEST_SERVICE(fp_fill)
+{
+	const double value = 0.75;
+	fill_fp_registers(value);
+	EXPECT_EQ(spci_yield().func, SPCI_SUCCESS_32);
+
+	ASSERT_TRUE(check_fp_register(value));
+	spci_yield();
+}
+
+TEST_SERVICE(fp_fpcr)
+{
+	uintreg_t value = 3 << 22; /* Set RMode to RZ */
+	write_msr(fpcr, value);
+	EXPECT_EQ(spci_yield().func, SPCI_SUCCESS_32);
+
+	ASSERT_EQ(read_msr(fpcr), value);
+	spci_yield();
+}
diff --git a/test/vmapi/primary_with_secondaries/services/interruptible.c b/test/vmapi/primary_with_secondaries/services/interruptible.c
new file mode 100644
index 0000000..fc79d20
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/interruptible.c
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+#include "vmapi/hf/spci.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+
+/*
+ * Secondary VM that sends messages in response to interrupts, and interrupts
+ * itself when it receives a message.
+ */
+
+static void irq(void)
+{
+	uint32_t interrupt_id = hf_interrupt_get();
+	char buffer[] = "Got IRQ xx.";
+	int size = sizeof(buffer);
+	dlog("secondary IRQ %d from current\n", interrupt_id);
+	buffer[8] = '0' + interrupt_id / 10;
+	buffer[9] = '0' + interrupt_id % 10;
+	memcpy_s(SERVICE_SEND_BUFFER(), SPCI_MSG_PAYLOAD_MAX, buffer, size);
+	spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, size, 0);
+	dlog("secondary IRQ %d ended\n", interrupt_id);
+}
+
+/**
+ * Try to receive a message from the mailbox, blocking if necessary, and
+ * retrying if interrupted.
+ */
+struct spci_value mailbox_receive_retry()
+{
+	struct spci_value received;
+
+	do {
+		received = spci_msg_wait();
+	} while (received.func == SPCI_ERROR_32 &&
+		 received.arg2 == SPCI_INTERRUPTED);
+
+	return received;
+}
+
+TEST_SERVICE(interruptible)
+{
+	spci_vm_id_t this_vm_id = hf_vm_get_id();
+	void *recv_buf = SERVICE_RECV_BUFFER();
+
+	exception_setup(irq, NULL);
+	hf_interrupt_enable(SELF_INTERRUPT_ID, true);
+	hf_interrupt_enable(EXTERNAL_INTERRUPT_ID_A, true);
+	hf_interrupt_enable(EXTERNAL_INTERRUPT_ID_B, true);
+	arch_irq_enable();
+
+	for (;;) {
+		const char ping_message[] = "Ping";
+		const char enable_message[] = "Enable interrupt C";
+
+		struct spci_value ret = mailbox_receive_retry();
+
+		ASSERT_EQ(ret.func, SPCI_MSG_SEND_32);
+		if (spci_msg_send_sender(ret) == HF_PRIMARY_VM_ID &&
+		    spci_msg_send_size(ret) == sizeof(ping_message) &&
+		    memcmp(recv_buf, ping_message, sizeof(ping_message)) == 0) {
+			/* Interrupt ourselves */
+			hf_interrupt_inject(this_vm_id, 0, SELF_INTERRUPT_ID);
+		} else if (spci_msg_send_sender(ret) == HF_PRIMARY_VM_ID &&
+			   spci_msg_send_size(ret) == sizeof(enable_message) &&
+			   memcmp(recv_buf, enable_message,
+				  sizeof(enable_message)) == 0) {
+			/* Enable interrupt ID C. */
+			hf_interrupt_enable(EXTERNAL_INTERRUPT_ID_C, true);
+		} else {
+			dlog("Got unexpected message from VM %d, size %d.\n",
+			     spci_msg_send_sender(ret),
+			     spci_msg_send_size(ret));
+			FAIL("Unexpected message");
+		}
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	}
+}
diff --git a/test/vmapi/primary_with_secondaries/services/interruptible_echo.c b/test/vmapi/primary_with_secondaries/services/interruptible_echo.c
new file mode 100644
index 0000000..8903990
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/interruptible_echo.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+
+static void irq(void)
+{
+	/* Clear the interrupt. */
+	hf_interrupt_get();
+}
+
+TEST_SERVICE(interruptible_echo)
+{
+	exception_setup(irq, NULL);
+	hf_interrupt_enable(EXTERNAL_INTERRUPT_ID_A, true);
+	arch_irq_enable();
+
+	for (;;) {
+		struct spci_value res = spci_msg_wait();
+		void *message = SERVICE_SEND_BUFFER();
+		void *recv_message = SERVICE_RECV_BUFFER();
+
+		/* Retry if interrupted but made visible with the yield. */
+		while (res.func == SPCI_ERROR_32 &&
+		       res.arg2 == SPCI_INTERRUPTED) {
+			spci_yield();
+			res = spci_msg_wait();
+		}
+
+		ASSERT_EQ(res.func, SPCI_MSG_SEND_32);
+		memcpy_s(message, SPCI_MSG_PAYLOAD_MAX, recv_message,
+			 spci_msg_send_size(res));
+
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		spci_msg_send(SERVICE_VM1, HF_PRIMARY_VM_ID,
+			      spci_msg_send_size(res), 0);
+	}
+}
diff --git a/test/vmapi/primary_with_secondaries/services/memory.c b/test/vmapi/primary_with_secondaries/services/memory.c
new file mode 100644
index 0000000..34909cc
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/memory.c
@@ -0,0 +1,841 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/mm.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/exception_handler.h"
+#include "test/vmapi/spci.h"
+
+alignas(PAGE_SIZE) static uint8_t page[PAGE_SIZE];
+
+TEST_SERVICE(memory_increment)
+{
+	/* Loop, writing message to the shared memory. */
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		size_t i;
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		struct spci_memory_region *memory_region =
+			(struct spci_memory_region *)recv_buf;
+		struct spci_memory_region_constituent *constituents =
+			spci_memory_region_get_constituents(memory_region);
+		spci_vm_id_t sender = memory_region->sender;
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		EXPECT_EQ(spci_msg_send_attributes(ret),
+			  SPCI_MSG_SEND_LEGACY_MEMORY_SHARE);
+
+		ptr = (uint8_t *)constituents[0].address;
+
+		/* Check the memory was cleared. */
+		for (i = 0; i < PAGE_SIZE; ++i) {
+			ASSERT_EQ(ptr[i], 0);
+		}
+
+		/* Allow the memory to be populated. */
+		EXPECT_EQ(spci_yield().func, SPCI_SUCCESS_32);
+
+		/* Increment each byte of memory. */
+		for (i = 0; i < PAGE_SIZE; ++i) {
+			++ptr[i];
+		}
+
+		/* Signal completion and reset. */
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		spci_msg_send(hf_vm_get_id(), sender, sizeof(ptr), 0);
+	}
+}
+
+TEST_SERVICE(give_memory_and_fault)
+{
+	void *send_buf = SERVICE_SEND_BUFFER();
+
+	/* Give memory to the primary. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)&page, .page_count = 1},
+	};
+	uint32_t msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, SPCI_MEMORY_REGION_FLAG_CLEAR,
+		SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+		SPCI_MEMORY_CACHE_WRITE_BACK, SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	exception_setup(NULL, exception_handler_yield);
+
+	/* Try using the memory that isn't valid unless it's been returned. */
+	page[16] = 123;
+
+	FAIL("Exception not generated by invalid access.");
+}
+
+TEST_SERVICE(lend_memory_and_fault)
+{
+	void *send_buf = SERVICE_SEND_BUFFER();
+
+	/* Lend memory to the primary. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)&page, .page_count = 1},
+	};
+	uint32_t msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, SPCI_MEMORY_REGION_FLAG_CLEAR,
+		SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+		SPCI_MEMORY_CACHE_WRITE_BACK, SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_LEND)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	exception_setup(NULL, exception_handler_yield);
+
+	/* Try using the memory that isn't valid unless it's been returned. */
+	page[633] = 180;
+
+	FAIL("Exception not generated by invalid access.");
+}
+
+TEST_SERVICE(spci_memory_return)
+{
+	exception_setup(NULL, exception_handler_yield);
+
+	/* Loop, giving memory back to the sender. */
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		uint32_t msg_size;
+		size_t i;
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		void *send_buf = SERVICE_SEND_BUFFER();
+		struct spci_memory_region *memory_region;
+		struct spci_memory_region_constituent *constituents;
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		/*
+		 * The memory may have been sent in one of several different
+		 * ways, but there shouldn't be any other attributes to the
+		 * message.
+		 */
+		EXPECT_NE(spci_msg_send_attributes(ret) &
+				  SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+			  0);
+		EXPECT_EQ(spci_msg_send_attributes(ret) &
+				  ~SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+			  0);
+
+		memory_region = (struct spci_memory_region *)recv_buf;
+		constituents =
+			spci_memory_region_get_constituents(memory_region);
+		ptr = (uint8_t *)constituents[0].address;
+
+		/* Check that one has access to the shared region. */
+		for (i = 0; i < PAGE_SIZE; ++i) {
+			ptr[i]++;
+		}
+
+		/* Give the memory back and notify the sender. */
+		msg_size = spci_memory_region_init(
+			send_buf, hf_vm_get_id(), memory_region->sender,
+			constituents, memory_region->constituent_count, 0, 0,
+			SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+			SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		EXPECT_EQ(spci_msg_send(spci_msg_send_receiver(ret),
+					spci_msg_send_sender(ret), msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+				  .func,
+			  SPCI_SUCCESS_32);
+
+		/*
+		 * Try and access the memory which will cause a fault unless the
+		 * memory has been shared back again.
+		 */
+		ptr[0] = 123;
+
+		FAIL("Exception not generated by invalid access.");
+	}
+}
+
+TEST_SERVICE(spci_donate_check_upper_bound)
+{
+	exception_setup(NULL, exception_handler_yield);
+
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		uint8_t index;
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		struct spci_memory_region *memory_region;
+		struct spci_memory_region_constituent *constituents;
+
+		exception_handler_reset();
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		EXPECT_EQ(spci_msg_send_attributes(ret),
+			  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+
+		memory_region = (struct spci_memory_region *)recv_buf;
+		constituents =
+			spci_memory_region_get_constituents(memory_region);
+
+		/* Choose which constituent we want to test. */
+		index = *(uint8_t *)constituents[0].address;
+		ptr = (uint8_t *)constituents[index].address;
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+		/*
+		 * Check that one cannot access out of bounds after donated
+		 * region. This should trigger the exception handler.
+		 */
+		ptr[PAGE_SIZE]++;
+	}
+}
+
+TEST_SERVICE(spci_donate_check_lower_bound)
+{
+	exception_setup(NULL, exception_handler_yield);
+
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		uint8_t index;
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		struct spci_memory_region *memory_region;
+		struct spci_memory_region_constituent *constituents;
+
+		exception_handler_reset();
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		EXPECT_EQ(spci_msg_send_attributes(ret),
+			  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+
+		memory_region = (struct spci_memory_region *)recv_buf;
+		constituents =
+			spci_memory_region_get_constituents(memory_region);
+
+		/* Choose which constituent we want to test. */
+		index = *(uint8_t *)constituents[0].address;
+		ptr = (uint8_t *)constituents[index].address;
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+		/*
+		 * Check that one cannot access out of bounds after donated
+		 * region. This should trigger the exception handler.
+		 */
+		ptr[-1]++;
+	}
+}
+
+/**
+ * Attempt to donate memory and then modify.
+ */
+TEST_SERVICE(spci_donate_secondary_and_fault)
+{
+	struct spci_value ret = spci_msg_wait();
+	uint8_t *ptr;
+	uint32_t msg_size;
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	void *send_buf = SERVICE_SEND_BUFFER();
+	struct spci_memory_region *memory_region =
+		(struct spci_memory_region *)recv_buf;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(ret),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+
+	exception_setup(NULL, exception_handler_yield);
+
+	ptr = (uint8_t *)constituents[0].address;
+
+	/* Donate memory to next VM. */
+	msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), SERVICE_VM2, constituents,
+		memory_region->constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	EXPECT_EQ(spci_msg_send(spci_msg_send_receiver(ret), SERVICE_VM2,
+				msg_size, SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Ensure that we are unable to modify memory any more. */
+	ptr[0] = 'c';
+
+	FAIL("Exception not generated by invalid access.");
+}
+
+/**
+ * Attempt to donate memory twice from VM.
+ */
+TEST_SERVICE(spci_donate_twice)
+{
+	uint32_t msg_size;
+	struct spci_value ret = spci_msg_wait();
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	void *send_buf = SERVICE_SEND_BUFFER();
+	struct spci_memory_region *memory_region =
+		(struct spci_memory_region *)recv_buf;
+	struct spci_memory_region_constituent constituent =
+		spci_memory_region_get_constituents(memory_region)[0];
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(ret),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Yield to allow attempt to re donate from primary. */
+	spci_yield();
+
+	/* Give the memory back and notify the sender. */
+	msg_size = spci_memory_region_init(
+		send_buf, SERVICE_VM1, HF_PRIMARY_VM_ID, &constituent, 1, 0, 0,
+		SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+		SPCI_MEMORY_CACHE_WRITE_BACK, SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(SERVICE_VM1, HF_PRIMARY_VM_ID, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Attempt to donate the memory to another VM. */
+	msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), SERVICE_VM2, &constituent, 1, 0, 0,
+		SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+		SPCI_MEMORY_CACHE_WRITE_BACK, SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(spci_msg_send_receiver(ret), SERVICE_VM2,
+			      msg_size, SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+		SPCI_INVALID_PARAMETERS);
+
+	spci_yield();
+}
+
+/**
+ * Continually receive memory, check if we have access and ensure it is not
+ * changed by a third party.
+ */
+TEST_SERVICE(spci_memory_receive)
+{
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		struct spci_memory_region *memory_region =
+			(struct spci_memory_region *)recv_buf;
+		struct spci_memory_region_constituent *constituents =
+			spci_memory_region_get_constituents(memory_region);
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		EXPECT_EQ(spci_msg_send_attributes(ret),
+			  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+
+		ptr = (uint8_t *)constituents[0].address;
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		ptr[0] = 'd';
+		spci_yield();
+
+		/* Ensure memory has not changed. */
+		EXPECT_EQ(ptr[0], 'd');
+		spci_yield();
+	}
+}
+
+/**
+ * Receive memory and attempt to donate from primary VM.
+ */
+TEST_SERVICE(spci_donate_invalid_source)
+{
+	uint32_t msg_size;
+	struct spci_value ret = spci_msg_wait();
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	void *send_buf = SERVICE_SEND_BUFFER();
+	struct spci_memory_region *memory_region =
+		(struct spci_memory_region *)recv_buf;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(ret),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+
+	/* Give the memory back and notify the sender. */
+	msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID, constituents,
+		memory_region->constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(spci_msg_send_receiver(ret), HF_PRIMARY_VM_ID,
+				msg_size, SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Fail to donate the memory from the primary to VM2. */
+	msg_size = spci_memory_region_init(
+		send_buf, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+		memory_region->constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	EXPECT_SPCI_ERROR(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_DONATE),
+			  SPCI_INVALID_PARAMETERS);
+	spci_yield();
+}
+
+TEST_SERVICE(spci_memory_lend_relinquish)
+{
+	exception_setup(NULL, exception_handler_yield);
+
+	/* Loop, giving memory back to the sender. */
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		uint8_t *ptr2;
+		uint32_t count;
+		uint32_t count2;
+		uint32_t msg_size;
+		size_t i;
+
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		void *send_buf = SERVICE_SEND_BUFFER();
+		struct spci_memory_region *memory_region =
+			(struct spci_memory_region *)recv_buf;
+		struct spci_memory_region_constituent *constituents =
+			spci_memory_region_get_constituents(memory_region);
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		/*
+		 * The memory may have been sent in one of several different
+		 * ways, but there shouldn't be any other attributes to the
+		 * message.
+		 */
+		EXPECT_NE(spci_msg_send_attributes(ret) &
+				  SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+			  0);
+		EXPECT_EQ(spci_msg_send_attributes(ret) &
+				  ~SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+			  0);
+
+		ptr = (uint8_t *)constituents[0].address;
+		count = constituents[0].page_count;
+		ptr2 = (uint8_t *)constituents[1].address;
+		count2 = constituents[1].page_count;
+
+		/* Check that one has access to the shared region. */
+		for (i = 0; i < PAGE_SIZE * count; ++i) {
+			ptr[i]++;
+		}
+		for (i = 0; i < PAGE_SIZE * count2; ++i) {
+			ptr2[i]++;
+		}
+
+		/* Give the memory back and notify the sender. */
+		msg_size = spci_memory_region_init(
+			send_buf, hf_vm_get_id(), memory_region->sender,
+			constituents, memory_region->constituent_count, 0, 0,
+			SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+			SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		/* Relevant information read, mailbox can be cleared. */
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		EXPECT_EQ(spci_msg_send(spci_msg_send_receiver(ret),
+					spci_msg_send_sender(ret), msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH)
+				  .func,
+			  SPCI_SUCCESS_32);
+
+		/*
+		 * Try and access the memory which will cause a fault unless the
+		 * memory has been shared back again.
+		 */
+		ptr[0] = 123;
+	}
+}
+
+/**
+ * Ensure that we can't relinquish donated memory.
+ */
+TEST_SERVICE(spci_memory_donate_relinquish)
+{
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		uint32_t msg_size;
+		size_t i;
+
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		void *send_buf = SERVICE_SEND_BUFFER();
+		struct spci_memory_region *memory_region =
+			(struct spci_memory_region *)recv_buf;
+		struct spci_memory_region_constituent *constituents =
+			spci_memory_region_get_constituents(memory_region);
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		EXPECT_EQ(spci_msg_send_attributes(ret),
+			  SPCI_MSG_SEND_LEGACY_MEMORY_DONATE);
+
+		ptr = (uint8_t *)constituents[0].address;
+
+		/* Check that one has access to the shared region. */
+		for (i = 0; i < PAGE_SIZE; ++i) {
+			ptr[i]++;
+		}
+		/* Give the memory back and notify the sender. */
+		msg_size = spci_memory_region_init(
+			send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID,
+			constituents, memory_region->constituent_count, 0, 0,
+			SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+			SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(spci_msg_send_receiver(ret),
+				      HF_PRIMARY_VM_ID, msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH),
+			SPCI_INVALID_PARAMETERS);
+
+		/* Ensure we still have access to the memory. */
+		ptr[0] = 123;
+
+		spci_yield();
+	}
+}
+
+/**
+ * Receive memory and attempt to donate from primary VM.
+ */
+TEST_SERVICE(spci_lend_invalid_source)
+{
+	uint32_t msg_size;
+	struct spci_value ret = spci_msg_wait();
+
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	void *send_buf = SERVICE_SEND_BUFFER();
+	struct spci_memory_region *memory_region =
+		(struct spci_memory_region *)recv_buf;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(ret),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_LEND);
+
+	/* Attempt to relinquish from primary VM. */
+	msg_size = spci_memory_region_init(
+		send_buf, HF_PRIMARY_VM_ID, hf_vm_get_id(), constituents,
+		memory_region->constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(HF_PRIMARY_VM_ID, hf_vm_get_id(), msg_size,
+			      SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH),
+		SPCI_INVALID_PARAMETERS);
+
+	/* Give the memory back and notify the sender. */
+	msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID, constituents,
+		memory_region->constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Ensure we cannot lend from the primary to another secondary. */
+	msg_size = spci_memory_region_init(
+		send_buf, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+		memory_region->constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_SPCI_ERROR(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_LEND),
+			  SPCI_INVALID_PARAMETERS);
+
+	/* Ensure we cannot share from the primary to another secondary. */
+	msg_size = spci_memory_region_init(
+		send_buf, HF_PRIMARY_VM_ID, SERVICE_VM2, constituents,
+		memory_region->constituent_count, 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+	EXPECT_SPCI_ERROR(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_SHARE),
+			  SPCI_INVALID_PARAMETERS);
+
+	spci_yield();
+}
+
+/**
+ * Attempt to execute an instruction from the lent memory.
+ */
+TEST_SERVICE(spci_memory_lend_relinquish_X)
+{
+	exception_setup(NULL, exception_handler_yield);
+
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint64_t *ptr;
+		uint32_t msg_size;
+
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		void *send_buf = SERVICE_SEND_BUFFER();
+		struct spci_memory_region *memory_region =
+			(struct spci_memory_region *)recv_buf;
+		struct spci_memory_region_constituent *constituents =
+			spci_memory_region_get_constituents(memory_region);
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		EXPECT_EQ(spci_msg_send_attributes(ret),
+			  SPCI_MSG_SEND_LEGACY_MEMORY_LEND);
+
+		ptr = (uint64_t *)constituents[0].address;
+		/*
+		 * Verify that the instruction in memory is the encoded RET
+		 * instruction.
+		 */
+		EXPECT_EQ(*ptr, 0xD65F03C0);
+		/* Try to execute instruction from the shared memory region. */
+		__asm__ volatile("blr %0" ::"r"(ptr));
+
+		/* Release the memory again. */
+		msg_size = spci_memory_region_init(
+			send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID,
+			constituents, memory_region->constituent_count, 0, 0,
+			SPCI_MEMORY_RW_X, SPCI_MEMORY_NORMAL_MEM,
+			SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		EXPECT_EQ(spci_msg_send(spci_msg_send_receiver(ret),
+					HF_PRIMARY_VM_ID, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH)
+				  .func,
+			  SPCI_SUCCESS_32);
+	}
+}
+
+/**
+ * Attempt to read and write to a shared page.
+ */
+TEST_SERVICE(spci_memory_lend_relinquish_RW)
+{
+	exception_setup(NULL, exception_handler_yield);
+
+	for (;;) {
+		struct spci_value ret = spci_msg_wait();
+		uint8_t *ptr;
+		uint32_t msg_size;
+		size_t i;
+
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		void *send_buf = SERVICE_SEND_BUFFER();
+		struct spci_memory_region *memory_region =
+			(struct spci_memory_region *)recv_buf;
+		struct spci_memory_region_constituent *constituents =
+			spci_memory_region_get_constituents(memory_region);
+		struct spci_memory_region_constituent constituent_copy =
+			constituents[0];
+
+		EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+		/*
+		 * The memory may have been sent in one of several different
+		 * ways, but there shouldn't be any other attributes to the
+		 * message.
+		 */
+		EXPECT_NE(spci_msg_send_attributes(ret) &
+				  SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+			  0);
+		EXPECT_EQ(spci_msg_send_attributes(ret) &
+				  ~SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+			  0);
+
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+		ptr = (uint8_t *)constituent_copy.address;
+
+		/* Check that we have read access. */
+		for (i = 0; i < PAGE_SIZE; ++i) {
+			EXPECT_EQ(ptr[i], 'b');
+		}
+
+		/* Return control to primary, to verify shared access. */
+		spci_yield();
+
+		/* Attempt to modify the memory. */
+		for (i = 0; i < PAGE_SIZE; ++i) {
+			ptr[i]++;
+		}
+
+		msg_size = spci_memory_region_init(
+			send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID,
+			&constituent_copy, 1, 0, 0, SPCI_MEMORY_RW_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_EQ(spci_msg_send(spci_msg_send_receiver(ret),
+					HF_PRIMARY_VM_ID, msg_size,
+					SPCI_MSG_SEND_LEGACY_MEMORY_RELINQUISH)
+				  .func,
+			  SPCI_SUCCESS_32);
+	}
+}
+
+/**
+ * Attempt to modify above the upper bound for the lent memory.
+ */
+TEST_SERVICE(spci_lend_check_upper_bound)
+{
+	struct spci_value ret = spci_msg_wait();
+	uint8_t *ptr;
+	uint8_t index;
+
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	struct spci_memory_region *memory_region =
+		(struct spci_memory_region *)recv_buf;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(ret),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_LEND);
+
+	exception_setup(NULL, exception_handler_yield);
+
+	/* Choose which constituent we want to test. */
+	index = *(uint8_t *)constituents[0].address;
+	ptr = (uint8_t *)constituents[index].address;
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Check that one cannot access after lent region. */
+	ASSERT_EQ(ptr[PAGE_SIZE], 0);
+
+	FAIL("Exception not generated by invalid access.");
+}
+
+/**
+ * Attempt to modify below the lower bound for the lent memory.
+ */
+TEST_SERVICE(spci_lend_check_lower_bound)
+{
+	struct spci_value ret = spci_msg_wait();
+	uint8_t *ptr;
+	uint8_t index;
+
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	struct spci_memory_region *memory_region =
+		(struct spci_memory_region *)recv_buf;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	exception_setup(NULL, exception_handler_yield);
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_attributes(ret),
+		  SPCI_MSG_SEND_LEGACY_MEMORY_LEND);
+
+	/* Choose which constituent we want to test. */
+	index = *(uint8_t *)constituents[0].address;
+	ptr = (uint8_t *)constituents[index].address;
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Check that one cannot access after lent region. */
+	ptr[-1]++;
+
+	FAIL("Exception not generated by invalid access.");
+}
+
+TEST_SERVICE(spci_memory_lend_twice)
+{
+	struct spci_value ret = spci_msg_wait();
+	uint8_t *ptr;
+	uint32_t msg_size;
+	size_t i;
+
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	void *send_buf = SERVICE_SEND_BUFFER();
+	struct spci_memory_region *memory_region =
+		(struct spci_memory_region *)recv_buf;
+	struct spci_memory_region_constituent *constituents =
+		spci_memory_region_get_constituents(memory_region);
+	struct spci_memory_region_constituent constituent_copy =
+		constituents[0];
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+	/*
+	 * The memory may have been sent in one of several different ways, but
+	 * there shouldn't be any other attributes to the message.
+	 */
+	EXPECT_NE(spci_msg_send_attributes(ret) &
+			  SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+		  0);
+	EXPECT_EQ(spci_msg_send_attributes(ret) &
+			  ~SPCI_MSG_SEND_LEGACY_MEMORY_MASK,
+		  0);
+
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	ptr = (uint8_t *)constituent_copy.address;
+
+	/* Check that we have read access. */
+	for (i = 0; i < PAGE_SIZE; ++i) {
+		EXPECT_EQ(ptr[i], 'b');
+	}
+
+	/* Attempt to modify the memory. */
+	for (i = 0; i < PAGE_SIZE; ++i) {
+		ptr[i]++;
+	}
+
+	for (i = 1; i < PAGE_SIZE * 2; i++) {
+		constituent_copy.address = (uint64_t)ptr + i;
+
+		/* Fail to lend or share the memory from the primary. */
+		msg_size = spci_memory_region_init(
+			send_buf, HF_PRIMARY_VM_ID, SERVICE_VM2,
+			&constituent_copy, 1, 0, 0, SPCI_MEMORY_RW_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_LEND),
+			SPCI_INVALID_PARAMETERS);
+		msg_size = spci_memory_region_init(
+			send_buf, HF_PRIMARY_VM_ID, SERVICE_VM2,
+			&constituent_copy, 1, 0, 0, SPCI_MEMORY_RW_X,
+			SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+			SPCI_MEMORY_OUTER_SHAREABLE);
+		EXPECT_SPCI_ERROR(
+			spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM2, msg_size,
+				      SPCI_MSG_SEND_LEGACY_MEMORY_SHARE),
+			SPCI_INVALID_PARAMETERS);
+	}
+
+	/* Return control to primary. */
+	spci_yield();
+}
diff --git a/test/vmapi/primary_with_secondaries/services/perfmon.c b/test/vmapi/primary_with_secondaries/services/perfmon.c
new file mode 100644
index 0000000..1e19ffb
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/perfmon.c
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/dlog.h"
+
+#include "../sysregs.h"
+#include "test/vmapi/exception_handler.h"
+
+TEST_SERVICE(perfmon_secondary_basic)
+{
+	exception_setup(NULL, exception_handler_skip_instruction);
+
+	EXPECT_GT(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+	TRY_READ(PMCCFILTR_EL0);
+	TRY_READ(PMCR_EL0);
+	write_msr(PMINTENSET_EL1, 0xf);
+
+	EXPECT_EQ(exception_handler_get_num(), 3);
+	spci_yield();
+}
diff --git a/test/vmapi/primary_with_secondaries/services/receive_block.c b/test/vmapi/primary_with_secondaries/services/receive_block.c
new file mode 100644
index 0000000..49ea5f2
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/receive_block.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/dlog.h"
+#include "hf/spci.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+/*
+ * Secondary VM that enables an interrupt, disables interrupts globally, and
+ * calls hf_mailbox_receive with block=true but expects it to fail.
+ */
+
+static void irq(void)
+{
+	uint32_t interrupt_id = hf_interrupt_get();
+	FAIL("Unexpected secondary IRQ %d from current", interrupt_id);
+}
+
+TEST_SERVICE(receive_block)
+{
+	int32_t i;
+	const char message[] = "Done waiting";
+
+	exception_setup(irq, NULL);
+	arch_irq_disable();
+	hf_interrupt_enable(EXTERNAL_INTERRUPT_ID_A, true);
+
+	for (i = 0; i < 10; ++i) {
+		struct spci_value res = spci_msg_wait();
+		EXPECT_SPCI_ERROR(res, SPCI_INTERRUPTED);
+	}
+
+	memcpy_s(SERVICE_SEND_BUFFER(), SPCI_MSG_PAYLOAD_MAX, message,
+		 sizeof(message));
+
+	spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, sizeof(message), 0);
+}
diff --git a/test/vmapi/primary_with_secondaries/services/relay.c b/test/vmapi/primary_with_secondaries/services/relay.c
new file mode 100644
index 0000000..7917251
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/relay.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2018 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "test/hftest.h"
+
+TEST_SERVICE(relay)
+{
+	/*
+	 * Loop, forward messages to the next VM.
+	 *
+	 * The first 32-bits of the message are the little-endian 32-bit ID of
+	 * the VM to forward the message to. This ID will be dropped from the
+	 * message so multiple IDs can be places at the start of the message.
+	 */
+	for (;;) {
+		spci_vm_id_t *chain;
+		spci_vm_id_t next_vm_id;
+		void *next_message;
+		uint32_t next_message_size;
+
+		/* Receive the message to relay. */
+		struct spci_value ret = spci_msg_wait();
+		ASSERT_EQ(ret.func, SPCI_MSG_SEND_32);
+
+		/* Prepare to relay the message. */
+		void *recv_buf = SERVICE_RECV_BUFFER();
+		void *send_buf = SERVICE_SEND_BUFFER();
+		ASSERT_GE(spci_msg_send_size(ret), sizeof(spci_vm_id_t));
+
+		chain = (spci_vm_id_t *)recv_buf;
+		next_vm_id = le16toh(*chain);
+		next_message = chain + 1;
+		next_message_size =
+			spci_msg_send_size(ret) - sizeof(spci_vm_id_t);
+
+		/* Send the message to the next stage. */
+		memcpy_s(send_buf, SPCI_MSG_PAYLOAD_MAX, next_message,
+			 next_message_size);
+
+		EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+		spci_msg_send(hf_vm_get_id(), next_vm_id, next_message_size, 0);
+	}
+}
diff --git a/test/vmapi/primary_with_secondaries/services/smp.c b/test/vmapi/primary_with_secondaries/services/smp.c
new file mode 100644
index 0000000..9030fb5
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/smp.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdalign.h>
+#include <stdint.h>
+
+#include "hf/arch/vm/power_mgmt.h"
+
+#include "hf/dlog.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+#include "vmapi/hf/spci.h"
+
+#include "../psci.h"
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+
+#define ARG_VALUE 42
+
+/*
+ * Secondary VM that starts a second vCPU and then sends messages from both.
+ */
+
+alignas(4096) static char stack[4096];
+
+/** Send a message back to the primary. */
+void send_message(const char *message, uint32_t size)
+{
+	memcpy_s(SERVICE_SEND_BUFFER(), SPCI_MSG_PAYLOAD_MAX, message, size);
+
+	ASSERT_EQ(spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, size, 0).func,
+		  SPCI_SUCCESS_32);
+}
+
+/**
+ * Entry point of the second vCPU.
+ */
+static void vm_cpu_entry(uintptr_t arg)
+{
+	ASSERT_EQ(arg, ARG_VALUE);
+
+	/* Check that vCPU statuses are as expected. */
+	ASSERT_EQ(arch_cpu_status(0), POWER_STATUS_ON);
+	ASSERT_EQ(arch_cpu_status(1), POWER_STATUS_ON);
+
+	dlog("Secondary second vCPU started.\n");
+	send_message("vCPU 1", sizeof("vCPU 1"));
+	dlog("Secondary second vCPU finishing\n");
+}
+
+TEST_SERVICE(smp)
+{
+	/* Check that vCPU statuses are as expected. */
+	ASSERT_EQ(arch_cpu_status(0), POWER_STATUS_ON);
+	ASSERT_EQ(arch_cpu_status(1), POWER_STATUS_OFF);
+
+	/* Start second vCPU. */
+	dlog("Secondary starting second vCPU.\n");
+	ASSERT_TRUE(hftest_cpu_start(1, stack, sizeof(stack), vm_cpu_entry,
+				     ARG_VALUE));
+	dlog("Secondary started second vCPU.\n");
+
+	/* Check that vCPU statuses are as expected. */
+	ASSERT_EQ(arch_cpu_status(0), POWER_STATUS_ON);
+	ASSERT_EQ(arch_cpu_status(1), POWER_STATUS_ON);
+
+	send_message("vCPU 0", sizeof("vCPU 0"));
+}
diff --git a/test/vmapi/primary_with_secondaries/services/spci_check.c b/test/vmapi/primary_with_secondaries/services/spci_check.c
new file mode 100644
index 0000000..9e342ba
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/spci_check.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/spci.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+TEST_SERVICE(spci_check)
+{
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	const char message[] = "spci_msg_send";
+
+	/* Wait for single message to be sent by the primary VM. */
+	struct spci_value ret = spci_msg_wait();
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+
+	/* Ensure message header has all fields correctly set. */
+	EXPECT_EQ(spci_msg_send_size(ret), sizeof(message));
+	EXPECT_EQ(spci_msg_send_receiver(ret), hf_vm_get_id());
+	EXPECT_EQ(spci_msg_send_sender(ret), HF_PRIMARY_VM_ID);
+
+	/* Ensure that the payload was correctly transmitted. */
+	EXPECT_EQ(memcmp(recv_buf, message, sizeof(message)), 0);
+
+	spci_yield();
+}
+
+TEST_SERVICE(spci_length)
+{
+	void *recv_buf = SERVICE_RECV_BUFFER();
+	const char message[] = "this should be truncated";
+
+	/* Wait for single message to be sent by the primary VM. */
+	struct spci_value ret = spci_msg_wait();
+
+	EXPECT_EQ(ret.func, SPCI_MSG_SEND_32);
+
+	/* Verify the length is as expected. */
+	EXPECT_EQ(16, spci_msg_send_size(ret));
+
+	/* Check only part of the message is sent correctly. */
+	EXPECT_NE(memcmp(recv_buf, message, sizeof(message)), 0);
+	EXPECT_EQ(memcmp(recv_buf, message, spci_msg_send_size(ret)), 0);
+
+	spci_yield();
+}
+
+TEST_SERVICE(spci_recv_non_blocking)
+{
+	/* Wait for single message to be sent by the primary VM. */
+	struct spci_value ret = spci_msg_poll();
+
+	EXPECT_SPCI_ERROR(ret, SPCI_RETRY);
+
+	spci_yield();
+}
diff --git a/test/vmapi/primary_with_secondaries/services/unmapped.c b/test/vmapi/primary_with_secondaries/services/unmapped.c
new file mode 100644
index 0000000..13d78c5
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/unmapped.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/mm.h"
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "../sysregs.h"
+#include "test/hftest.h"
+#include "test/vmapi/exception_handler.h"
+
+alignas(PAGE_SIZE) static uint8_t pages[2 * PAGE_SIZE];
+
+TEST_SERVICE(data_unmapped)
+{
+	/* Not using NULL so static analysis doesn't complain. */
+	int *p = (int *)1;
+	exception_setup(NULL, exception_handler_yield);
+	*p = 12;
+	FAIL("Exception not generated by invalid access.");
+}
+
+TEST_SERVICE(straddling_data_unmapped)
+{
+	void *send_buf = SERVICE_SEND_BUFFER();
+	/* Give some memory to the primary VM so that it's unmapped. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)(&pages[PAGE_SIZE]), .page_count = 1},
+	};
+	uint32_t msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+	exception_setup(NULL, exception_handler_yield);
+
+	EXPECT_EQ(spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	*(volatile uint64_t *)(&pages[PAGE_SIZE - 6]);
+	FAIL("Exception not generated by invalid access.");
+}
+
+TEST_SERVICE(instruction_unmapped)
+{
+	/* Not using NULL so static analysis doesn't complain. */
+	int (*f)(void) = (int (*)(void))4;
+	exception_setup(NULL, exception_handler_yield);
+	f();
+	FAIL("Exception not generated by invalid access.");
+}
+
+TEST_SERVICE(straddling_instruction_unmapped)
+{
+	void *send_buf = SERVICE_SEND_BUFFER();
+
+	/*
+	 * Get a function pointer which, when branched to, will attempt to
+	 * execute a 4-byte instruction straddling two pages.
+	 */
+	int (*f)(void) = (int (*)(void))(&pages[PAGE_SIZE - 2]);
+
+	/* Give second page to the primary VM so that it's unmapped. */
+	struct spci_memory_region_constituent constituents[] = {
+		{.address = (uint64_t)(&pages[PAGE_SIZE]), .page_count = 1},
+	};
+	uint32_t msg_size = spci_memory_region_init(
+		send_buf, hf_vm_get_id(), HF_PRIMARY_VM_ID, constituents,
+		ARRAY_SIZE(constituents), 0, 0, SPCI_MEMORY_RW_X,
+		SPCI_MEMORY_NORMAL_MEM, SPCI_MEMORY_CACHE_WRITE_BACK,
+		SPCI_MEMORY_OUTER_SHAREABLE);
+
+	exception_setup(NULL, exception_handler_yield);
+
+	EXPECT_EQ(spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, msg_size,
+				SPCI_MSG_SEND_LEGACY_MEMORY_DONATE)
+			  .func,
+		  SPCI_SUCCESS_32);
+
+	/* Branch to instruction whose 2 bytes are now in an unmapped page. */
+	f();
+	FAIL("Exception not generated by invalid access.");
+}
diff --git a/test/vmapi/primary_with_secondaries/services/wfi.c b/test/vmapi/primary_with_secondaries/services/wfi.c
new file mode 100644
index 0000000..6d61194
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/services/wfi.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/arch/irq.h"
+#include "hf/arch/vm/interrupts.h"
+
+#include "hf/dlog.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+
+/*
+ * Secondary VM that enables an interrupt, disables interrupts globally, and
+ * calls WFI.
+ */
+
+static void irq(void)
+{
+	uint32_t interrupt_id = hf_interrupt_get();
+	FAIL("Unexpected secondary IRQ %d from current", interrupt_id);
+}
+
+TEST_SERVICE(wfi)
+{
+	int32_t i;
+	const char message[] = "Done waiting";
+
+	exception_setup(irq, NULL);
+	arch_irq_disable();
+	hf_interrupt_enable(EXTERNAL_INTERRUPT_ID_A, true);
+
+	for (i = 0; i < 10; ++i) {
+		interrupt_wait();
+	}
+
+	memcpy_s(SERVICE_SEND_BUFFER(), SPCI_MSG_PAYLOAD_MAX, message,
+		 sizeof(message));
+
+	spci_msg_send(hf_vm_get_id(), HF_PRIMARY_VM_ID, sizeof(message), 0);
+}
diff --git a/test/vmapi/primary_with_secondaries/smp.c b/test/vmapi/primary_with_secondaries/smp.c
new file mode 100644
index 0000000..d76b5f0
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/smp.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+TEAR_DOWN(smp)
+{
+	EXPECT_SPCI_ERROR(spci_rx_release(), SPCI_DENIED);
+}
+
+/**
+ * Run a service that starts a second vCPU, and check that both the first and
+ * second vCPU send messages to us.
+ */
+TEST(smp, two_vcpus)
+{
+	const char expected_response_0[] = "vCPU 0";
+	const char expected_response_1[] = "vCPU 1";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM3, "smp", mb.send);
+
+	/* Let the first vCPU start the second vCPU. */
+	run_res = spci_run(SERVICE_VM3, 0);
+	EXPECT_EQ(run_res.func, HF_SPCI_RUN_WAKE_UP);
+	EXPECT_EQ(spci_vm_id(run_res), SERVICE_VM3);
+	EXPECT_EQ(spci_vcpu_index(run_res), 1);
+
+	/* Run the second vCPU and wait for a message. */
+	dlog("Run second vCPU for message\n");
+	run_res = spci_run(SERVICE_VM3, 1);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response_1));
+	EXPECT_EQ(memcmp(mb.recv, expected_response_1,
+			 sizeof(expected_response_1)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Run the first vCPU and wait for a different message. */
+	dlog("Run first vCPU for message\n");
+	run_res = spci_run(SERVICE_VM3, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_msg_send_size(run_res), sizeof(expected_response_0));
+	EXPECT_EQ(memcmp(mb.recv, expected_response_0,
+			 sizeof(expected_response_0)),
+		  0);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	/* Run the second vCPU again, and expect it to turn itself off. */
+	dlog("Run second vCPU for poweroff.\n");
+	run_res = spci_run(SERVICE_VM3, 1);
+	EXPECT_EQ(run_res.func, HF_SPCI_RUN_WAIT_FOR_INTERRUPT);
+	EXPECT_EQ(run_res.arg2, SPCI_SLEEP_INDEFINITE);
+}
diff --git a/test/vmapi/primary_with_secondaries/spci.c b/test/vmapi/primary_with_secondaries/spci.c
new file mode 100644
index 0000000..6ffc85f
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/spci.c
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/spci.h"
+
+#include <stdint.h>
+
+#include "hf/std.h"
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/spci.h"
+
+/**
+ * Send a message to a secondary VM which checks the validity of the received
+ * header.
+ */
+TEST(spci, msg_send)
+{
+	const char message[] = "spci_msg_send";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_check", mb.send);
+
+	/* Set the payload, init the message header and send the message. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_EQ(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, sizeof(message), 0)
+			.func,
+		SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Send a message to a secondary VM spoofing the source VM id.
+ */
+TEST(spci, msg_send_spoof)
+{
+	const char message[] = "spci_msg_send";
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_check", mb.send);
+
+	/* Set the payload, init the message header and send the message. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(SERVICE_VM2, SERVICE_VM1, sizeof(message), 0),
+		SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Send a message to a secondary VM with incorrect destination id.
+ */
+TEST(spci, spci_invalid_destination_id)
+{
+	const char message[] = "fail to send";
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_check", mb.send);
+	/* Set the payload, init the message header and send the message. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(HF_PRIMARY_VM_ID, -1, sizeof(message), 0),
+		SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Ensure that the length parameter is respected when sending messages.
+ */
+TEST(spci, spci_incorrect_length)
+{
+	const char message[] = "this should be truncated";
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "spci_length", mb.send);
+
+	/* Send the message and compare if truncated. */
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	/* Hard code incorrect length. */
+	EXPECT_EQ(spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, 16, 0).func,
+		  SPCI_SUCCESS_32);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
+
+/**
+ * Attempt to send a message larger than what is supported.
+ */
+TEST(spci, spci_large_message)
+{
+	const char message[] = "fail to send";
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	memcpy_s(mb.send, SPCI_MSG_PAYLOAD_MAX, message, sizeof(message));
+	/* Send a message that is larger than the mailbox supports (4KB). */
+	EXPECT_SPCI_ERROR(
+		spci_msg_send(HF_PRIMARY_VM_ID, SERVICE_VM1, 4 * 1024 + 1, 0),
+		SPCI_INVALID_PARAMETERS);
+}
+
+/**
+ * Verify secondary VM non blocking recv.
+ */
+TEST(spci, spci_recv_non_blocking)
+{
+	struct mailbox_buffers mb = set_up_mailbox();
+	struct spci_value run_res;
+
+	/* Check is performed in secondary VM. */
+	SERVICE_SELECT(SERVICE_VM1, "spci_recv_non_blocking", mb.send);
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_YIELD_32);
+}
diff --git a/test/vmapi/primary_with_secondaries/sysregs.c b/test/vmapi/primary_with_secondaries/sysregs.c
new file mode 100644
index 0000000..42f0636
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/sysregs.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "sysregs.h"
+
+#include "hf/arch/vm/interrupts.h"
+
+#include "primary_with_secondary.h"
+#include "test/vmapi/exception_handler.h"
+#include "test/vmapi/spci.h"
+
+SET_UP(sysregs)
+{
+	exception_setup(NULL, exception_handler_skip_instruction);
+}
+
+/**
+ * Test that accessing LOR registers would inject an exception.
+ */
+TEST(sysregs, lor_exception)
+{
+	EXPECT_EQ(hf_vm_get_id(), HF_PRIMARY_VM_ID);
+	TRY_READ(MSR_LORC_EL1);
+
+	EXPECT_EQ(exception_handler_get_num(), 1);
+}
diff --git a/test/vmapi/primary_with_secondaries/sysregs.h b/test/vmapi/primary_with_secondaries/sysregs.h
new file mode 100644
index 0000000..50faab4
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/sysregs.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "vmapi/hf/call.h"
+
+#include "../msr.h"
+#include "test/hftest.h"
+
+#define TRY_READ(REG) dlog(#REG "=%#x\n", read_msr(REG))
+
+#define CHECK_READ(REG, VALUE)       \
+	do {                         \
+		uintreg_t x;         \
+		x = read_msr(REG);   \
+		EXPECT_EQ(x, VALUE); \
+	} while (0)
+
+#define TRY_WRITE_READ(REG, VALUE)     \
+	do {                           \
+		uintreg_t x;           \
+		x = read_msr(REG);     \
+		EXPECT_NE(x, VALUE);   \
+		write_msr(REG, VALUE); \
+		x = read_msr(REG);     \
+		EXPECT_EQ(x, VALUE);   \
+	} while (0)
diff --git a/test/vmapi/primary_with_secondaries/unmapped.c b/test/vmapi/primary_with_secondaries/unmapped.c
new file mode 100644
index 0000000..0b9c116
--- /dev/null
+++ b/test/vmapi/primary_with_secondaries/unmapped.c
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "vmapi/hf/call.h"
+
+#include "primary_with_secondary.h"
+#include "test/hftest.h"
+#include "test/vmapi/exception_handler.h"
+#include "test/vmapi/spci.h"
+
+/**
+ * Accessing unmapped memory traps the VM.
+ */
+TEST(unmapped, data_unmapped)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "data_unmapped", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Accessing partially unmapped memory traps the VM.
+ */
+TEST(unmapped, straddling_data_unmapped)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "straddling_data_unmapped", mb.send);
+
+	/*
+	 * First we get a message about the memory being donated to us, then we
+	 * get the trap.
+	 */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Executing unmapped memory traps the VM.
+ */
+TEST(unmapped, instruction_unmapped)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "instruction_unmapped", mb.send);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
+
+/**
+ * Executing partially unmapped memory traps the VM.
+ */
+TEST(unmapped, straddling_instruction_unmapped)
+{
+	struct spci_value run_res;
+	struct mailbox_buffers mb = set_up_mailbox();
+
+	SERVICE_SELECT(SERVICE_VM1, "straddling_instruction_unmapped", mb.send);
+
+	/*
+	 * First we get a message about the memory being donated to us, then we
+	 * get the trap.
+	 */
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(run_res.func, SPCI_MSG_SEND_32);
+	EXPECT_EQ(spci_rx_release().func, SPCI_SUCCESS_32);
+
+	run_res = spci_run(SERVICE_VM1, 0);
+	EXPECT_EQ(exception_handler_receive_exception_count(&run_res, mb.recv),
+		  1);
+}
diff --git a/third_party/googletest b/third_party/googletest
new file mode 160000
index 0000000..90435b5
--- /dev/null
+++ b/third_party/googletest
@@ -0,0 +1 @@
+Subproject commit 90435b5c42b2492a7105c56bd36505b190ce54be
diff --git a/third_party/linux b/third_party/linux
new file mode 160000
index 0000000..0f672f6
--- /dev/null
+++ b/third_party/linux
@@ -0,0 +1 @@
+Subproject commit 0f672f6c0b52b7b0700b0915c72b540721af4465
diff --git a/vmlib/BUILD.gn b/vmlib/BUILD.gn
new file mode 100644
index 0000000..9ba8c3d
--- /dev/null
+++ b/vmlib/BUILD.gn
@@ -0,0 +1,21 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/platform.gni")
+
+source_set("vmlib") {
+  sources = [
+    "spci.c",
+  ]
+}
diff --git a/vmlib/aarch64/BUILD.gn b/vmlib/aarch64/BUILD.gn
new file mode 100644
index 0000000..4ddd659
--- /dev/null
+++ b/vmlib/aarch64/BUILD.gn
@@ -0,0 +1,21 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build/toolchain/platform.gni")
+
+source_set("call") {
+  sources = [
+    "call.c",
+  ]
+}
diff --git a/vmlib/aarch64/call.c b/vmlib/aarch64/call.c
new file mode 100644
index 0000000..0c1a078
--- /dev/null
+++ b/vmlib/aarch64/call.c
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/call.h"
+
+#include "hf/spci.h"
+#include "hf/types.h"
+
+int64_t hf_call(uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3)
+{
+	register uint64_t r0 __asm__("x0") = arg0;
+	register uint64_t r1 __asm__("x1") = arg1;
+	register uint64_t r2 __asm__("x2") = arg2;
+	register uint64_t r3 __asm__("x3") = arg3;
+
+	__asm__ volatile(
+		"hvc #0"
+		: /* Output registers, also used as inputs ('+' constraint). */
+		"+r"(r0), "+r"(r1), "+r"(r2), "+r"(r3)
+		:
+		: /* Clobber registers. */
+		"x4", "x5", "x6", "x7");
+
+	return r0;
+}
+
+struct spci_value spci_call(struct spci_value args)
+{
+	register uint64_t r0 __asm__("x0") = args.func;
+	register uint64_t r1 __asm__("x1") = args.arg1;
+	register uint64_t r2 __asm__("x2") = args.arg2;
+	register uint64_t r3 __asm__("x3") = args.arg3;
+	register uint64_t r4 __asm__("x4") = args.arg4;
+	register uint64_t r5 __asm__("x5") = args.arg5;
+	register uint64_t r6 __asm__("x6") = args.arg6;
+	register uint64_t r7 __asm__("x7") = args.arg7;
+
+	__asm__ volatile(
+		"hvc #0"
+		: /* Output registers, also used as inputs ('+' constraint). */
+		"+r"(r0), "+r"(r1), "+r"(r2), "+r"(r3), "+r"(r4), "+r"(r5),
+		"+r"(r6), "+r"(r7));
+
+	return (struct spci_value){.func = r0,
+				   .arg1 = r1,
+				   .arg2 = r2,
+				   .arg3 = r3,
+				   .arg4 = r4,
+				   .arg5 = r5,
+				   .arg6 = r6,
+				   .arg7 = r7};
+}
diff --git a/vmlib/spci.c b/vmlib/spci.c
new file mode 100644
index 0000000..c6f745a
--- /dev/null
+++ b/vmlib/spci.c
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019 The Hafnium Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hf/spci.h"
+
+#include "hf/types.h"
+
+/*
+ * Copied from hf/arch/std.h because we can't include Hafnium internal headers
+ * here.
+ */
+#ifndef align_up
+#define align_up(v, a) (((uintptr_t)(v) + ((a)-1)) & ~((a)-1))
+#endif
+
+/**
+ * Initialises the given `spci_memory_region` and copies the constituent
+ * information to it. Returns the length in bytes occupied by the data copied to
+ * `memory_region` (attributes, constituents and memory region header size).
+ */
+uint32_t spci_memory_region_init(
+	struct spci_memory_region *memory_region, spci_vm_id_t sender, spci_vm_id_t receiver,
+	const struct spci_memory_region_constituent constituents[],
+	uint32_t constituent_count, uint32_t tag, spci_memory_region_flags_t flags,
+	enum spci_memory_access access, enum spci_memory_type type,
+	enum spci_memory_cacheability cacheability,
+	enum spci_memory_shareability shareability)
+{
+	uint32_t constituents_length =
+		constituent_count *
+		sizeof(struct spci_memory_region_constituent);
+	uint32_t index;
+	struct spci_memory_region_constituent *region_constituents;
+	uint16_t attributes = 0;
+
+	/* Set memory region's page attributes. */
+	spci_set_memory_access_attr(&attributes, access);
+	spci_set_memory_type_attr(&attributes, type);
+	spci_set_memory_cacheability_attr(&attributes, cacheability);
+	spci_set_memory_shareability_attr(&attributes, shareability);
+
+	memory_region->tag = tag;
+	memory_region->flags = flags;
+	memory_region->sender = sender;
+	memory_region->reserved = 0;
+	memory_region->page_count = 0;
+	memory_region->constituent_count = constituent_count;
+	memory_region->attribute_count = 1;
+	memory_region->attributes[0].receiver = receiver;
+	memory_region->attributes[0].memory_attributes = attributes;
+
+	/*
+	 * Constituent offset must be aligned to a 64-bit boundary so that
+	 * 64-bit addresses can be copied without alignment faults.
+	 */
+	memory_region->constituent_offset = align_up(
+		sizeof(struct spci_memory_region) +
+			memory_region->attribute_count *
+				sizeof(struct spci_memory_region_attributes),
+		8);
+	region_constituents =
+		spci_memory_region_get_constituents(memory_region);
+
+	for (index = 0; index < constituent_count; index++) {
+		region_constituents[index] = constituents[index];
+		region_constituents[index].reserved = 0;
+		memory_region->page_count += constituents[index].page_count;
+	}
+
+	/*
+	 * TODO: Add assert ensuring that the specified message
+	 * length is not greater than SPCI_MSG_PAYLOAD_MAX.
+	 */
+
+	return memory_region->constituent_offset + constituents_length;
+}