Hermetic builds inside a container
Adds 'build/docker/Dockerfile' which describes the base container image of
Hafnium compilation environment. This image is built and uploaded to GCP
where users download it from. The feature is always enabled for Kokoro
and can optionally be enabled for local builds too. Once rootless
containers are easier to set up, we might make it the default for local
builds too.
An arbitrary command can be executed inside the container with
'build/run_in_container.sh [-i] <command> ...'. This is done
automatically inside 'Makefile' and 'kokoro/ubuntu/build.sh' which
detect whether they are already running inside the container and respawn
themselves using 'run_in_container.sh' if not.
The feature is guarded with HAFNIUM_HERMETIC_BUILD environment variable,
switched on if the value is "true". All other values switch it off, e.g.
'run_in_container.sh' sets it to 'inside' to avoid recursion.
Bug: 132428451
Test: HAFNIUM_HERMETIC_BUILD=<value> make
Test: HAFNIUM_HERMETIC_BUILD=<value> kokoro/ubuntu/build.sh
Change-Id: I0737a868ab4f67c0fdbf78fa8a97cc91714d2e10
diff --git a/Makefile b/Makefile
index 619ac1c..4d7b384 100644
--- a/Makefile
+++ b/Makefile
@@ -12,6 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# If HAFNIUM_HERMETIC_BUILD is "true" (not default), invoke `make` inside
+# a container. The 'run_in_container.sh' script will set the variable value to
+# 'inside' to avoid recursion.
+ifeq ($(HAFNIUM_HERMETIC_BUILD),true)
+
+# TODO: This is not ideal as (a) we invoke the container once per command-line
+# target, and (b) we cannot pass `make` arguments to the script. We could
+# consider creating a bash alias for `make` to invoke the script directly.
+
+# Need to define at least one non-default target.
+all:
+ @$(PWD)/build/run_in_container.sh make $@
+
+# Catch-all target.
+.DEFAULT:
+ @$(PWD)/build/run_in_container.sh make $@
+
+else # HAFNIUM_HERMETIC_BUILD
+
# Set path to prebuilts used in the build.
UNNAME_S := $(shell uname -s | tr '[:upper:]' '[:lower:]')
PREBUILTS := $(PWD)/prebuilts/$(UNNAME_S)-x64
@@ -93,3 +112,5 @@
prebuilts/linux-aarch64/linux/vmlinuz: $(OUT_DIR)/build.ninja
@$(NINJA) -C $(OUT_DIR) "third_party:linux"
cp out/reference/obj/third_party/linux.bin $@
+
+endif # HAFNIUM_HERMETIC_BUILD
diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
new file mode 100644
index 0000000..93c8caa
--- /dev/null
+++ b/build/docker/Dockerfile
@@ -0,0 +1,43 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Base container image to be uploaded to Google Cloud Platform as
+# "eu.gcr.io/hafnium-build/hafnium_ci". Each user derives their own container
+# with local user permissions from this base image. It should contain everything
+# needed to build and test Hafnium.
+#
+FROM launcher.gcr.io/google/ubuntu1804
+MAINTAINER Hafnium Team <hafnium-team+build@google.com>
+
+# Install dependencies. Clear APT cache at the end to save space.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+ && apt-get install -y \
+ bc `# for Linux headers` \
+ binutils-aarch64-linux-gnu \
+ bison \
+ build-essential \
+ cpio \
+ device-tree-compiler \
+ flex \
+ git \
+ libpixman-1-0 `# for QEMU` \
+ libsdl2-2.0-0 `# for QEMU` \
+ libglib2.0 `# for QEMU` \
+ libssl-dev `# for Linux headers` \
+ python \
+ python-git `# for Linux checkpatch` \
+ python-ply `# for Linux checkpatch` \
+ && rm -rf /var/lib/apt/lists/*
diff --git a/build/docker/Dockerfile.local b/build/docker/Dockerfile.local
new file mode 100644
index 0000000..67eb92f
--- /dev/null
+++ b/build/docker/Dockerfile.local
@@ -0,0 +1,35 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Container derived from the base image hosted on Google Cloud Platform.
+# It sets up a user with the same UID/GID as the local user, so that generated
+# files can be accessed by the host.
+# Please keep the diff between base and local images as small as possible.
+#
+FROM eu.gcr.io/hafnium-build/hafnium_ci
+ARG LOCAL_UID=1000
+ARG LOCAL_GID=1000
+
+RUN addgroup \
+ --gid "${LOCAL_GID}" \
+ hafnium \
+ && adduser \
+ -disabled-password \
+ -gecos "" \
+ --uid "${LOCAL_UID}" \
+ --shell "/bin/bash" \
+ --ingroup hafnium \
+ hafnium
+USER hafnium
\ No newline at end of file
diff --git a/build/docker/build.sh b/build/docker/build.sh
new file mode 100755
index 0000000..5972038
--- /dev/null
+++ b/build/docker/build.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -euo pipefail
+
+SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+source "${SCRIPT_DIR}/common.inc"
+
+${DOCKER} build \
+ --pull \
+ -f "${SCRIPT_DIR}/Dockerfile" \
+ -t "${CONTAINER_TAG}" \
+ "${SCRIPT_DIR}"
diff --git a/build/docker/common.inc b/build/docker/common.inc
new file mode 100644
index 0000000..0d1e1db
--- /dev/null
+++ b/build/docker/common.inc
@@ -0,0 +1,21 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CONTAINER_TAG="eu.gcr.io/hafnium-build/hafnium_ci"
+
+if [[ ! -v DOCKER ]]
+then
+ DOCKER="$(which docker)" \
+ || (echo "ERROR: Could not find Docker binary" 1>&2; exit 1)
+fi
\ No newline at end of file
diff --git a/build/docker/publish.sh b/build/docker/publish.sh
new file mode 100755
index 0000000..96ec2f1
--- /dev/null
+++ b/build/docker/publish.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -euo pipefail
+
+SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+source "${SCRIPT_DIR}/common.inc"
+
+# Requires for the user to be an owner of the GCP 'hafnium-build' project and
+# have gcloud SDK installed and authenticated.
+
+${DOCKER} push "${CONTAINER_TAG}"
diff --git a/build/run_in_container.sh b/build/run_in_container.sh
new file mode 100755
index 0000000..e3c6bd0
--- /dev/null
+++ b/build/run_in_container.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -euo pipefail
+
+SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+ROOT_DIR="$(realpath ${SCRIPT_DIR}/..)"
+
+source "${SCRIPT_DIR}/docker/common.inc"
+
+if [ "${HAFNIUM_HERMETIC_BUILD:-}" == "inside" ]
+then
+ echo "ERROR: Invoked $0 recursively" 1>&2
+ exit 1
+fi
+
+# Set up a temp directory and register a cleanup function on exit.
+TMP_DIR="$(mktemp -d)"
+function cleanup() {
+ rm -rf "${TMP_DIR}"
+}
+trap cleanup EXIT
+
+# Build local image and write its hash to a temporary file.
+IID_FILE="${TMP_DIR}/imgid.txt"
+"${DOCKER}" build \
+ --build-arg LOCAL_UID="$(id -u)" \
+ --build-arg LOCAL_GID="$(id -g)" \
+ --iidfile="${IID_FILE}" \
+ -f "${SCRIPT_DIR}/docker/Dockerfile.local" \
+ "${SCRIPT_DIR}/docker"
+IMAGE_ID="$(cat ${IID_FILE})"
+
+# Check if script was invoked with '-i' as first argument. If so, run
+# container in interactive mode.
+INTERACTIVE=false
+if [ "${1:-}" == "-i" ]
+then
+ INTERACTIVE=true
+ shift
+fi
+
+ARGS=()
+# Run with a pseduo-TTY for nicer logging.
+ARGS+=(-t)
+# Run interactive if this script was invoked with '-i'.
+if [ "${INTERACTIVE}" == "true" ]
+then
+ ARGS+=(-i)
+fi
+# Set environment variable informing the build that we are running inside
+# a container.
+ARGS+=(-e HAFNIUM_HERMETIC_BUILD=inside)
+# Bind-mount the Hafnium root directory. We mount it at the same absolute
+# location so that all paths match across the host and guest.
+ARGS+=(-v "${ROOT_DIR}":"${ROOT_DIR}")
+# Make all files outside of the Hafnium directory read-only to ensure that all
+# generated files are written there.
+ARGS+=(--read-only)
+# Mount a writable /tmp folder. Required by LLVM/Clang for intermediate files.
+ARGS+=(--tmpfs /tmp)
+# Set working directory.
+ARGS+=(-w "${ROOT_DIR}")
+
+echo "Running in container: $*" 1>&2
+${DOCKER} run \
+ ${ARGS[@]} \
+ "${IMAGE_ID}" \
+ /bin/bash -c "$*"
\ No newline at end of file
diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md
index 7182544..eae6a16 100644
--- a/docs/GettingStarted.md
+++ b/docs/GettingStarted.md
@@ -35,8 +35,8 @@
make PROJECT=<project_name>
```
-The compiled image can be found under `out/<project>`, for example the QEMU image is at
-at `out/reference/qemu_aarch64_clang/hafnium.bin`.
+The compiled image can be found under `out/<project>`, for example the QEMU
+image is at `out/reference/qemu_aarch64_clang/hafnium.bin`.
## Running on QEMU
diff --git a/docs/HermeticBuild.md b/docs/HermeticBuild.md
new file mode 100644
index 0000000..ca1026e
--- /dev/null
+++ b/docs/HermeticBuild.md
@@ -0,0 +1,87 @@
+# Hermetic build
+
+Hafnium build is not hermetic as it uses some system tools and libraries, e.g.
+`bison` and `libssl`. To ensure consistency and repeatability, the team
+maintains and periodically publishes a container image as the reference build
+environment. The image is hosted on Google Cloud Platform as
+`eu.gcr.io/hafnium-build/hafnium_ci`.
+
+Building inside a container is always enabled only for Kokoro pre-submit tests
+but can be enabled for local builds too. It is disabled by default as it
+requires the use of Docker which currently supports rootless containers only
+in nightly builds. As rootless container tools mature, Hafnium may change the
+default settings. For now, running the hermetic build locally is intended
+primarily to reproduce issues in pre-submit tests.
+
+## Installing Docker
+
+### Stable
+
+If you don't mind running a Docker daemon with root privileges on your system,
+you can follow the [official guide](https://docs.docker.com/install/) to install
+Docker, or [go/installdocker](https://goto.google.com/installdocker) if you are
+a Googler.
+
+Because the daemon runs as root, files generated by the container are owned by
+root as well. To work around this, the build will automatically derive a local
+container image from the base container, adding user `hafnium` with the same
+UID/GID as the local user.
+
+### Nightly with rootless
+
+The latest nightly version of Docker has support for running containers with
+user namespaces, thus eliminating the need for a daemon with root privileges.
+It can be installed into the local user's `bin` directory with a script:
+``` shell
+curl -fsSL https://get.docker.com/rootless -o get-docker.sh
+sh get-docker.sh
+```
+
+The script will also walk you through the installation of dependencies,
+changes to system configuration files and environment variable values needed
+by the client to discover the rootless daemon.
+
+## Enabling for local builds
+
+Hermetic builds are controlled by the `HAFNIUM_HERMETIC_BUILD` environment
+variable. Setting it to `true` instructs the build to run commands inside the
+container. Any other value disables the feature.
+
+To always enable hermetic builds, put this line in your `~/.bashrc`:
+``` shell
+export HAFNIUM_HERMETIC_BUILD=true
+```
+
+When you now run `make`, you should see the following line:
+``` shell
+$ make
+Running in container: make all
+...
+```
+## Running commands inside the container
+
+An arbitrary command can be executed inside the container with
+`build/run_in_container.sh [-i] <command> ...`. This is done
+automatically inside `Makefile` and `kokoro/ubuntu/build.sh` which
+detect whether they are already running inside the container and respawn
+themselves using `run_in_container.sh` if not.
+
+For example, you can spawn a shell with:
+``` shell
+./build/run_in_container.sh -i bash
+```
+
+## Building container image
+
+The container image is defined in `build/docker/Dockerfile` and can be built
+locally:
+``` shell
+./build/docker/build.sh
+```
+
+Owners of the `hafnium-build` GCP repository can publish the new image
+(requires [go/cloud-sdk](https://goto.google.com/cloud-sdk) installed and
+authenticated):
+``` shell
+./build/docker/publish.sh
+```
diff --git a/kokoro/ubuntu/build.sh b/kokoro/ubuntu/build.sh
index 42bd693..662fe98 100755
--- a/kokoro/ubuntu/build.sh
+++ b/kokoro/ubuntu/build.sh
@@ -14,6 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+SCRIPT_NAME="$(realpath "${BASH_SOURCE[0]}")"
+ROOT_DIR="$(realpath $(dirname "${SCRIPT_NAME}")/../..)"
+
# Fail on any error.
set -e
# Fail on any part of a pipeline failing.
@@ -23,6 +26,20 @@
# Display commands being run.
set -x
+# Default value of HAFNIUM_HERMETIC_BUILD is "true" for Kokoro builds.
+if [ -v KOKORO_JOB_NAME -a ! -v HAFNIUM_HERMETIC_BUILD ]
+then
+ HAFNIUM_HERMETIC_BUILD=true
+fi
+
+# If HAFNIUM_HERMETIC_BUILD is "true" (not default), relaunch this script inside
+# a container. The 'run_in_container.sh' script will set the variable value to
+# 'inside' to avoid recursion.
+if [ "${HAFNIUM_HERMETIC_BUILD:-}" == "true" ]
+then
+ exec "${ROOT_DIR}/build/run_in_container.sh" ${SCRIPT_NAME} $@
+fi
+
USE_FVP=0
while test $# -gt 0
@@ -43,9 +60,6 @@
then
# Server
cd git/hafnium
-else
- # Local
- echo "Testing kokoro build locally..."
fi
CLANG=${PWD}/prebuilts/linux-x64/clang/bin/clang
