Update build scripts to work under Kokoro+Repo
Kokoro does not support <linkfile> tags. That means two workarounds are
needed:
(a) build.sh is called as "./core/kokoro" (not "./kokoro") and therefore
must adjust ROOT_DIR location - done by checking for presence of
"../.repo_manifest" folder
(b) build.sh must first symlink files specified in <linkfile> tags of
the manifest - done by a Python script
Move init logic like this to a .inc file that can be shared between
build scripts for different projects.
Change-Id: I4d632c7200d34d08981e58d290883bd2cd9e5a04
diff --git a/build/bash/common.inc b/build/bash/common.inc
new file mode 100644
index 0000000..4d7bd73
--- /dev/null
+++ b/build/bash/common.inc
@@ -0,0 +1,130 @@
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail on any error.
+set -e
+# Fail on any part of a pipeline failing.
+set -o pipefail
+# Treat unset variables as an error.
+set -u
+# Display commands being run.
+set -x
+
+# Returns absolute path to a file or a directory. The path must exist.
+function abs_path() {
+ local rel_path="$1"
+ if [ -d "$rel_path" ]
+ then
+ # Parameter is a directory
+ local rel_dir="$rel_path"
+ local rel_base=""
+ elif [ -f "$rel_path" ]
+ then
+ # Parameter is a regular file
+ local rel_dir="$(dirname "$rel_path")"
+ local rel_base="/$(basename "$rel_path")"
+ else
+ # File does not exist
+ echo "File not found: $rel_path" >&2
+ exit 1
+ fi
+ echo "$(cd $rel_dir && pwd)$rel_base"
+ return 0
+}
+
+# Returns true if the environment contains Kokoro build variables.
+function is_kokoro_build() {
+ [ -v KOKORO_JOB_NAME ]
+ return $?
+}
+
+# Returns true if the `.repo_manifest` folder exists. The folder should contain
+# the manifest and be present in all Repo builds. Eventually this should be
+# obsolete as we switch exclusively to Repo.
+function is_repo_build() {
+ [ -d "${ROOT_DIR}/.repo_manifest" ]
+ return $?
+}
+
+# Returns absolute path to the source file that called this function.
+function get_script_path() {
+ abs_path "${BASH_SOURCE[1]}"
+}
+
+# Returns absolute path to the directory containing the source file that called
+# this function.
+function get_script_dir() {
+ local caller_path="${BASH_SOURCE[1]}"
+ local caller_abs_path="$(abs_path $caller_path)"
+ dirname "$caller_abs_path"
+}
+
+# Assigns value (second arg) of a variable (first arg) if it is not set already.
+function default_value {
+ local var_name=$1
+ local value=$2
+ export ${var_name}=${!var_name:-${value}}
+}
+
+# Returns true if `git status` reports uncommitted changes in the source tree.
+# Runs on all projects if Repo is detected.
+function is_repo_dirty() {
+ local cmd=(git status --porcelain=v1)
+ if is_repo_build
+ then
+ # This source tree was checked out using `repo`. Check the
+ # status of all projects.
+ cmd=(${REPO} forall -c "${cmd[@]}")
+ fi
+ ! (u="$(${cmd[@]})" && test -z "$u")
+ return $?
+}
+
+# Prepares the source directory for building. This includes setting global
+# variables and workaronuds for different build environments.
+function init_build() {
+ ##
+ ## Find Hafnium's root directory.
+ ##
+ ROOT_DIR="$(abs_path $(get_script_dir)/../..)"
+ # Temporary workaround for Repo builds. Check if there is a project
+ # folder specific to Repo builds in a parent directory. If it is, the
+ # root is further one level up.
+ if [ -d "${ROOT_DIR}/../.repo_manifest" ]
+ then
+ ROOT_DIR="$(dirname ${ROOT_DIR})"
+ fi
+
+ ##
+ ## Paths to tools
+ ##
+ CLANG="${ROOT_DIR}/prebuilts/linux-x64/clang/bin/clang"
+ REPO="${ROOT_DIR}/prebuilts/generic/repo/repo"
+
+ ##
+ ## Workarounds for Kokoro+Repo builds.
+ ##
+ if is_kokoro_build && is_repo_build
+ then
+ # Kokoro does not correctly initialize the `.repo` folder which
+ # causes `is_repo_dirty` checks to fail. We check out the
+ # manifest as one of the projects and use it to initialize the
+ # `.repo` folder here.
+ (cd "${ROOT_DIR}/.repo_manifest" && git branch master)
+ (cd "${ROOT_DIR}" && "${REPO}" init -u .repo_manifest)
+ # Kokoro does not support '<linkfile>' manifest tags. Parse the
+ # manifest and symlink files here.
+ "$(get_script_dir)/symlink_repo.py" "${ROOT_DIR}"
+ fi
+}
diff --git a/build/bash/symlink_repo.py b/build/bash/symlink_repo.py
new file mode 100755
index 0000000..eefa46b
--- /dev/null
+++ b/build/bash/symlink_repo.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Hafnium Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Parse Repo manifest and symlink files specified in <linkfile> tags.
+
+This is a workaround for Kokoro which does not support <linkfile>.
+"""
+
+import argparse
+import os
+import sys
+import xml.etree.ElementTree as ET
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("root_dir", help="root directory")
+ args = parser.parse_args()
+
+ manifest = os.path.join(args.root_dir, ".repo", "manifest.xml")
+ tree = ET.parse(manifest)
+ root = tree.getroot()
+ assert(root.tag == "manifest");
+
+ for proj in root:
+ if proj.tag != "project":
+ continue
+
+ proj_name = proj.attrib["name"]
+ proj_path = proj.attrib["path"]
+
+ for linkfile in proj:
+ if linkfile.tag != "linkfile":
+ continue
+
+ linkfile_src = linkfile.attrib["src"]
+ linkfile_dest = linkfile.attrib["dest"]
+ src_path = os.path.join(
+ args.root_dir, proj_path, linkfile_src)
+ dest_path = os.path.join(args.root_dir, linkfile_dest)
+
+ os.symlink(src_path, dest_path)
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/kokoro/build.sh b/kokoro/build.sh
index 4c65e41..db81d6e 100755
--- a/kokoro/build.sh
+++ b/kokoro/build.sh
@@ -14,43 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SCRIPT_PATH="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")"
-ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
+source "$(dirname ${BASH_SOURCE[0]})/../build/bash/common.inc"
-REPO="${ROOT_DIR}/prebuilts/generic/repo/repo"
-
-# Fail on any error.
-set -e
-# Fail on any part of a pipeline failing.
-set -o pipefail
-# Treat unset variables as an error.
-set -u
-# Display commands being run.
-set -x
-
-# Returns true if `git status` reports uncommitted changes in the source tree.
-function is_repo_dirty() {
- local cmd=(git status --porcelain=v1)
- if [ -d ".repo" ]
- then
- # This source tree was checked out using `repo`. Check the
- # status of all projects.
- cmd=(${REPO} forall -c "${cmd[@]}")
- fi
- ! (u="$(${cmd[@]})" && test -z "$u")
- return $?
-}
-
-# Assigns value (second arg) of a variable (first arg) if it is not set already.
-function default_value {
- local var_name=$1
- local value=$2
- export ${var_name}=${!var_name:-${value}}
-}
+# Initialize global variables, prepare repo for building.
+init_build
# Assign default values to variables.
-if [ -v KOKORO_JOB_NAME ]
+if is_kokoro_build
then
# Default config for Kokoro builds.
default_value HAFNIUM_HERMETIC_BUILD true
@@ -68,7 +38,7 @@
# avoid recursion.
if [ "${HAFNIUM_HERMETIC_BUILD}" == "true" ]
then
- exec "${ROOT_DIR}/build/run_in_container.sh" ${SCRIPT_PATH} $@
+ exec "${ROOT_DIR}/build/run_in_container.sh" "$(get_script_path)" $@
fi
USE_FVP=false
@@ -93,8 +63,6 @@
shift
done
-CLANG=${PWD}/prebuilts/linux-x64/clang/bin/clang
-
# Kokoro does something weird that makes all files look dirty to git diff-index;
# this fixes it so that the Linux build doesn't think it has a dirty tree for
# building the Hafnium kernel module (and so end up with a version magic string
@@ -184,5 +152,5 @@
export ARCH=arm64 &&
export CROSS_COMPILE=aarch64-linux-gnu- &&
cd driver/linux &&
-make checkpatch
+make HAFNIUM_PATH="${ROOT_DIR}" checkpatch
)
diff --git a/kokoro/public/presubmit.cfg b/kokoro/public/presubmit.cfg
index 9bb14aa..d71b9c7 100644
--- a/kokoro/public/presubmit.cfg
+++ b/kokoro/public/presubmit.cfg
@@ -1,7 +1,7 @@
# Format: //devtools/kokoro/config/proto/build.proto
# Location of the presubmit bash script in Git.
-build_file: "git/kokoro/build.sh"
+build_file: "git/core/kokoro/build.sh"
action {
define_artifacts {
