This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 159936067 IMPALA-12354: Add aarch64 native-toolchain build
159936067 is described below
commit 15993606785f9a5fcce76226728eeefbf33a92fe
Author: Michael Smith <[email protected]>
AuthorDate: Wed Sep 20 11:25:06 2023 -0700
IMPALA-12354: Add aarch64 native-toolchain build
Pre-built toolchains are identified by a TOOLCHAIN_BUILD_ID. This commit
adds an aarch64 (64-bit ARM) native-toolchain build, separate from the
x86_64 native-toolchain build, with its own environment variable set in
impala-config.sh. bootstrap_toolchain.py selects which version to use
based on 'uname -m'.
impala-config.sh also verifies that IMPALA_TOOLCHAIN_BUILD_ID_AARCH64
and IMPALA_TOOLCHAIN_BUILD_ID_X86_64 were produced from the same
native-toolchain ref by checking the 2nd token of the build ID.
Updates package version to include the architecture tag to match how
native-toolchain now names them.
Testing:
- successfully built on ARM, and tests passed (exceptions noted in
IMPALA-12490)
Change-Id: I9bfa7125dbc647b33041c5572d97b7f7ccad6258
Reviewed-on: http://gerrit.cloudera.org:8080/20519
Reviewed-by: Joe McDonnell <[email protected]>
Tested-by: Michael Smith <[email protected]>
---
bin/bootstrap_system.sh | 9 ---------
bin/bootstrap_toolchain.py | 11 ++++++++---
bin/impala-config.sh | 14 ++++++++++++--
buildall.sh | 2 +-
4 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index 3a50b25ad..2c645c788 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -480,15 +480,6 @@ SET_IMPALA_HOME="export IMPALA_HOME=$(pwd)"
echo -e "\n$SET_IMPALA_HOME" >> ~/.bashrc
eval "$SET_IMPALA_HOME"
-if [[ $ARCH_NAME == 'aarch64' ]]; then
- SET_TOOLCHAIN_HOME="export
NATIVE_TOOLCHAIN_HOME=${IMPALA_HOME}/../native-toolchain"
- echo -e "\n$SET_TOOLCHAIN_HOME" >> ~/.bashrc
- echo -e "\n$SET_TOOLCHAIN_HOME" >>
"${IMPALA_HOME}/bin/impala-config-local.sh"
- eval "$SET_TOOLCHAIN_HOME"
- # Provide access to ~/.cache on build machines so we can use ccache.
- sudo chmod 755 ~/.cache
-fi
-
# Try to prepopulate the m2 directory to save time
if [[ "${PREPOPULATE_M2_REPOSITORY:-true}" == true ]] ; then
echo ">>> Populating m2 directory..."
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 45cd1abd4..48f0caece 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -253,12 +253,17 @@ class ToolchainPackage(EnvVersionedPackage):
target_comp = parts[1]
compiler = get_toolchain_compiler()
label = get_platform_release_label(release=platform_release).toolchain
- toolchain_build_id = os.environ["IMPALA_TOOLCHAIN_BUILD_ID"]
+ # Most common return values for machine are x86_64 or aarch64
+ arch = platform.machine()
+ if arch not in ['aarch64', 'x86_64']:
+ raise Exception("Unsupported architecture '{}' for pre-built
native-toolchain. "
+ "Fetch and build it locally by setting
NATIVE_TOOLCHAIN_HOME".format(arch))
+ toolchain_build_id =
os.environ["IMPALA_TOOLCHAIN_BUILD_ID_{}".format(arch.upper())]
toolchain_host = os.environ["IMPALA_TOOLCHAIN_HOST"]
- template_subs = {'compiler': compiler, 'label': label,
+ template_subs = {'compiler': compiler, 'label': label, 'arch': arch,
'toolchain_build_id': toolchain_build_id,
'toolchain_host': toolchain_host}
- archive_basename_tmpl = "${name}-${version}-${compiler}-${label}"
+ archive_basename_tmpl = "${name}-${version}-${compiler}-${label}-${arch}"
url_prefix_tmpl = "https://${toolchain_host}/build/${toolchain_build_id}/";
+ \
"${name}/${version}-${compiler}/"
unpack_directory_tmpl = "${name}-${version}"
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 074d92c5f..f26b2ee43 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -85,12 +85,22 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false}
# moving to a different build of the toolchain, e.g. when a version is bumped
or a
# compile option is changed. The build id can be found in the output of the
toolchain
# build jobs, it is constructed from the build number and toolchain git hash
prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=358-e7cfab15d3
+export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=1-244b88b440
+export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=375-244b88b440
export IMPALA_TOOLCHAIN_REPO=\
${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git}
export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master}
export IMPALA_TOOLCHAIN_COMMIT_HASH=\
-${IMPALA_TOOLCHAIN_COMMIT_HASH-e7cfab15d36ae051747252b676f0a11a9c58fe05}
+${IMPALA_TOOLCHAIN_COMMIT_HASH-244b88b440d10d28510312caaa8d134a916d3ce5}
+# Compare the build ref in build IDs by removing everything
up-to-and-including the
+# first hyphen.
+if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
+ != "${IMPALA_TOOLCHAIN_BUILD_ID_X86_64#*-}" ]; then
+ echo "IMPALA_TOOLCHAIN_BUILD_ID_AARCH64 and IMPALA_TOOLCHAIN_BUILD_ID_X86_64
must" \
+ "come from the same commit hash."
+ exit 1
+fi
+
# Versions of toolchain dependencies.
# -----------------------------------
if $USE_AVRO_CPP; then
diff --git a/buildall.sh b/buildall.sh
index ba32813eb..cdaf319f3 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -424,7 +424,7 @@ bootstrap_dependencies() {
git remote add toolchain "${IMPALA_TOOLCHAIN_REPO}"
git fetch toolchain "${IMPALA_TOOLCHAIN_BRANCH}"
# Specifying a branch avoids a large message from git about detached
HEADs.
- git checkout "${IMPALA_TOOLCHAIN_COMMIT_HASH}" -b
"${IMPALA_TOOLCHAIN_BUILD_ID}"
+ git checkout "${IMPALA_TOOLCHAIN_COMMIT_HASH}" -b
"${IMPALA_TOOLCHAIN_BRANCH}"
else
pushd "${NATIVE_TOOLCHAIN_HOME}"
fi
