This is an automated email from the ASF dual-hosted git repository.
vinayakumarb pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 52b360a HADOOP-16797. Add Dockerfile for ARM builds. Contributed by
Vinayakumar B. (#1801)
52b360a is described below
commit 52b360a92865d2c7cbd113a82b45c6b5a191ce24
Author: Vinayakumar B <vinayakum...@apache.org>
AuthorDate: Mon Jan 13 10:40:29 2020 +0530
HADOOP-16797. Add Dockerfile for ARM builds. Contributed by Vinayakumar B.
(#1801)
---
dev-support/bin/create-release | 16 ++-
dev-support/docker/Dockerfile_aarch64 | 235 ++++++++++++++++++++++++++++++++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +
start-build-env.sh | 11 +-
4 files changed, 265 insertions(+), 3 deletions(-)
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index d14c007..f4851d1 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -204,6 +204,11 @@ function set_defaults
DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile"
DOCKERRAN=false
+ CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
+ if [ "$CPU_ARCH" = "aarch64" ]; then
+ DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
+ fi
+
# Extract Java version from ${BASEDIR}/pom.xml
# doing this outside of maven means we can do this before
# the docker container comes up...
@@ -249,7 +254,9 @@ function startgpgagent
eval $("${GPGAGENT}" --daemon \
--options "${LOGDIR}/gpgagent.conf" \
--log-file="${LOGDIR}/create-release-gpgagent.log")
- GPGAGENTPID=$(echo "${GPG_AGENT_INFO}" | cut -f 2 -d:)
+ GPGAGENTPID=$(pgrep "${GPGAGENT}")
+ GPG_AGENT_INFO="$HOME/.gnupg/S.gpg-agent:$GPGAGENTPID:1"
+ export GPG_AGENT_INFO
fi
if [[ -n "${GPG_AGENT_INFO}" ]]; then
@@ -499,7 +506,12 @@ function dockermode
# we always force build with the OpenJDK JDK
# but with the correct version
- echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
+ if [ "$CPU_ARCH" = "aarch64" ]; then
+ echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
+ else
+ echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
+ fi
+
echo "USER ${user_name}"
printf "\n\n"
) | docker build -t "${imgname}" -
diff --git a/dev-support/docker/Dockerfile_aarch64
b/dev-support/docker/Dockerfile_aarch64
new file mode 100644
index 0000000..8d3c3ad
--- /dev/null
+++ b/dev-support/docker/Dockerfile_aarch64
@@ -0,0 +1,235 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM ubuntu:xenial
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+######
+# Install common dependencies from packages. Versions here are either
+# sufficient or irrelevant.
+#
+# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
+# Ubuntu Java. See Java section below!
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends \
+ apt-utils \
+ build-essential \
+ bzip2 \
+ clang \
+ curl \
+ doxygen \
+ fuse \
+ g++ \
+ gcc \
+ git \
+ gnupg-agent \
+ libbz2-dev \
+ libcurl4-openssl-dev \
+ libfuse-dev \
+ libprotobuf-dev \
+ libprotoc-dev \
+ libsasl2-dev \
+ libsnappy-dev \
+ libssl-dev \
+ libtool \
+ libzstd1-dev \
+ locales \
+ make \
+ pinentry-curses \
+ pkg-config \
+ python \
+ python2.7 \
+ python-pip \
+ python-pkg-resources \
+ python-setuptools \
+ python-wheel \
+ rsync \
+ software-properties-common \
+ snappy \
+ sudo \
+ valgrind \
+ zlib1g-dev \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+
+#######
+# OpenJDK 8
+#######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends openjdk-8-jdk
libbcprov-java \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+
+######
+# Install cmake 3.1.0 (3.5.1 ships with Xenial)
+# There is no cmake binary available for aarch64. Build from source.
+######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/cmake/src \
+ && curl -L -s -S \
+ https://cmake.org/files/v3.1/cmake-3.1.0-1-src.tar.bz2 \
+ -o /opt/cmake/cmake-src.tar.bz2 \
+ && tar xvjf /opt/cmake/cmake-src.tar.bz2 -C /opt/cmake/src \
+ && cd /opt/cmake/src \
+ && tar xvjf cmake-3.1.0.tar.bz2 \
+ && cd cmake-3.1.0 && patch -p0 -i ../cmake-3.1.0-1.patch && mkdir .build
&& cd .build \
+ && ../bootstrap --parallel=2 \
+ && make -j2 && ./bin/cpack \
+ && tar xzf cmake-3.1.0-Linux-aarch64.tar.gz --strip-components 1 -C
/opt/cmake \
+ && cd /opt/cmake && rm -rf /opt/cmake/src
+ENV CMAKE_HOME /opt/cmake
+ENV PATH "${PATH}:/opt/cmake/bin"
+
+######
+# Install Google Protobuf 3.7.1 (2.6.0 ships with Xenial)
+######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/protobuf-src \
+ && curl -L -s -S \
+
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz
\
+ -o /opt/protobuf.tar.gz \
+ && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
+ && cd /opt/protobuf-src \
+ && ./configure --prefix=/opt/protobuf \
+ && make install \
+ && cd /root \
+ && rm -rf /opt/protobuf-src
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+######
+# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends maven \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+ENV MAVEN_HOME /usr
+
+######
+# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
+# Ant is needed for findbugs
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends findbugs ant \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+ENV FINDBUGS_HOME /usr
+
+####
+# Install shellcheck (0.4.6, the latest as of 2017-09-26)
+####
+# hadolint ignore=DL3008
+RUN add-apt-repository -y ppa:hvr/ghc \
+ && apt-get -q update \
+ && apt-get -q install -y --no-install-recommends shellcheck \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+####
+# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
+####
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends bats \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+####
+# Install pylint at fixed version (2.0.0 removed python2 support)
+# https://github.com/PyCQA/pylint/issues/2294
+####
+RUN pip2 install pylint==1.9.2
+
+####
+# Install dateutil.parser
+####
+RUN pip2 install python-dateutil==2.7.3
+
+###
+# Install node.js 8.17.0 for web UI framework (4.2.6 ships with Xenial)
+###
+RUN curl -L -s -S https://deb.nodesource.com/setup_8.x | bash - \
+ && apt-get install -y --no-install-recommends nodejs=8.17.0-1nodesource1 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && npm install -g bower@1.8.8
+
+###
+## Install Yarn 1.12.1 for web UI framework
+####
+RUN curl -s -S https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' >
/etc/apt/sources.list.d/yarn.list \
+ && apt-get -q update \
+ && apt-get install -y --no-install-recommends yarn=1.21.1-1 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+###
+# Install phantomjs built for aarch64
+####
+RUN mkdir -p /opt/phantomjs \
+ && curl -L -s -S \
+
https://github.com/liusheng/phantomjs/releases/download/2.1.1/phantomjs-2.1.1-linux-aarch64.tar.bz2
\
+ -o /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
+ && tar xvjf /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2
--strip-components 1 -C /opt/phantomjs \
+ && cp /opt/phantomjs/bin/phantomjs /usr/bin/ \
+ && rm -rf /opt/phantomjs
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx1536m
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
+# Hugo static website generator (for new hadoop site docs)
+RUN curl -L -o hugo.deb
https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-ARM64.deb
\
+ && dpkg --install hugo.deb \
+ && rm hugo.deb
+
+
+# Add a welcome message and environment checks.
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
+RUN chmod 755 /root/hadoop_env_checks.sh
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 700a5ad..1c6365a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -142,6 +142,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<artifactId>findbugs</artifactId>
<version>3.0.1</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>xml-apis</groupId>
+ <artifactId>xml-apis</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git a/start-build-env.sh b/start-build-env.sh
index 12266af..643770c 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -19,7 +19,15 @@ set -e # exit on error
cd "$(dirname "$0")" # connect to root
-docker build -t hadoop-build dev-support/docker
+DOCKER_DIR=dev-support/docker
+DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
+
+CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
+if [ "$CPU_ARCH" = "aarch64" ]; then
+ DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
+fi
+
+docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR
USER_NAME=${SUDO_USER:=$USER}
USER_ID=$(id -u "${USER_NAME}")
@@ -80,5 +88,6 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \
-v "${PWD}:/home/${USER_NAME}/hadoop${V_OPTS:-}" \
-w "/home/${USER_NAME}/hadoop" \
-v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \
+ -v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg${V_OPTS:-}" \
-u "${USER_NAME}" \
"hadoop-build-${USER_ID}" "$@"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
