This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 31d6086 [DRILL-7803] Docker based developer build environment
31d6086 is described below
commit 31d6086c4f814c1d7fc476095611e37cc3d95d1c
Author: Niels Basjes <[email protected]>
AuthorDate: Fri Nov 13 16:37:10 2020 +0100
[DRILL-7803] Docker based developer build environment
---
dev-support/docker/Dockerfile | 155 +++++++++++++++++++++++++++++++++
dev-support/docker/bashcolors.sh | 93 ++++++++++++++++++++
dev-support/docker/drill_env_checks.sh | 121 +++++++++++++++++++++++++
dev-support/docker/time.sh | 20 +++++
docs/dev/Environment.md | 8 ++
start-build-env.sh | 128 +++++++++++++++++++++++++++
6 files changed, 525 insertions(+)
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
new file mode 100644
index 0000000..44b1d45
--- /dev/null
+++ b/dev-support/docker/Dockerfile
@@ -0,0 +1,155 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Dockerfile for installing the necessary dependencies for building Drill.
+# See BUILDING.txt.
+
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+
+# hadolint ignore=DL3008
+RUN apt -q update \
+ && apt install -y software-properties-common apt-utils apt-transport-https
\
+ # Repo for different Python versions
+ && add-apt-repository -y ppa:deadsnakes/ppa \
+ && apt-get -q install -y --no-install-recommends \
+ ant \
+ bats \
+ bash-completion \
+ build-essential \
+ bzip2 \
+ ca-certificates \
+ clang \
+ cmake \
+ curl \
+ docker.io \
+ doxygen \
+ findbugs \
+ fuse \
+ g++ \
+ gcc \
+ git \
+ gnupg-agent \
+ libaio1 \
+ libbcprov-java \
+ libbz2-dev \
+ libcurl4-openssl-dev \
+ libfuse-dev \
+ libnuma-dev \
+ libncurses5 \
+ libprotobuf-dev \
+ libprotoc-dev \
+ libsasl2-dev \
+ libsnappy-dev \
+ libssl-dev \
+ libtool \
+ libzstd-dev \
+ locales \
+ make \
+ maven \
+# openjdk-11-jdk \
+ openjdk-8-jdk \
+ pinentry-curses \
+ pkg-config \
+ python \
+ python2.7 \
+# python-pip \
+ python-pkg-resources \
+ python-setuptools \
+# python-wheel \
+ python3-setuptools \
+ python3-pip \
+ python3.5 \
+ python3.6 \
+ python3.7 \
+ python2.7 \
+ virtualenv \
+ tox \
+ rsync \
+ shellcheck \
+ software-properties-common \
+ sudo \
+ valgrind \
+ vim \
+ wget \
+ zlib1g-dev \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+###
+# Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist`
to work
+###
+RUN pip3 install grpcio-tools mypy-protobuf
+
+###
+# Install Go
+###
+RUN mkdir -p /goroot \
+ && curl https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz | tar xvzf -
-C /goroot --strip-components=1 \
+ && chmod a+rwX -R /goroot
+
+# Set environment variables for Go
+ENV GOROOT /goroot
+ENV GOPATH /gopath
+ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
+CMD go get github.com/linkedin/goavro
+
+###
+# Miscelaneous fixes...
+###
+# Turns out some build tools use 'time' and in this docker image this is no
longer
+# an executable but ONLY an internal command of bash
+COPY time.sh /usr/bin/time
+RUN chmod 755 /usr/bin/time
+
+# Force the complete use of Java 8
+RUN apt remove -y openjdk-11-jre openjdk-11-jre-headless
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xmx4g -XX:MaxPermSize=512m
+
+###
+# Add a welcome message and environment checks.
+###
+RUN mkdir /scripts
+COPY drill_env_checks.sh /scripts/drill_env_checks.sh
+COPY bashcolors.sh /scripts/bashcolors.sh
+RUN chmod 755 /scripts /scripts/drill_env_checks.sh /scripts/bashcolors.sh
+
+# hadolint ignore=SC2016
+RUN echo '. /etc/bash_completion' >> /root/.bash_aliases
+RUN echo '. /scripts/drill_env_checks.sh' >> /root/.bash_aliases
diff --git a/dev-support/docker/bashcolors.sh b/dev-support/docker/bashcolors.sh
new file mode 100755
index 0000000..0f0e05c
--- /dev/null
+++ b/dev-support/docker/bashcolors.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Based upon info on https://wiki.archlinux.org/index.php/Color_Bash_Prompt
+
+# Reset
+export Color_Off='\e[0m' # Text Reset
+
+# Regular Colors
+export Black='\e[0;30m' # Black
+export Red='\e[0;31m' # Red
+export Green='\e[0;32m' # Green
+export Yellow='\e[0;33m' # Yellow
+export Blue='\e[0;34m' # Blue
+export Purple='\e[0;35m' # Purple
+export Cyan='\e[0;36m' # Cyan
+export White='\e[0;37m' # White
+
+# Bold
+export BBlack='\e[1;30m' # Black
+export BRed='\e[1;31m' # Red
+export BGreen='\e[1;32m' # Green
+export BYellow='\e[1;33m' # Yellow
+export BBlue='\e[1;34m' # Blue
+export BPurple='\e[1;35m' # Purple
+export BCyan='\e[1;36m' # Cyan
+export BWhite='\e[1;37m' # White
+
+# Underline
+export UBlack='\e[4;30m' # Black
+export URed='\e[4;31m' # Red
+export UGreen='\e[4;32m' # Green
+export UYellow='\e[4;33m' # Yellow
+export UBlue='\e[4;34m' # Blue
+export UPurple='\e[4;35m' # Purple
+export UCyan='\e[4;36m' # Cyan
+export UWhite='\e[4;37m' # White
+
+# Background
+export On_Black='\e[40m' # Black
+export On_Red='\e[41m' # Red
+export On_Green='\e[42m' # Green
+export On_Yellow='\e[43m' # Yellow
+export On_Blue='\e[44m' # Blue
+export On_Purple='\e[45m' # Purple
+export On_Cyan='\e[46m' # Cyan
+export On_White='\e[47m' # White
+
+# High Intensity
+export IBlack='\e[0;90m' # Black
+export IRed='\e[0;91m' # Red
+export IGreen='\e[0;92m' # Green
+export IYellow='\e[0;93m' # Yellow
+export IBlue='\e[0;94m' # Blue
+export IPurple='\e[0;95m' # Purple
+export ICyan='\e[0;96m' # Cyan
+export IWhite='\e[0;97m' # White
+
+# Bold High Intensity
+export BIBlack='\e[1;90m' # Black
+export BIRed='\e[1;91m' # Red
+export BIGreen='\e[1;92m' # Green
+export BIYellow='\e[1;93m' # Yellow
+export BIBlue='\e[1;94m' # Blue
+export BIPurple='\e[1;95m' # Purple
+export BICyan='\e[1;96m' # Cyan
+export BIWhite='\e[1;97m' # White
+
+# High Intensity backgrounds
+export On_IBlack='\e[0;100m' # Black
+export On_IRed='\e[0;101m' # Red
+export On_IGreen='\e[0;102m' # Green
+export On_IYellow='\e[0;103m' # Yellow
+export On_IBlue='\e[0;104m' # Blue
+export On_IPurple='\e[0;105m' # Purple
+export On_ICyan='\e[0;106m' # Cyan
+export On_IWhite='\e[0;107m' # White
diff --git a/dev-support/docker/drill_env_checks.sh
b/dev-support/docker/drill_env_checks.sh
new file mode 100755
index 0000000..b92182f
--- /dev/null
+++ b/dev-support/docker/drill_env_checks.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# SHELLDOC-IGNORE
+
+# -------------------------------------------------------
+function showWelcome {
+# http://patorjk.com/software/taag/#p=display&f=Doom&t=Drill%20Build%20Env.
+cat << "Welcome-message"
+
+______ _ _ _ ______ _ _ _ _____
+| _ \ (_) | | | ___ \ (_) | | | | ___|
+| | | |_ __ _| | | | |_/ /_ _ _| | __| | | |__ _ ____ __
+| | | | '__| | | | | ___ \ | | | | |/ _` | | __| '_ \ \ / /
+| |/ /| | | | | | | |_/ / |_| | | | (_| | | |__| | | \ V /
+|___/ |_| |_|_|_| \____/ \__,_|_|_|\__,_| \____/_| |_|\_(_)
+
+This is the standard Drill Developer build environment.
+This has all the right tools installed required to build
+Apache Drill from source.
+
+Welcome-message
+}
+
+# -------------------------------------------------------
+
+function showAbort {
+ cat << "Abort-message"
+
+ ___ _ _ _
+ / _ \| | | | (_)
+/ /_\ \ |__ ___ _ __| |_ _ _ __ __ _
+| _ | '_ \ / _ \| '__| __| | '_ \ / _\` |
+| | | | |_) | (_) | | | |_| | | | | (_| |
+\_| |_/_.__/ \___/|_| \__|_|_| |_|\__, |
+ __/ |
+ |___/
+
+Abort-message
+}
+
+# -------------------------------------------------------
+
+function failIfUserIsRoot {
+ if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong.
+ then
+ cat <<End-of-message
+
+Apparently you are inside this docker container as the user root.
+Putting it simply:
+
+ This should not occur.
+
+Known possible causes of this are:
+1) Running this script as the root user ( Just don't )
+2) Running an old docker version ( upgrade to 1.4.1 or higher )
+
+End-of-message
+
+ showAbort
+
+ logout
+
+ fi
+}
+
+# -------------------------------------------------------
+
+function warnIfLowMemory {
+ MINIMAL_MEMORY=2046755
+ INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
+ if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then
+ cat <<End-of-message
+
+ _ ___ ___
+| | | \\/ |
+| | _____ __ | . . | ___ _ __ ___ ___ _ __ _ _
+| | / _ \\ \\ /\\ / / | |\\/| |/ _ \\ '_ \` _ \\ / _ \\| '__| | | |
+| |___| (_) \\ V V / | | | | __/ | | | | | (_) | | | |_| |
+\\_____/\\___/ \\_/\\_/ \\_| |_/\\___|_| |_| |_|\\___/|_| \\__, |
+ __/ |
+ |___/
+
+Your system is running on very little memory.
+This means it may work but it wil most likely be slower than needed.
+
+If you are running this via boot2docker you can simply increase
+the available memory to at least ${MINIMAL_MEMORY}KiB
+(you have ${INSTALLED_MEMORY}KiB )
+
+End-of-message
+ fi
+}
+
+# -------------------------------------------------------
+
+showWelcome
+warnIfLowMemory
+failIfUserIsRoot
+
+# -------------------------------------------------------
+
+. "/scripts/bashcolors.sh"
+. "/usr/lib/git-core/git-sh-prompt"
+export PS1='\['${IBlue}${On_Black}'\] \u@\['${IWhite}${On_Red}'\][Drill Build
Env.]\['${IBlue}${On_Black}'\]:\['${Cyan}${On_Black}'\]\w$(declare -F __git_ps1
&>/dev/null && __git_ps1 "
\['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\]
]\['${Color_Off}'\]\n$ '
diff --git a/dev-support/docker/time.sh b/dev-support/docker/time.sh
new file mode 100644
index 0000000..94af172
--- /dev/null
+++ b/dev-support/docker/time.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+time "$@"
diff --git a/docs/dev/Environment.md b/docs/dev/Environment.md
index 47d37c1..4660082 100644
--- a/docs/dev/Environment.md
+++ b/docs/dev/Environment.md
@@ -6,6 +6,14 @@ Currently, the Apache Drill build process is known to work on
Linux, Windows and
* Java 8
* Maven 3.6.3 or greater
+## Docker based build environment
+
+The `start-build-env.sh` script in the root of the project source builds and
starts a preconfigured environment
+that contains all the tools needed to build Apache Drill from source.
+
+This is known to work on Ubuntu 20.04 with Docker installed.
+On other systems your success may vary. On Redhat/CentOS based systems no
longer have Docker.
+
## Confirm settings
# java -version
java version "1.8.0_161"
diff --git a/start-build-env.sh b/start-build-env.sh
new file mode 100755
index 0000000..0740149
--- /dev/null
+++ b/start-build-env.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e # exit on error
+
+cd "$(dirname "$0")" # connect to root
+
+DOCKER_DIR=dev-support/docker
+DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
+
+CONTAINER_NAME=drill-dev-${USER}-$$
+
+#CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
+#if [ "$CPU_ARCH" = "aarch64" ]; then
+# DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
+#fi
+
+docker build -t drill-build -f $DOCKER_FILE $DOCKER_DIR
+
+USER_NAME=${SUDO_USER:=$USER}
+USER_ID=$(id -u "${USER_NAME}")
+
+if [ "$(uname -s)" = "Darwin" ]; then
+ GROUP_ID=100
+fi
+
+if [ "$(uname -s)" = "Linux" ]; then
+ GROUP_ID=$(id -g "${USER_NAME}")
+ # man docker-run
+ # When using SELinux, mounted directories may not be accessible
+ # to the container. To work around this, with Docker prior to 1.7
+ # one needs to run the "chcon -Rt svirt_sandbox_file_t" command on
+ # the directories. With Docker 1.7 and later the z mount option
+ # does this automatically.
+ if command -v selinuxenabled >/dev/null && selinuxenabled; then
+ DCKR_VER=$(docker -v|
+ awk '$1 == "Docker" && $2 == "version" {split($3,ver,".");print
ver[1]"."ver[2]}')
+ DCKR_MAJ=${DCKR_VER%.*}
+ DCKR_MIN=${DCKR_VER#*.}
+ if [ "${DCKR_MAJ}" -eq 1 ] && [ "${DCKR_MIN}" -ge 7 ] ||
+ [ "${DCKR_MAJ}" -gt 1 ]; then
+ V_OPTS=:z
+ else
+ for d in "${PWD}" "${HOME}/.m2"; do
+ ctx=$(stat --printf='%C' "$d"|cut -d':' -f3)
+ if [ "$ctx" != svirt_sandbox_file_t ] && [ "$ctx" != container_file_t
]; then
+ printf 'INFO: SELinux is enabled.\n'
+ printf '\tMounted %s may not be accessible to the container.\n' "$d"
+ printf 'INFO: If so, on the host, run the following command:\n'
+ printf '\t# chcon -Rt svirt_sandbox_file_t %s\n' "$d"
+ fi
+ done
+ fi
+ fi
+fi
+
+# Set the home directory in the Docker container.
+DOCKER_HOME_DIR=${DOCKER_HOME_DIR:-/home/${USER_NAME}}
+
+DOCKER_GROUP_ID=$(getent group docker | cut -d':' -f3)
+
+docker build -t "drill-build-${USER_ID}" - <<UserSpecificDocker
+FROM drill-build
+RUN rm -f /var/log/faillog /var/log/lastlog
+RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
+RUN groupmod -g ${DOCKER_GROUP_ID} docker
+RUN useradd -g ${GROUP_ID} -G docker -u ${USER_ID} -k /root -m ${USER_NAME} -d
"${DOCKER_HOME_DIR}"
+RUN echo "${USER_NAME} ALL=NOPASSWD: ALL" >
"/etc/sudoers.d/drill-build-${USER_ID}"
+ENV HOME "${DOCKER_HOME_DIR}"
+
+UserSpecificDocker
+
+echo ""
+echo "Docker image build completed."
+echo
"=============================================================================================="
+echo ""
+
+#If this env varible is empty, docker will be started
+# in non interactive mode
+DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"}
+
+DOCKER_SOCKET_MOUNT=""
+if [ -S /var/run/docker.sock ];
+then
+ DOCKER_SOCKET_MOUNT="-v /var/run/docker.sock:/var/run/docker.sock${V_OPTS:-}"
+ echo "Enabling Docker support with the docker build environment."
+else
+ echo "There is NO Docker support with the docker build environment."
+fi
+
+COMMAND=( "$@" )
+if [ $# -eq 0 ];
+then
+ COMMAND=( "bash" )
+fi
+
+[ -d "${HOME}/.gradle_drill_build_env" ] || mkdir -p
"${HOME}/.gradle_drill_build_env"
+
+# By mapping the .m2 directory you can do an mvn install from
+# within the container and use the result on your normal
+# system. And this also is a significant speedup in subsequent
+# builds because the dependencies are downloaded only once.
+docker run --rm=true ${DOCKER_INTERACTIVE_RUN} \
+ --name "${CONTAINER_NAME}" \
+ -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \
+ -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \
+ -v
"${HOME}/.gradle_drill_build_env:${DOCKER_HOME_DIR}/.gradle${V_OPTS:-}" \
+ -v "${PWD}:${DOCKER_HOME_DIR}/drill${V_OPTS:-}" \
+ -w "${DOCKER_HOME_DIR}/drill" \
+ ${DOCKER_SOCKET_MOUNT} \
+ -u "${USER_ID}" \
+ "drill-build-${USER_ID}" "${COMMAND[@]}"
