busbey commented on a change in pull request #1195: HBASE-23876 [WIP] Add JDK11 compilation and unit test support to nightly job URL: https://github.com/apache/hbase/pull/1195#discussion_r384755666
########## File path: dev-support/docker/Dockerfile ########## @@ -14,47 +14,162 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Dockerfile for installing the necessary dependencies for building Hadoop. -# See BUILDING.txt. - -FROM maven:3.5-jdk-8 - -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ - git \ - bats \ - findbugs \ - libperl-critic-perl \ - pylint \ - python-dateutil \ - rsync \ - make \ - gcc \ - libc6-dev \ - ruby \ - ruby-dev \ - wget \ - && \ - gem install --no-document rake rubocop ruby-lint - -ENV FINDBUGS_HOME /usr - -#### -# Install shellcheck -### -RUN mkdir -p /opt/shellcheck && \ - curl -L -s -S \ - https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz \ - -o /opt/shellcheck.tar.xz && \ - tar xJf /opt/shellcheck.tar.xz --strip-components 1 -C /opt/shellcheck && \ - ln -s /opt/shellcheck/shellcheck /usr/bin/shellcheck && \ - rm -f /opt/shellcheck.tar.xz +# +# Dockerfile used as the build and test environment for Yetus. +# +# Built in multiple stages so as to avoid re-downloading large binaries when +# tweaking unrelated aspects of the image. -### -# Avoid out of memory errors in builds -### -ENV MAVEN_OPTS -Xmx3g +# start with a minimal image into which we can download remote tarballs +FROM ubuntu:18.04 AS BASE_IMAGE +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3009 +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \ + DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends -y \ + ca-certificates=20180409 \ + curl=7.58.0-2ubuntu3.8 \ + locales=2.27-3ubuntu1 + +RUN locale-gen en_US.UTF-8 +ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 + +## +# download sundry dependencies +# + +FROM BASE_IMAGE AS FINDBUGS_DOWNLOAD_IMAGE +ENV FINDBUGS_VERSION '3.0.1' +ENV FINDBUGS_URL "https://downloads.sourceforge.net/project/findbugs/findbugs/${FINDBUGS_VERSION}/findbugs-${FINDBUGS_VERSION}.tar.gz"; +ENV FINDBUGS_SHA256 'e80e0da0c213a27504ef3188ef25f107651700ffc66433eac6a7454bbe336419' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -LfsSo /tmp/findbugs.tar.gz "${FINDBUGS_URL}" && \ + echo "${FINDBUGS_SHA256} */tmp/findbugs.tar.gz" | sha256sum -c - + +FROM BASE_IMAGE AS HADOLINT_DOWNLOAD_IMAGE +ENV HADOLINT_VERSION '1.17.5' +ENV HADOLINT_URL "https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VERSION}/hadolint-Linux-x86_64"; +ENV HADOLINT_SHA256 '20dd38bc0602040f19268adc14c3d1aae11af27b463af43f3122076baf827a35' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -LfsSo /tmp/hadolint "${HADOLINT_URL}" && \ + echo "${HADOLINT_SHA256} */tmp/hadolint" | sha256sum -c - + +FROM BASE_IMAGE AS MAVEN_DOWNLOAD_IMAGE +ENV MAVEN_VERSION='3.5.4' +ENV MAVEN_URL "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"; +ENV MAVEN_SHA256 'ce50b1c91364cb77efe3776f756a6d92b76d9038b0a0782f7d53acf1e997a14d' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -LfsSo /tmp/maven.tar.gz "${MAVEN_URL}" && \ + echo "${MAVEN_SHA256} */tmp/maven.tar.gz" | sha256sum -c - + +FROM BASE_IMAGE AS OPENJDK8_DOWNLOAD_IMAGE +ENV OPENJDK8_URL 'https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u232-b09/OpenJDK8U-jdk_x64_linux_hotspot_8u232b09.tar.gz' +ENV OPENJDK8_SHA256 '7b7884f2eb2ba2d47f4c0bf3bb1a2a95b73a3a7734bd47ebf9798483a7bcc423' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -LfsSo /tmp/adoptopenjdk8.tar.gz "${OPENJDK8_URL}" && \ + echo "${OPENJDK8_SHA256} */tmp/adoptopenjdk8.tar.gz" | sha256sum -c - + +FROM BASE_IMAGE AS OPENJDK11_DOWNLOAD_IMAGE +ENV OPENJDK11_URL 'https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.6%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.6_10.tar.gz' +ENV OPENJDK11_SHA256 '330d19a2eaa07ed02757d7a785a77bab49f5ee710ea03b4ee2fa220ddd0feffc' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -LfsSo /tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \ + echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c - + +## +# build the final image +# + +FROM BASE_IMAGE +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +## +# install dependencies from system packages. +# be careful not to install any system packages (i.e., findbugs) that will +# pull in the default-jre. +# + +# bring the base image into conformance with the expectations imposed by +# Yetus and our personality file of what a build environment looks like. +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends -y \ + bash=4.4.18-2ubuntu1.2 \ + build-essential=12.4ubuntu1 \ + curl=7.58.0-2ubuntu3.8 \ + diffutils=1:3.6-1 \ + git=1:2.17.1-1ubuntu0.5 \ + rsync=3.1.2-2.1ubuntu1 \ + tar=1.29b-2ubuntu0.1 \ + wget=1.19.4-1ubuntu2.2 + +# install the dependencies required in order to enable the sundry precommit +# checks/features provided by Yetus plugins. +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends -y \ + bats=0.4.0-1.1 \ + libperl-critic-perl=1.130-1 \ + python3=3.6.7-1~18.04 \ + python3-pip=9.0.1-2.3~ubuntu1.18.04.1 \ + python3-setuptools=39.0.1-2 \ + ruby=1:2.5.1 \ + ruby-dev=1:2.5.1 \ + shellcheck=0.4.6-1 \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN python3 -mpip install --upgrade pip && \ + python3 -mpip install pylint==2.4.4 + +RUN gem install --no-document \ + rake:13.0.1 \ + rubocop:0.80.0 \ + ruby-lint:2.3.1 + +# hadolint ignore=DL3010 +COPY --from=FINDBUGS_DOWNLOAD_IMAGE /tmp/findbugs.tar.gz /tmp/findbugs.tar.gz +RUN tar xzf /tmp/findbugs.tar.gz -C /opt && \ + ln -s "/opt/$(dirname "$(tar -tf /tmp/findbugs.tar.gz | head -n1)")" /opt/findbugs && \ + rm /tmp/findbugs.tar.gz + +COPY --from=HADOLINT_DOWNLOAD_IMAGE /tmp/hadolint /tmp/hadolint +RUN mv /tmp/hadolint /usr/local/bin && \ + chmod a+x /usr/local/bin/hadolint + +# hadolint ignore=DL3010 +COPY --from=MAVEN_DOWNLOAD_IMAGE /tmp/maven.tar.gz /tmp/maven.tar.gz +RUN tar xzf /tmp/maven.tar.gz -C /opt && \ + ln -s "/opt/$(dirname "$(tar -tf /tmp/maven.tar.gz | head -n1)")" /opt/maven && \ + rm /tmp/maven.tar.gz + +## +# ensure JVMs are available under `/usr/lib/jvm` and prefix each installation +# as `java-` so as to conform with Yetus's assumptions. +# + Review comment: what happened to jdk7? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
