This is an automated email from the ASF dual-hosted git repository.
vinayakumarb pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5279af9b2fb HADOOP-19540. [JDK17] Add ubuntu:noble as a build platform
with JDK-17 as default (#7608)
5279af9b2fb is described below
commit 5279af9b2fbfbdf894b0ce408437b061db7ae124
Author: Vinayakumar B <vinayakum...@apache.org>
AuthorDate: Wed May 7 00:04:06 2025 +0530
HADOOP-19540. [JDK17] Add ubuntu:noble as a build platform with JDK-17 as
default (#7608)
1. Added "ubuntu:noble" to supported docker platforms.
2. Modified `start-build-env.sh` to use expected Dockerfiles dynamically.
ex: `bash start-build-env.sh ubuntu_24` where `ubuntu_24` is the extra
suffix in the name of the Dockerfile.
3. Added `jdk17++` profile activated when jdk17 is available.
Successfully built the entire Hadoop tar with native support on env created
using `bash start-build-env.sh ubuntu_24`
---
BUILDING.txt | 6 +-
dev-support/docker/Dockerfile_ubuntu_24 | 106 +++++++++++++++++++++++++
dev-support/docker/pkg-resolver/packages.json | 66 ++++++++++++++-
dev-support/docker/pkg-resolver/platforms.json | 1 +
hadoop-project/pom.xml | 10 +++
start-build-env.sh | 26 +++++-
6 files changed, 209 insertions(+), 6 deletions(-)
diff --git a/BUILDING.txt b/BUILDING.txt
index f1529d6e515..4dfb48fb089 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -29,7 +29,11 @@ This requires a recent version of docker (1.4.1 and higher
are known to work).
On Linux / Mac:
Install Docker and run this command:
- $ ./start-build-env.sh
+ $ ./start-build-env.sh [OS platform]
+
+ - [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20,
ubuntu_24, windows_10].
+ Default is 'ubuntu_20'.
+ Note: Currently only default ('ubuntu_20') is supported on arm machine
The prompt which is then presented is located at a mounted version of the
source tree
and all required tools for testing and building have been installed and
configured.
diff --git a/dev-support/docker/Dockerfile_ubuntu_24
b/dev-support/docker/Dockerfile_ubuntu_24
new file mode 100644
index 00000000000..28ab04c3e09
--- /dev/null
+++ b/dev-support/docker/Dockerfile_ubuntu_24
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM ubuntu:noble
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+ && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from apt
+######
+# hadolint ignore=DL3008,SC2046
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends wget apt-transport-https
gpg gpg-agent gawk ca-certificates \
+ && apt-get -q install -y --no-install-recommends python3 \
+ && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F=
'/^VERSION_CODENAME/{print$2}' /etc/os-release) main" >
/etc/apt/sources.list.d/adoptium.list \
+ && wget -q -O -
https://packages.adoptium.net/artifactory/api/gpg/key/public >
/etc/apt/trusted.gpg.d/adoptium.asc \
+ && apt-get -q update \
+ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py
ubuntu:noble) \
+ && apt-get clean \
+ && update-java-alternatives -s java-1.17.0-openjdk-amd64 \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN locale-gen en_US.UTF-8
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
+ENV PYTHONIOENCODING=utf-8
+
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /usr
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64
+
+#######
+# Set env vars for SpotBugs 4.2.2
+#######
+ENV SPOTBUGS_HOME /opt/spotbugs
+
+#######
+# Set env vars for Google Protobuf 3.21.12
+#######
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx3072m
+
+# Skip gpg verification when downloading Yetus via yetus-wrapper
+ENV HADOOP_SKIP_YETUS_VERIFICATION true
+
+####
+# Install packages
+####
+RUN pkg-resolver/install-spotbugs.sh ubuntu:noble
+RUN pkg-resolver/install-boost.sh ubuntu:noble
+RUN pkg-resolver/install-protobuf.sh ubuntu:noble
+RUN pkg-resolver/install-hadolint.sh ubuntu:noble
+RUN pkg-resolver/install-intel-isa-l.sh ubuntu:noble
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
+# Add a welcome message and environment checks.
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
+RUN chmod 755 /root/hadoop_env_checks.sh
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
diff --git a/dev-support/docker/pkg-resolver/packages.json
b/dev-support/docker/pkg-resolver/packages.json
index 87ee435703c..4b37ea62b42 100644
--- a/dev-support/docker/pkg-resolver/packages.json
+++ b/dev-support/docker/pkg-resolver/packages.json
@@ -3,17 +3,20 @@
"debian:10": "ant",
"ubuntu:focal": "ant",
"ubuntu:focal::arch64": "ant",
+ "ubuntu:noble": "ant",
"centos:7": "ant",
"centos:8": "ant"
},
"apt-utils": {
"debian:10": "apt-utils",
"ubuntu:focal": "apt-utils",
+ "ubuntu:noble": "apt-utils",
"ubuntu:focal::arch64": "apt-utils"
},
"automake": {
"debian:10": "automake",
"ubuntu:focal": "automake",
+ "ubuntu:noble": "automake",
"ubuntu:focal::arch64": "automake",
"centos:7": "automake",
"centos:8": "automake"
@@ -24,11 +27,13 @@
"bats": {
"debian:10": "bats",
"ubuntu:focal": "bats",
+ "ubuntu:noble": "bats",
"ubuntu:focal::arch64": "bats"
},
"build-essential": {
"debian:10": "build-essential",
"ubuntu:focal": "build-essential",
+ "ubuntu:noble": "build-essential",
"ubuntu:focal::arch64": "build-essential",
"centos:7": "build-essential"
},
@@ -41,6 +46,10 @@
"bzip2",
"libbz2-dev"
],
+ "ubuntu:noble": [
+ "bzip2",
+ "libbz2-dev"
+ ],
"ubuntu:focal::arch64": [
"bzip2",
"libbz2-dev"
@@ -57,12 +66,14 @@
"clang": {
"debian:10": "clang",
"ubuntu:focal": "clang",
+ "ubuntu:noble": "clang",
"ubuntu:focal::arch64": "clang",
"centos:7": "clang",
"centos:8": "clang"
},
"cmake": {
"ubuntu:focal": "cmake",
+ "ubuntu:noble": "cmake",
"ubuntu:focal::arch64": "cmake"
},
"curl": {
@@ -74,6 +85,10 @@
"curl",
"libcurl4-openssl-dev"
],
+ "ubuntu:noble": [
+ "curl",
+ "libcurl4-openssl-dev"
+ ],
"ubuntu:focal::arch64": [
"curl",
"libcurl4-openssl-dev"
@@ -90,6 +105,7 @@
"doxygen": {
"debian:10": "doxygen",
"ubuntu:focal": "doxygen",
+ "ubuntu:noble": "doxygen",
"ubuntu:focal::arch64": "doxygen",
"centos:7": "doxygen"
},
@@ -105,6 +121,10 @@
"fuse",
"libfuse-dev"
],
+ "ubuntu:noble": [
+ "fuse",
+ "libfuse-dev"
+ ],
"ubuntu:focal::arch64": [
"fuse",
"libfuse-dev"
@@ -131,6 +151,10 @@
"gcc",
"g++"
],
+ "ubuntu:noble": [
+ "gcc",
+ "g++"
+ ],
"ubuntu:focal::arch64": [
"gcc",
"g++"
@@ -146,27 +170,32 @@
"git": {
"debian:10": "git",
"ubuntu:focal": "git",
+ "ubuntu:noble": "git",
"ubuntu:focal::arch64": "git",
"centos:8": "git"
},
"gnupg-agent": {
"debian:10": "gnupg-agent",
"ubuntu:focal": "gnupg-agent",
+ "ubuntu:noble": "gnupg-agent",
"ubuntu:focal::arch64": "gnupg-agent"
},
"hugo": {
"debian:10": "hugo",
"ubuntu:focal": "hugo",
+ "ubuntu:noble": "hugo",
"ubuntu:focal::arch64": "hugo"
},
"libbcprov-java": {
"debian:10": "libbcprov-java",
"ubuntu:focal": "libbcprov-java",
+ "ubuntu:noble": "libbcprov-java",
"ubuntu:focal::arch64": "libbcprov-java"
},
"libtool": {
"debian:10": "libtool",
"ubuntu:focal": "libtool",
+ "ubuntu:noble": "libtool",
"ubuntu:focal::arch64": "libtool",
"centos:7": "libtool",
"centos:8": "libtool"
@@ -174,6 +203,7 @@
"openssl": {
"debian:10": "libssl-dev",
"ubuntu:focal": "libssl-dev",
+ "ubuntu:noble": "libssl-dev",
"ubuntu:focal::arch64": "libssl-dev",
"centos:7": "openssl-devel",
"centos:8": "openssl-devel"
@@ -193,6 +223,10 @@
"libprotobuf-dev",
"libprotoc-dev"
],
+ "ubuntu:noble": [
+ "libprotobuf-dev",
+ "libprotoc-dev"
+ ],
"ubuntu:focal::arch64": [
"libprotobuf-dev",
"libprotoc-dev"
@@ -201,6 +235,7 @@
"sasl": {
"debian:10": "libsasl2-dev",
"ubuntu:focal": "libsasl2-dev",
+ "ubuntu:noble": "libsasl2-dev",
"ubuntu:focal::arch64": "libsasl2-dev",
"centos:7": "cyrus-sasl-devel",
"centos:8": "cyrus-sasl-devel"
@@ -208,6 +243,7 @@
"snappy": {
"debian:10": "libsnappy-dev",
"ubuntu:focal": "libsnappy-dev",
+ "ubuntu:noble": "libsnappy-dev",
"ubuntu:focal::arch64": "libsnappy-dev",
"centos:7": "snappy-devel"
},
@@ -220,6 +256,10 @@
"libzstd-dev",
"zlib1g-dev"
],
+ "ubuntu:noble": [
+ "libzstd-dev",
+ "zlib1g-dev"
+ ],
"ubuntu:focal::arch64": [
"libzstd-dev",
"zlib1g-dev"
@@ -236,11 +276,13 @@
"locales": {
"debian:10": "locales",
"ubuntu:focal": "locales",
+ "ubuntu:noble": "locales",
"ubuntu:focal::arch64": "locales"
},
"libtirpc-devel": {
"centos:7": "libtirpc-devel",
- "centos:8": "libtirpc-devel"
+ "centos:8": "libtirpc-devel",
+ "ubuntu:noble": "libtirpc-dev"
},
"libpmem": {
"centos:7": "libpmem-devel"
@@ -248,6 +290,7 @@
"make": {
"debian:10": "make",
"ubuntu:focal": "make",
+ "ubuntu:noble": "make",
"ubuntu:focal::arch64": "make",
"centos:7": "make",
"centos:8": "make"
@@ -255,6 +298,7 @@
"maven": {
"debian:10": "maven",
"ubuntu:focal": "maven",
+ "ubuntu:noble": "maven",
"ubuntu:focal::arch64": "maven"
},
"java": {
@@ -265,6 +309,11 @@
"openjdk-11-jdk",
"openjdk-17-jdk"
],
+ "ubuntu:noble": [
+ "temurin-24-jdk",
+ "openjdk-11-jdk",
+ "openjdk-17-jdk"
+ ],
"ubuntu:focal::arch64": [
"temurin-24-jdk",
"openjdk-8-jdk",
@@ -275,6 +324,7 @@
"pinentry-curses": {
"debian:10": "pinentry-curses",
"ubuntu:focal": "pinentry-curses",
+ "ubuntu:noble": "pinentry-curses",
"ubuntu:focal::arch64": "pinentry-curses",
"centos:7": "pinentry-curses",
"centos:8": "pinentry-curses"
@@ -282,6 +332,7 @@
"pkg-config": {
"debian:10": "pkg-config",
"ubuntu:focal": "pkg-config",
+ "ubuntu:noble": "pkg-config",
"ubuntu:focal::arch64": "pkg-config",
"centos:8": "pkg-config"
},
@@ -300,6 +351,13 @@
"python3-setuptools",
"python3-wheel"
],
+ "ubuntu:noble": [
+ "python3",
+ "python3-pip",
+ "python3-pkg-resources",
+ "python3-setuptools",
+ "python3-wheel"
+ ],
"ubuntu:focal::arch64": [
"python2.7",
"python3",
@@ -324,6 +382,7 @@
"rsync": {
"debian:10": "rsync",
"ubuntu:focal": "rsync",
+ "ubuntu:noble": "rsync",
"ubuntu:focal::arch64": "rsync",
"centos:7": "rsync",
"centos:8": "rsync"
@@ -331,6 +390,7 @@
"shellcheck": {
"debian:10": "shellcheck",
"ubuntu:focal": "shellcheck",
+ "ubuntu:noble": "shellcheck",
"ubuntu:focal::arch64": "shellcheck"
},
"shasum": {
@@ -340,11 +400,13 @@
"software-properties-common": {
"debian:10": "software-properties-common",
"ubuntu:focal": "software-properties-common",
+ "ubuntu:noble": "software-properties-common",
"ubuntu:focal::arch64": "software-properties-common"
},
"sudo": {
"debian:10": "sudo",
"ubuntu:focal": "sudo",
+ "ubuntu:noble": "sudo",
"ubuntu:focal::arch64": "sudo",
"centos:7": "sudo",
"centos:8": "sudo"
@@ -352,6 +414,7 @@
"valgrind": {
"debian:10": "valgrind",
"ubuntu:focal": "valgrind",
+ "ubuntu:noble": "valgrind",
"ubuntu:focal::arch64": "valgrind",
"centos:7": "valgrind",
"centos:8": "valgrind"
@@ -359,6 +422,7 @@
"yasm": {
"debian:10": "yasm",
"ubuntu:focal": "yasm",
+ "ubuntu:noble": "yasm",
"ubuntu:focal::arch64": "yasm"
}
}
diff --git a/dev-support/docker/pkg-resolver/platforms.json
b/dev-support/docker/pkg-resolver/platforms.json
index 93e2a93df42..2ff65b52823 100644
--- a/dev-support/docker/pkg-resolver/platforms.json
+++ b/dev-support/docker/pkg-resolver/platforms.json
@@ -1,6 +1,7 @@
[
"ubuntu:focal",
"ubuntu:focal::arch64",
+ "ubuntu:noble",
"centos:7",
"centos:8",
"debian:10"
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 9b4439cd2e9..d210994a834 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -2747,6 +2747,16 @@
</dependencies>
</dependencyManagement>
</profile>
+ <profile>
+ <id>jdk17+</id>
+ <activation>
+ <jdk>[17,)</jdk>
+ </activation>
+ <properties>
+ <javac.version>17</javac.version>
+ <maven.compiler.release>${javac.version}</maven.compiler.release>
+ </properties>
+ </profile>
</profiles>
<repositories>
diff --git a/start-build-env.sh b/start-build-env.sh
index 3db4fb0663e..cead1eb8cdb 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -19,12 +19,29 @@ set -e # exit on error
cd "$(dirname "$0")" # connect to root
+OS_PLATFORM="${1:-}"
+[ "$#" -gt 0 ] && shift
+
+DEFAULT_OS_PLATFORM="ubuntu_20"
+
+OS_PLATFORM_SUFFIX=""
+
+if [[ -n ${OS_PLATFORM} ]] && [[ "${OS_PLATFORM}" != "${DEFAULT_OS_PLATFORM}"
]]; then
+ # ubuntu_20 (default) platform does not have suffix in Dockerfile.
+ OS_PLATFORM_SUFFIX="_${OS_PLATFORM}"
+fi
+
DOCKER_DIR=dev-support/docker
-DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
+DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}"
CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
if [[ "$CPU_ARCH" == "aarch64" || "$CPU_ARCH" == "arm64" ]]; then
- DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
+ DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}_aarch64"
+fi
+
+if [ ! -e "${DOCKER_FILE}" ] ; then
+ echo "'${OS_PLATFORM}' environment not available yet for '${CPU_ARCH}'"
+ exit 1
fi
docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR
@@ -69,7 +86,7 @@ fi
# Set the home directory in the Docker container.
DOCKER_HOME_DIR=${DOCKER_HOME_DIR:-/home/${USER_NAME}}
-docker build -t "hadoop-build-${USER_ID}" - <<UserSpecificDocker
+docker build -t "hadoop-build${OS_PLATFORM_SUFFIX}-${USER_ID}" -
<<UserSpecificDocker
FROM hadoop-build
RUN rm -f /var/log/faillog /var/log/lastlog
RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
@@ -93,4 +110,5 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \
-v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \
-v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \
-u "${USER_ID}" \
- "hadoop-build-${USER_ID}" "$@"
+ --name "hadoop-build${OS_PLATFORM_SUFFIX}" \
+ "hadoop-build${OS_PLATFORM_SUFFIX}-${USER_ID}" "$@"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
