This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 9faeca0351bc52b7c1fb9986bc4a0823307e59ec Author: kartikeyaagrawal <[email protected]> AuthorDate: Thu Apr 30 15:10:08 2026 +0530 chore(docker): reduce base_java17 and spark_base image size (#18542) Addresses #18523. Shrinks the Java 17 integration-test image from ~3.56 GB to ~2.58 GB (~27%) without changing any runtime behavior. The container commands, environment variables, exposed ports, and entrypoint are identical. base_java17/Dockerfile: - switch base image from eclipse-temurin:17-jdk to 17-jre-jammy. The container only runs Hadoop; no Java compilation happens inside it, so the JDK toolchain is not needed. - convert to a multi-stage build. Stage 1 downloads and extracts the Hadoop tarball; stage 2 only COPYs the extracted tree. curl, ca-certificates, and the tar.gz no longer land in the final layer. - use --no-install-recommends and clean apt lists in the runtime stage. - drop the unused .asc signature download and the now-dead wget dep. spark_base/Dockerfile: - replace the Python-3.10.14-from-source build (which pulled in build-essential and a full compile toolchain, then built CPython with --enable-optimizations inside the image) with the distro python3-minimal + python3-pip packages. PySpark only needs a Python runtime at runtime. (cherry picked from commit 0a28d695eb1558b01bb38b11996a436a2e3e23dd) --- docker/hoodie/hadoop/base_java17/Dockerfile | 41 ++++++++++++++++++----------- docker/hoodie/hadoop/spark_base/Dockerfile | 21 +++------------ 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/docker/hoodie/hadoop/base_java17/Dockerfile b/docker/hoodie/hadoop/base_java17/Dockerfile index 45108610b19e..a5c265d3aaac 100644 --- a/docker/hoodie/hadoop/base_java17/Dockerfile +++ b/docker/hoodie/hadoop/base_java17/Dockerfile @@ -15,7 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM eclipse-temurin:17-jdk +# --- Stage 1: fetch + extract Hadoop (throwaway) --- +FROM eclipse-temurin:17-jre-jammy AS hadoop-builder + +ARG HADOOP_VERSION=3.4.0 +ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + +RUN set -x \ + && DEBIAN_FRONTEND=noninteractive apt-get -yq update \ + && apt-get -yq install --no-install-recommends curl ca-certificates \ + && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \ + && mkdir -p /opt \ + && tar -xzf /tmp/hadoop.tar.gz -C /opt/ \ + && rm /tmp/hadoop.tar.gz \ + && mkdir -p /opt/hadoop-${HADOOP_VERSION}/logs + +# --- Stage 2: runtime --- +FROM eclipse-temurin:17-jre-jammy LABEL maintainer="Hoodie" USER root @@ -23,28 +39,23 @@ USER root ENV LANG C.UTF-8 ARG HADOOP_VERSION=3.4.0 -ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz -ENV HADOOP_VERSION ${HADOOP_VERSION} -ENV HADOOP_URL ${HADOOP_URL} +ENV HADOOP_VERSION=${HADOOP_VERSION} -RUN set -x \ - && DEBIAN_FRONTEND=noninteractive apt-get -yq update && apt-get -yq install curl wget netcat-openbsd procps \ - && echo "Fetch URL2 is : ${HADOOP_URL}" \ - && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \ - && curl -fSL "${HADOOP_URL}.asc" -o /tmp/hadoop.tar.gz.asc \ - && mkdir -p /opt/hadoop-$HADOOP_VERSION/logs \ - && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \ - && rm /tmp/hadoop.tar.gz* \ - && ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop \ +RUN DEBIAN_FRONTEND=noninteractive apt-get -yq update \ + && apt-get -yq install --no-install-recommends netcat-openbsd procps \ + && rm -rf /var/lib/apt/lists/* \ && mkdir /hadoop-data -ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION +COPY --from=hadoop-builder /opt/hadoop-${HADOOP_VERSION} /opt/hadoop-${HADOOP_VERSION} +RUN ln -s /opt/hadoop-${HADOOP_VERSION}/etc/hadoop /etc/hadoop + +ENV HADOOP_PREFIX=/opt/hadoop-${HADOOP_VERSION} ENV HADOOP_CONF_DIR=/etc/hadoop ENV MULTIHOMED_NETWORK=1 ENV HADOOP_HOME=${HADOOP_PREFIX} ENV HADOOP_INSTALL=${HADOOP_HOME} ENV USER=root -ENV PATH /usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH +ENV PATH=/usr/bin:/bin:${HADOOP_PREFIX}/bin/:$PATH # Exposing a union of ports across hadoop versions # Well known ports including ssh diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile index 68bfbaae76d8..4210faf21bfe 100644 --- a/docker/hoodie/hadoop/spark_base/Dockerfile +++ b/docker/hoodie/hadoop/spark_base/Dockerfile @@ -41,23 +41,10 @@ RUN echo "Installing Spark-version (${SPARK_VERSION})" \ && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \ && cd / -# Install build dependencies -RUN apt-get update && apt-get install -y \ - wget build-essential libncursesw5-dev \ - libssl-dev libgdbm-dev libreadline-dev libbz2-dev \ - libsqlite3-dev libffi-dev zlib1g-dev curl \ - && cd /usr/src \ - && wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz \ - && tar xzf Python-3.10.14.tgz \ - && cd Python-3.10.14 \ - && ./configure --enable-optimizations \ - && make -j"$(nproc)" \ - && make altinstall \ - && ln -sf /usr/local/bin/python3.10 /usr/bin/python \ - && ln -sf /usr/local/bin/python3.10 /usr/bin/python3 \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python \ - && pip install --upgrade pip \ - && cd / && rm -rf /usr/src/Python-3.10.14* \ +# Install Python runtime from distro package (avoids ~400MB build toolchain) +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3-minimal python3-pip \ + && ln -sf /usr/bin/python3 /usr/bin/python \ && rm -rf /var/lib/apt/lists/* #Give permission to execute scripts
