This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new fe8b70606a remove fatjar from default integration tests images, fixes
#7459 (#7460)
fe8b70606a is described below
commit fe8b70606a30c5002e29a85eb68cf15d4af8bc00
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Jul 8 15:41:56 2026 +0200
remove fatjar from default integration tests images, fixes #7459 (#7460)
---
...flink.yaml => integration-tests-beam-base.yaml} | 43 +++++++---------------
.../integration-tests/integration-tests-flink.yaml | 2 +-
...tests-flink.yaml => integration-tests-gcp.yaml} | 35 +++---------------
.../integration-tests/integration-tests-spark.yaml | 2 +-
.../integration-tests/unit-tests-beam.Dockerfile | 31 ++++++++++++++++
docker/integration-tests/unit-tests.Dockerfile | 4 +-
integration-tests/scripts/run-tests-docker.sh | 14 +++++++
7 files changed, 67 insertions(+), 64 deletions(-)
diff --git a/docker/integration-tests/integration-tests-flink.yaml
b/docker/integration-tests/integration-tests-beam-base.yaml
similarity index 54%
copy from docker/integration-tests/integration-tests-flink.yaml
copy to docker/integration-tests/integration-tests-beam-base.yaml
index 008eafef75..fb62f84eca 100644
--- a/docker/integration-tests/integration-tests-flink.yaml
+++ b/docker/integration-tests/integration-tests-beam-base.yaml
@@ -15,35 +15,18 @@
# specific language governing permissions and limitations
# under the License.
+# Same as integration-tests-base.yaml, but backed by hop-beam-image, which
adds the Hop fat jar
+# at /tmp/hop-fatjar.jar. Only the Beam-based projects (spark, flink, gcp)
extend this service.
+# hop-beam-image is FROM hop-base-image, so the base image must be built first
(run-tests-docker.sh
+# builds the base image up-front and this beam image on demand when a Beam
project runs).
services:
- integration_test_flink:
- extends:
- file: integration-tests-base.yaml
- service: integration_test
- depends_on:
- - taskmanager
- links:
- - jobmanager
- - taskmanager
-
- jobmanager:
- image: flink:1.18.1-java17
- ports:
- - "8081"
- command: jobmanager
- environment:
- - |
- FLINK_PROPERTIES=
- jobmanager.rpc.address: jobmanager
-
- taskmanager:
- image: flink:1.18.1-java17
- depends_on:
- - jobmanager
- command: taskmanager
- scale: 1
+ integration_test:
+ image: hop-beam-image
+ build:
+ context: ../../.
+ dockerfile: docker/integration-tests/unit-tests-beam.Dockerfile
+ volumes:
+ - ../../integration-tests/:/files
environment:
- - |
- FLINK_PROPERTIES=
- jobmanager.rpc.address: jobmanager
- taskmanager.numberOfTaskSlots: 2
+ - FLASK_ENV=docker
+ command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]
diff --git a/docker/integration-tests/integration-tests-flink.yaml
b/docker/integration-tests/integration-tests-flink.yaml
index 008eafef75..8d0c66a2ba 100644
--- a/docker/integration-tests/integration-tests-flink.yaml
+++ b/docker/integration-tests/integration-tests-flink.yaml
@@ -18,7 +18,7 @@
services:
integration_test_flink:
extends:
- file: integration-tests-base.yaml
+ file: integration-tests-beam-base.yaml
service: integration_test
depends_on:
- taskmanager
diff --git a/docker/integration-tests/integration-tests-flink.yaml
b/docker/integration-tests/integration-tests-gcp.yaml
similarity index 57%
copy from docker/integration-tests/integration-tests-flink.yaml
copy to docker/integration-tests/integration-tests-gcp.yaml
index 008eafef75..a0eeed3d85 100644
--- a/docker/integration-tests/integration-tests-flink.yaml
+++ b/docker/integration-tests/integration-tests-gcp.yaml
@@ -15,35 +15,12 @@
# specific language governing permissions and limitations
# under the License.
+# The gcp project runs pipelines on Google Dataflow, which needs the Hop fat
jar. It therefore uses
+# hop-beam-image (via integration-tests-beam-base.yaml) instead of the plain
base image. Without
+# this file the runner would fall back to the fat-jar-less base image. The gcp
project is currently
+# disabled (integration-tests/gcp/disabled.txt) as it requires live Google
Cloud credentials.
services:
- integration_test_flink:
+ integration_test_gcp:
extends:
- file: integration-tests-base.yaml
+ file: integration-tests-beam-base.yaml
service: integration_test
- depends_on:
- - taskmanager
- links:
- - jobmanager
- - taskmanager
-
- jobmanager:
- image: flink:1.18.1-java17
- ports:
- - "8081"
- command: jobmanager
- environment:
- - |
- FLINK_PROPERTIES=
- jobmanager.rpc.address: jobmanager
-
- taskmanager:
- image: flink:1.18.1-java17
- depends_on:
- - jobmanager
- command: taskmanager
- scale: 1
- environment:
- - |
- FLINK_PROPERTIES=
- jobmanager.rpc.address: jobmanager
- taskmanager.numberOfTaskSlots: 2
diff --git a/docker/integration-tests/integration-tests-spark.yaml
b/docker/integration-tests/integration-tests-spark.yaml
index 36701d1050..871732ff0c 100644
--- a/docker/integration-tests/integration-tests-spark.yaml
+++ b/docker/integration-tests/integration-tests-spark.yaml
@@ -18,7 +18,7 @@
services:
integration_test_spark:
extends:
- file: integration-tests-base.yaml
+ file: integration-tests-beam-base.yaml
service: integration_test
depends_on:
- spark
diff --git a/docker/integration-tests/unit-tests-beam.Dockerfile
b/docker/integration-tests/unit-tests-beam.Dockerfile
new file mode 100644
index 0000000000..e807016997
--- /dev/null
+++ b/docker/integration-tests/unit-tests-beam.Dockerfile
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Beam variant of the integration-test image.
+#
+# The Beam-based run configurations (Spark, Flink, Google Dataflow) ship all
of Hop to their
+# workers as a single fat jar. Generating that fat jar takes a couple of
minutes and produces a
+# ~700MB file, so it is deliberately kept out of the shared base image
(unit-tests.Dockerfile) and
+# built here instead. This image is only built/used when a project that
references the fat jar
+# actually runs (see integration-tests-beam-base.yaml and
scripts/run-tests-docker.sh).
+
+FROM hop-base-image
+
+# Runs as the (already configured) hop/jenkins user inherited from the base
image, which owns the
+# Hop installation under ${DEPLOYMENT_PATH}/hop.
+RUN cd ${DEPLOYMENT_PATH}/hop \
+ && ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar
diff --git a/docker/integration-tests/unit-tests.Dockerfile
b/docker/integration-tests/unit-tests.Dockerfile
index 83263cb9ae..c10cccd66b 100644
--- a/docker/integration-tests/unit-tests.Dockerfile
+++ b/docker/integration-tests/unit-tests.Dockerfile
@@ -82,9 +82,7 @@ COPY --chown=${JENKINS_USER}:${JENKINS_GROUP}
./docker/integration-tests/resourc
# Unzip and install in correct location
RUN chown -R ${JENKINS_USER}:${JENKINS_GROUP} ${DEPLOYMENT_PATH}/hop \
- && chmod 700 ${DEPLOYMENT_PATH}/hop/*.sh \
- && cd ${DEPLOYMENT_PATH}/hop \
- && ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar
+ && chmod 700 ${DEPLOYMENT_PATH}/hop/*.sh
# make volume available so that hop pipeline and workflow files can be
provided easily
VOLUME ["/files"]
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
index 96635a4716..446a91e285 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -87,6 +87,11 @@ unzip -o -q
"${CURRENT_DIR}/../../assemblies/client/target/*.zip" -d ${CURRENT_D
# Build base image only once
docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml build
--build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID}
--build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg
JENKINS_GID=${JENKINS_GID} --build-arg GCP_KEY_FILE=${GCP_KEY_FILE}
+# The Hop fat jar (needed only by the Beam runners: spark/flink/gcp) is
expensive to build, so it
+# lives in a separate image (hop-beam-image) that we build lazily and only
once, the first time a
+# project that actually references the fat jar is about to run.
+BEAM_IMAGE_BUILT="false"
+
# Loop over project folders
for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
@@ -101,6 +106,15 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
echo "project path: $d"
echo "docker compose path: ${DOCKER_FILES_DIR}"
+ # If this project references the Hop fat jar (Beam runners), make sure
hop-beam-image exists.
+ # Built once per run, and only when such a project is actually enabled.
+ if [ "${BEAM_IMAGE_BUILT}" != "true" ] && grep -rqs "hop-fatjar.jar"
"$d" 2>/dev/null; then
+ echo "Project ${PROJECT_NAME} needs the Hop fat jar; building
hop-beam-image (once)."
+ docker compose -f ${DOCKER_FILES_DIR}/integration-tests-beam-base.yaml
build
+ EXECUTED_COMPOSE_FILES=("${EXECUTED_COMPOSE_FILES[@]}"
"${DOCKER_FILES_DIR}/integration-tests-beam-base.yaml")
+ BEAM_IMAGE_BUILT="true"
+ fi
+
# Check if specific compose exists
if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml" ];
then
