This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 3e44e3b2b8 fix google sheet IT test (#7660)
3e44e3b2b8 is described below
commit 3e44e3b2b8cc70b2d2dc25a3e5d46f7f58fe46b3
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Jul 28 11:37:49 2026 +0200
fix google sheet IT test (#7660)
---
.gitignore | 3 +++
.../integration-tests/integration-tests-base.yaml | 15 ++++++++----
.../integration-tests-beam-base.yaml | 3 +++
docker/integration-tests/unit-tests.Dockerfile | 8 ++++---
integration-tests/scripts/run-tests-docker.sh | 28 ++++++++++++++++++++--
integration-tests/scripts/run-tests.sh | 14 +++++++++++
6 files changed, 61 insertions(+), 10 deletions(-)
diff --git a/.gitignore b/.gitignore
index 28e2f00e63..893104b8aa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -86,3 +86,6 @@ docker/local-data/
# Jenkins-style local deploy output (verify-ci-snapshot-zips.sh)
local-snapshots-dir/
+
+# GCP service-account key staged for the integration tests (Jenkins
credentials gcp-access-hop)
+google-key-apache-hop-it.json
diff --git a/docker/integration-tests/integration-tests-base.yaml
b/docker/integration-tests/integration-tests-base.yaml
index 4bb32ece03..78fe2f9f76 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-base.yaml
@@ -22,13 +22,18 @@ services:
context: ../../.
dockerfile: docker/integration-tests/unit-tests.Dockerfile
args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- - GCP_KEY_FILE=./docker/integration-tests/resource/dummyfile
+ # Exported by run-tests-docker.sh so that a rebuild triggered by
"compose up --build"
+ # (hop_server, spark) creates the same container user as the up-front
build. A mismatch
+ # makes the mounts below unreadable/unwritable for the container user.
+ - JENKINS_USER=${JENKINS_USER:-jenkins}
+ - JENKINS_UID=${JENKINS_UID:-1000}
+ - JENKINS_GROUP=${JENKINS_GROUP:-jenkins}
+ - JENKINS_GID=${JENKINS_GID:-1000}
volumes:
- ../../integration-tests/:/files
+ # Real GCP service-account JSON, staged by run-tests-docker.sh
(GCP_KEY_FILE=...).
+ # Defaults to the checked-in placeholder, which makes the Google Sheets
ITs skip.
+ -
${GCP_KEY_HOST_PATH:-./resource/dummyfile}:/tmp/google-key-apache-hop-it.json:ro
environment:
- FLASK_ENV=docker
# Optional workflow basename filter (substring or glob). Set by
run-tests-docker.sh.
diff --git a/docker/integration-tests/integration-tests-beam-base.yaml
b/docker/integration-tests/integration-tests-beam-base.yaml
index f7c82ecdf0..1571cc8985 100644
--- a/docker/integration-tests/integration-tests-beam-base.yaml
+++ b/docker/integration-tests/integration-tests-beam-base.yaml
@@ -29,6 +29,9 @@ services:
HOP_SPARK_CLIENT_VERSION: ${HOP_SPARK_CLIENT_VERSION:-}
volumes:
- ../../integration-tests/:/files
+ # Same GCP key mount as integration-tests-base.yaml: hop-beam-image is
FROM hop-base-image,
+ # so it only carries the placeholder and needs the real key mounted in
as well.
+ -
${GCP_KEY_HOST_PATH:-./resource/dummyfile}:/tmp/google-key-apache-hop-it.json:ro
environment:
- FLASK_ENV=docker
# Keep driver Spark client pack in sync with the fat jar (empty =
default pack)
diff --git a/docker/integration-tests/unit-tests.Dockerfile
b/docker/integration-tests/unit-tests.Dockerfile
index a7419c935d..395ed96df9 100644
--- a/docker/integration-tests/unit-tests.Dockerfile
+++ b/docker/integration-tests/unit-tests.Dockerfile
@@ -30,7 +30,6 @@ ARG JENKINS_USER=hop
ARG JENKINS_GROUP=hop
ARG JENKINS_UID=1000
ARG JENKINS_GID=1000
-ARG GCP_KEY_FILE=
# Set system properties
ENV DEBIAN_FRONTEND=noninteractive
@@ -100,8 +99,11 @@ COPY --chown=${JENKINS_UID}:${JENKINS_GID}
./assemblies/client/target/hop ${DEPL
# into assemblies/client/target/hop before this build (run-tests-docker.sh and
Jenkins do this).
# If a plugin is still missing at runtime, corresponding ITs will fail clearly.
-# Copy gcp key
-COPY --chown=${JENKINS_UID}:${JENKINS_GID} ${GCP_KEY_FILE}
/tmp/google-key-apache-hop-it.json
+# Placeholder GCP key. The real service-account JSON is never baked into the
image: it is
+# bind-mounted over this path at run time (see integration-tests-base.yaml +
run-tests-docker.sh),
+# so the image is identical whether it is built explicitly, implicitly by
"compose up", or from
+# cache. Baking it in via a build-arg made the image content depend on how it
got built.
+COPY --chown=${JENKINS_UID}:${JENKINS_GID}
./docker/integration-tests/resource/dummyfile /tmp/google-key-apache-hop-it.json
# Copy mail keystore
COPY --chown=${JENKINS_UID}:${JENKINS_GID}
./docker/integration-tests/resource/mail/conf/keystore /tmp
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
index 643aed0e7c..29cbcc083e 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -100,6 +100,10 @@ if [ -z "${JENKINS_GID}" ]; then
JENKINS_GID="$(id -g 2>/dev/null || echo 1000)"
fi
+# Exported so the compose files pick the same identity up when they (re)build
the image
+# themselves, e.g. the "up --build" runs for hop_server and spark.
+export JENKINS_USER JENKINS_UID JENKINS_GROUP JENKINS_GID
+
echo "Integration-test container identity: user=${JENKINS_USER}
uid=${JENKINS_UID} group=${JENKINS_GROUP} gid=${JENKINS_GID}"
if [ -z "${SUREFIRE_REPORT}" ]; then
@@ -146,6 +150,21 @@ fi
export SKIP_GOOGLE_SHEETS
echo "SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS}"
+# The key is bind-mounted into the container at run time
(integration-tests-base.yaml) instead of
+# baked into the image, so the containers always see the file this script
validated above.
+# Compose needs an absolute path here: relative values would resolve against
the compose file dir.
+if [ "${GCP_KEY_OK}" = "true" ]; then
+ GCP_KEY_HOST_PATH="${GCP_KEY_PATH}"
+ case "${GCP_KEY_HOST_PATH}" in
+ /*) ;;
+ *) GCP_KEY_HOST_PATH="$(pwd)/${GCP_KEY_HOST_PATH#./}" ;;
+ esac
+else
+ GCP_KEY_HOST_PATH="$(cd "${CURRENT_DIR}/../.." &&
pwd)/docker/integration-tests/resource/dummyfile"
+fi
+export GCP_KEY_HOST_PATH
+echo "GCP_KEY_HOST_PATH=${GCP_KEY_HOST_PATH}"
+
if [ -z "${HOP_OPTIONS}" ] ; then
HOP_OPTIONS="${HOP_OPTIONS}
-Djavax.net.ssl.keyStore=./docker/integration-tests/resource/keystore.jks
-Djavax.net.ssl.keyStorePassword=password
-Djavax.net.ssl.trustStore=./docker/integration-tests/resource/mail/conf/keystore
"
fi
@@ -336,8 +355,13 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION}" ]; then
docker rmi hop-base-image 2>/dev/null || true
fi
-# Build base image only once (must run AFTER pack materialise so jars are in
the image)
-docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml build
--build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID}
--build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg
JENKINS_GID=${JENKINS_GID} --build-arg GCP_KEY_FILE=${GCP_KEY_FILE}
+# Build base image only once (must run AFTER pack materialise so jars are in
the image).
+# Bail out on failure: "docker compose up" would otherwise silently rebuild
hop-base-image from the
+# compose file defaults, hiding the real build error behind confusing test
failures.
+if ! docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml build
--build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID}
--build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg
JENKINS_GID=${JENKINS_GID}; then
+ echo "ERROR: could not build hop-base-image; aborting integration tests" >&2
+ exit 1
+fi
# The Hop fat jar (needed only by the Beam runners: spark/flink/gcp) is
expensive to build, so it
# lives in a separate image (hop-beam-image) that we build lazily and only
once, the first time a
diff --git a/integration-tests/scripts/run-tests.sh
b/integration-tests/scripts/run-tests.sh
index 6116c21845..a45db49389 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -136,6 +136,20 @@ if [ -z "${SKIP_GOOGLE_SHEETS}" ]; then
SKIP_GOOGLE_SHEETS="false"
fi
+# Double-check inside the container: the host may have a valid key while the
file that actually
+# reaches the pipelines is still the placeholder (bad mount, stale image,
...). Deciding here, on
+# the file the transforms open, turns that into a clean skip instead of a
Google credentials error.
+GCP_KEY_IN_CONTAINER="${GOOGLE_APPLICATION_CREDENTIALS:-/tmp/google-key-apache-hop-it.json}"
+if [ "${SKIP_GOOGLE_SHEETS}" != "true" ]; then
+ if [ ! -s "${GCP_KEY_IN_CONTAINER}" ] \
+ || ! grep -qE '"type"[[:space:]]*:[[:space:]]*"service_account"'
"${GCP_KEY_IN_CONTAINER}" 2>/dev/null; then
+ echo "WARNING: ${GCP_KEY_IN_CONTAINER} is not a service-account JSON key
even though the host"
+ echo " reported a valid one (check the GCP_KEY_HOST_PATH mount in
integration-tests-base.yaml)."
+ echo " Skipping the Google Sheets integration tests."
+ SKIP_GOOGLE_SHEETS="true"
+ fi
+fi
+
#set global variables
SPACER="==========================================="
