This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 473488a88e Issue 7476 : Spark / Beam validation (#7482)
473488a88e is described below
commit 473488a88ed2dca9126078df218e917a09655598
Author: Matt Casters <[email protected]>
AuthorDate: Wed Jul 15 12:55:53 2026 +0200
Issue 7476 : Spark / Beam validation (#7482)
* issue #7476 : spark pinned on 3.5.7, investigation continues
* issue #7476 : spark pinned on 3.5.7, MATRIX-REPORT.md
* issue #7476 : fat-jar building for specific Spark version
* issue #7476 : confirmed 3.5.7 baselines because of Java 21 / Spark
* issue #7476 : apache-rat markdown exclusion for the support matrices
* issue #7476 : hop script should include the spark-client as well
* rebase and bump to spark 3.5.8
---------
Co-authored-by: Hans Van Akelyen <[email protected]>
---
assemblies/static/src/main/resources/hop | 15 +-
assemblies/static/src/main/resources/hop-conf.bat | 10 +-
assemblies/static/src/main/resources/hop-conf.sh | 17 +-
.../static/src/main/resources/hop-encrypt.bat | 10 +-
.../static/src/main/resources/hop-encrypt.sh | 17 +-
assemblies/static/src/main/resources/hop-gui.bat | 10 +-
assemblies/static/src/main/resources/hop-gui.sh | 11 +-
.../static/src/main/resources/hop-import.bat | 10 +-
assemblies/static/src/main/resources/hop-import.sh | 17 +-
assemblies/static/src/main/resources/hop-run.bat | 10 +-
assemblies/static/src/main/resources/hop-run.sh | 17 +-
.../static/src/main/resources/hop-search.bat | 10 +-
assemblies/static/src/main/resources/hop-search.sh | 17 +-
.../static/src/main/resources/hop-server.bat | 10 +-
assemblies/static/src/main/resources/hop-server.sh | 17 +-
assemblies/static/src/main/resources/hop.bat | 200 ++++++++--------
.../integration-tests-beam-base.yaml | 4 +
.../integration-tests/integration-tests-spark.yaml | 127 +++-------
docker/integration-tests/spark/Dockerfile.master | 60 +++--
docker/integration-tests/spark/Dockerfile.worker | 58 +++--
.../integration-tests/unit-tests-beam.Dockerfile | 13 +-
.../pipeline/beam/getting-started-with-beam.adoc | 12 +
.../beam-spark-pipeline-engine.adoc | 26 ++-
.../scripts/run-spark-matched-matrix.sh | 204 +++++++++++++++++
integration-tests/scripts/run-spark-matrix.sh | 255 +++++++++++++++++++++
integration-tests/scripts/run-tests-docker.sh | 85 ++++++-
.../spark/0001-test-spark-cluster.hpl | 2 +-
integration-tests/spark/MATRIX-REPORT-MATCHED.md | 45 ++++
integration-tests/spark/MATRIX-REPORT.md | 61 +++++
integration-tests/spark/disabled.txt | 18 --
.../{spark.json => spark-local.json} | 10 +-
.../metadata/pipeline-run-configuration/spark.json | 6 +-
....hwf => optional-0002-test-hive-connection.hwf} | 0
....hwf => optional-0003-test-hive-write-read.hwf} | 0
plugins/engines/beam/pom.xml | 171 ++++++++------
plugins/engines/beam/src/assembly/assembly.xml | 26 +++
.../beam/config/GenerateFatJarConfigPlugin.java | 42 +++-
.../org/apache/hop/beam/gui/HopBeamGuiPlugin.java | 130 ++++++++++-
.../engines/spark/BeamSparkPipelineEngineTest.java | 17 +-
pom.xml | 1 +
tools/spark-client-pack/materialize-pack.sh | 117 ++++++++++
tools/spark-client-pack/pom.xml | 125 ++++++++++
42 files changed, 1631 insertions(+), 382 deletions(-)
diff --git a/assemblies/static/src/main/resources/hop
b/assemblies/static/src/main/resources/hop
index ace42c87dc..567efb609f 100755
--- a/assemblies/static/src/main/resources/hop
+++ b/assemblies/static/src/main/resources/hop
@@ -86,21 +86,28 @@ Linux)
export GDK_BACKEND=x11
fi
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/$(uname -m)/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/$(uname
-m)/*"
fi
;;
Darwin)
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/x86_64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/x86_64/*"
fi
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
esac
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+# Used by hop run (and other subcommands) the same as hop-run.sh.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
+
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.hop.Hop "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-conf.bat
b/assemblies/static/src/main/resources/hop-conf.bat
index b9268d7103..e33f891e3e 100644
--- a/assemblies/static/src/main/resources/hop-conf.bat
+++ b/assemblies/static/src/main/resources/hop-conf.bat
@@ -22,7 +22,15 @@ REM switch to script directory
cd /D %~dp0
set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
:NormalStart
REM set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
diff --git a/assemblies/static/src/main/resources/hop-conf.sh
b/assemblies/static/src/main/resources/hop-conf.sh
index d6e9cd0f72..b279b2ad93 100755
--- a/assemblies/static/src/main/resources/hop-conf.sh
+++ b/assemblies/static/src/main/resources/hop-conf.sh
@@ -78,22 +78,31 @@ Linux)
if [ "${XDG_SESSION_TYPE}" = "wayland" ]; then
export GDK_BACKEND=x11
fi
+
+
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/$(uname -m)/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/$(uname
-m)/*"
fi
;;
Darwin)
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/x86_64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/x86_64/*"
fi
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
esac
+
+
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.config.HopConfig "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-encrypt.bat
b/assemblies/static/src/main/resources/hop-encrypt.bat
index d57cfc86fe..a5877064c0 100644
--- a/assemblies/static/src/main/resources/hop-encrypt.bat
+++ b/assemblies/static/src/main/resources/hop-encrypt.bat
@@ -23,7 +23,15 @@ REM switch to script directory
cd /D %~dp0
set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
:NormalStart
REM set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
diff --git a/assemblies/static/src/main/resources/hop-encrypt.sh
b/assemblies/static/src/main/resources/hop-encrypt.sh
index c97b43da30..b89a53aded 100755
--- a/assemblies/static/src/main/resources/hop-encrypt.sh
+++ b/assemblies/static/src/main/resources/hop-encrypt.sh
@@ -78,22 +78,31 @@ Linux)
if [ "${XDG_SESSION_TYPE}" = "wayland" ]; then
export GDK_BACKEND=x11
fi
+
+
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/$(uname -m)/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/$(uname
-m)/*"
fi
;;
Darwin)
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/x86_64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/x86_64/*"
fi
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
esac
+
+
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -classpath "${CLASSPATH}"
org.apache.hop.encryption.HopEncrypt "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-gui.bat
b/assemblies/static/src/main/resources/hop-gui.bat
index bdd789d498..fa830f4630 100755
--- a/assemblies/static/src/main/resources/hop-gui.bat
+++ b/assemblies/static/src/main/resources/hop-gui.bat
@@ -25,7 +25,15 @@ REM Option to change the Characterset of the Windows Shell
to show foreign carac
if not "%HOP_WINDOWS_SHELL_ENCODING%"=="" chcp %HOP_WINDOWS_SHELL_ENCODING%
set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
set _temphelp=0
if [%1]==[help] set _temphelp=1
diff --git a/assemblies/static/src/main/resources/hop-gui.sh
b/assemblies/static/src/main/resources/hop-gui.sh
index ef3d194ae0..69f3c6a4b8 100755
--- a/assemblies/static/src/main/resources/hop-gui.sh
+++ b/assemblies/static/src/main/resources/hop-gui.sh
@@ -110,8 +110,17 @@ Darwin)
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
esac
-CLASSPATH="lib/core/*:lib/beam/*:lib/swt/$os_path/$arch_path/*"
+CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/$os_path/$arch_path/*"
+
+
+
+
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.ui.hopgui.HopGui "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-import.bat
b/assemblies/static/src/main/resources/hop-import.bat
index 55c5917e22..582dcdb396 100644
--- a/assemblies/static/src/main/resources/hop-import.bat
+++ b/assemblies/static/src/main/resources/hop-import.bat
@@ -23,7 +23,15 @@ REM switch to script directory
cd /D %~dp0
set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
:NormalStart
REM set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
diff --git a/assemblies/static/src/main/resources/hop-import.sh
b/assemblies/static/src/main/resources/hop-import.sh
index dd47db757f..5669e164e5 100755
--- a/assemblies/static/src/main/resources/hop-import.sh
+++ b/assemblies/static/src/main/resources/hop-import.sh
@@ -78,22 +78,31 @@ Linux)
if [ "${XDG_SESSION_TYPE}" = "wayland" ]; then
export GDK_BACKEND=x11
fi
+
+
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/$(uname -m)/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/$(uname
-m)/*"
fi
;;
Darwin)
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/x86_64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/x86_64/*"
fi
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
esac
+
+
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.imp.HopImport "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-run.bat
b/assemblies/static/src/main/resources/hop-run.bat
index 1dd10edc50..424a2737da 100755
--- a/assemblies/static/src/main/resources/hop-run.bat
+++ b/assemblies/static/src/main/resources/hop-run.bat
@@ -25,7 +25,15 @@ REM Option to change the Characterset of the Windows Shell
to show foreign carac
if not "%HOP_WINDOWS_SHELL_ENCODING%"=="" chcp %HOP_WINDOWS_SHELL_ENCODING%
set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
set _temphelp=0
if [%1]==[help] set _temphelp=1
diff --git a/assemblies/static/src/main/resources/hop-run.sh
b/assemblies/static/src/main/resources/hop-run.sh
index 8959684f67..f25c19bac8 100755
--- a/assemblies/static/src/main/resources/hop-run.sh
+++ b/assemblies/static/src/main/resources/hop-run.sh
@@ -78,22 +78,31 @@ Linux)
if [ "${XDG_SESSION_TYPE}" = "wayland" ]; then
export GDK_BACKEND=x11
fi
+
+
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/$(uname -m)/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/$(uname
-m)/*"
fi
;;
Darwin)
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/x86_64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/x86_64/*"
fi
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
esac
+
+
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.run.HopRun "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-search.bat
b/assemblies/static/src/main/resources/hop-search.bat
index 360a3df0ee..be6d8e37a5 100644
--- a/assemblies/static/src/main/resources/hop-search.bat
+++ b/assemblies/static/src/main/resources/hop-search.bat
@@ -22,7 +22,15 @@ REM switch to script directory
cd /D %~dp0
set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
:NormalStart
REM set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
diff --git a/assemblies/static/src/main/resources/hop-search.sh
b/assemblies/static/src/main/resources/hop-search.sh
index 00282a0a70..6bba102dea 100755
--- a/assemblies/static/src/main/resources/hop-search.sh
+++ b/assemblies/static/src/main/resources/hop-search.sh
@@ -78,22 +78,31 @@ Linux)
if [ "${XDG_SESSION_TYPE}" = "wayland" ]; then
export GDK_BACKEND=x11
fi
+
+
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/$(uname -m)/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/$(uname
-m)/*"
fi
;;
Darwin)
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/x86_64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/x86_64/*"
fi
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
esac
+
+
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.search.HopSearch "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-server.bat
b/assemblies/static/src/main/resources/hop-server.bat
index 1d39aa1527..40227e5c65 100644
--- a/assemblies/static/src/main/resources/hop-server.bat
+++ b/assemblies/static/src/main/resources/hop-server.bat
@@ -25,7 +25,15 @@ REM Option to change the Characterset of the Windows Shell
to show foreign carac
if not "%HOP_WINDOWS_SHELL_ENCODING%"=="" chcp %HOP_WINDOWS_SHELL_ENCODING%
set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
:NormalStart
REM set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
diff --git a/assemblies/static/src/main/resources/hop-server.sh
b/assemblies/static/src/main/resources/hop-server.sh
index 02604432c2..06cedd3d4a 100755
--- a/assemblies/static/src/main/resources/hop-server.sh
+++ b/assemblies/static/src/main/resources/hop-server.sh
@@ -78,17 +78,19 @@ Linux)
if [ "${XDG_SESSION_TYPE}" = "wayland" ]; then
export GDK_BACKEND=x11
fi
+
+
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/linux/$(uname -m)/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/linux/$(uname
-m)/*"
fi
;;
Darwin)
if "${_HOP_JAVA}" -XshowSettings:properties -version 2>&1 | grep -q "os.arch
= aarch64"; then
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/arm64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/arm64/*"
else
- CLASSPATH="lib/core/*:lib/beam/*:lib/swt/osx/x86_64/*"
+ CLASSPATH="lib/core/*:lib/beam/*:lib/spark-client/*:lib/swt/osx/x86_64/*"
fi
HOP_OPTIONS="${HOP_OPTIONS} -XstartOnFirstThread"
;;
@@ -99,6 +101,13 @@ if [ "${JAAS_LOGIN_MODULE_CONFIG}" != "" ] && [
"${JAAS_LOGIN_MODULE_NAME}" != "
HOP_OPTIONS="${HOP_OPTIONS} -Dloginmodulename=${JAAS_LOGIN_MODULE_NAME}"
fi
+
+
+# Optional versioned Spark client pack replaces default lib/spark-client on
the classpath.
+if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}" ]; then
+ CLASSPATH=$(echo "${CLASSPATH}" | sed 's|lib/spark-client/\*||g')
+ CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
+fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.www.HopServer "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop.bat
b/assemblies/static/src/main/resources/hop.bat
index af3dd79b3a..635a87f575 100755
--- a/assemblies/static/src/main/resources/hop.bat
+++ b/assemblies/static/src/main/resources/hop.bat
@@ -1,96 +1,104 @@
-@echo off
-
-REM
-REM Licensed to the Apache Software Foundation (ASF) under one or more
-REM contributor license agreements. See the NOTICE file distributed with
-REM this work for additional information regarding copyright ownership.
-REM The ASF licenses this file to You under the Apache License, Version 2.0
-REM (the "License"); you may not use this file except in compliance with
-REM the License. You may obtain a copy of the License at
-REM
-REM http://www.apache.org/licenses/LICENSE-2.0
-REM
-REM Unless required by applicable law or agreed to in writing, software
-REM distributed under the License is distributed on an "AS IS" BASIS,
-REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-REM See the License for the specific language governing permissions and
-REM limitations under the License.
-REM
-
-setlocal EnableDelayedExpansion
-
-REM switch to script directory
-cd /D %~dp0
-
-REM Option to change the character set of the Windows Shell to show foreign
characters
-if not "%HOP_WINDOWS_SHELL_ENCODING%"=="" chcp %HOP_WINDOWS_SHELL_ENCODING%
-
-set LIBSPATH=lib\core;lib\beam
-set CLASSPATH=lib\core\*;lib\beam\*;lib\swt\win64\*
-
-:NormalStart
-REM set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
-REM
-if not "%HOP_JAVA_HOME%"=="" (
- set _HOP_JAVA="%HOP_JAVA_HOME%\bin\java"
-) else if not "%JAVA_HOME%"=="" (
- set _HOP_JAVA="%JAVA_HOME%\bin\java"
-) else (
- set _HOP_JAVA="java"
-)
-
-REM # Settings for all OSses
-REM
-
-if "%HOP_OPTIONS%"=="" set HOP_OPTIONS="-Xmx2048m"
-
-REM See if we need to enable some remote debugging options for our developers.
-REM
-FOR %%a in (%*) DO (
- if "%%~a" == "--dev-debug" (
- set HOP_OPTIONS=%HOP_OPTIONS% -Xdebug -Xnoagent
-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5010
- )
- if "%%~a" == "--dev-debug-wait" (
- set HOP_OPTIONS=%HOP_OPTIONS% -Xdebug -Xnoagent
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5010
- )
-)
-
-REM Pass HOP variables if they're set.
-REM
-if not "%HOP_AUDIT_FOLDER%"=="" (
- set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AUDIT_FOLDER="%HOP_AUDIT_FOLDER%"
-) else (
- set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AUDIT_FOLDER=.\audit
-)
-if not "%HOP_CONFIG_FOLDER%"=="" (
- set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_CONFIG_FOLDER="%HOP_CONFIG_FOLDER%"
-)
-if not "%HOP_SHARED_JDBC_FOLDERS%"=="" (
- set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_SHARED_JDBC_FOLDERS="%HOP_SHARED_JDBC_FOLDERS%"
-)
-if not "%HOP_PLUGIN_BASE_FOLDERS%"=="" (
- set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_PLUGIN_BASE_FOLDERS="%HOP_PLUGIN_BASE_FOLDERS%"
-)
-if not "%HOP_PASSWORD_ENCODER_PLUGIN%"=="" (
- set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_PASSWORD_ENCODER_PLUGIN=%HOP_PASSWORD_ENCODER_PLUGIN%
-)
-if not "%HOP_AES_ENCODER_KEY%"=="" (
- set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AES_ENCODER_KEY=%HOP_AES_ENCODER_KEY%
-)
-if not "%HOP_AES_ENCODER_KEY_FILE%"=="" (
- set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_AES_ENCODER_KEY_FILE=%HOP_AES_ENCODER_KEY_FILE%
-)
-
-set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_PLATFORM_OS=Windows
-set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_PLATFORM_RUNTIME=Run
-set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AUTO_CREATE_CONFIG=Y
-set HOP_OPTIONS=%HOP_OPTIONS% --add-opens
java.xml/jdk.xml.internal=ALL-UNNAMED --add-opens
java.base/java.lang=ALL-UNNAMED --add-opens
java.base/java.lang.invoke=ALL-UNNAMED --add-opens
java.base/java.lang.reflect=ALL-UNNAMED --add-opens
java.base/java.io=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED
--add-opens java.base/java.nio=ALL-UNNAMED --add-opens
java.base/java.util=ALL-UNNAMED --add-opens
java.base/java.util.concurrent=ALL-UNNAMED --add-opens
java.base/java.util.concur [...]
-
-REM ===[Collect command line
arguments...]======================================
-REM
-set _cmdline=%*
-
-:Run
-
-%_HOP_JAVA% -classpath %CLASSPATH% -Djava.library.path=%LIBSPATH%
%HOP_OPTIONS% org.apache.hop.hop.Hop %_cmdline%%
-
+@echo off
+
+REM
+REM Licensed to the Apache Software Foundation (ASF) under one or more
+REM contributor license agreements. See the NOTICE file distributed with
+REM this work for additional information regarding copyright ownership.
+REM The ASF licenses this file to You under the Apache License, Version 2.0
+REM (the "License"); you may not use this file except in compliance with
+REM the License. You may obtain a copy of the License at
+REM
+REM http://www.apache.org/licenses/LICENSE-2.0
+REM
+REM Unless required by applicable law or agreed to in writing, software
+REM distributed under the License is distributed on an "AS IS" BASIS,
+REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+REM See the License for the specific language governing permissions and
+REM limitations under the License.
+REM
+
+setlocal EnableDelayedExpansion
+
+REM switch to script directory
+cd /D %~dp0
+
+REM Option to change the character set of the Windows Shell to show foreign
characters
+if not "%HOP_WINDOWS_SHELL_ENCODING%"=="" chcp %HOP_WINDOWS_SHELL_ENCODING%
+
+set LIBSPATH=lib\core;lib\beam
+set CLASSPATH=lib\core\*;lib\beam\*;lib\spark-client\*;lib\swt\win64\*
+
+REM Optional versioned Spark client pack
+if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\" (
+ set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
+)
+
+
+
+
+:NormalStart
+REM set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
+REM
+if not "%HOP_JAVA_HOME%"=="" (
+ set _HOP_JAVA="%HOP_JAVA_HOME%\bin\java"
+) else if not "%JAVA_HOME%"=="" (
+ set _HOP_JAVA="%JAVA_HOME%\bin\java"
+) else (
+ set _HOP_JAVA="java"
+)
+
+REM # Settings for all OSses
+REM
+
+if "%HOP_OPTIONS%"=="" set HOP_OPTIONS="-Xmx2048m"
+
+REM See if we need to enable some remote debugging options for our developers.
+REM
+FOR %%a in (%*) DO (
+ if "%%~a" == "--dev-debug" (
+ set HOP_OPTIONS=%HOP_OPTIONS% -Xdebug -Xnoagent
-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5010
+ )
+ if "%%~a" == "--dev-debug-wait" (
+ set HOP_OPTIONS=%HOP_OPTIONS% -Xdebug -Xnoagent
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5010
+ )
+)
+
+REM Pass HOP variables if they're set.
+REM
+if not "%HOP_AUDIT_FOLDER%"=="" (
+ set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AUDIT_FOLDER="%HOP_AUDIT_FOLDER%"
+) else (
+ set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AUDIT_FOLDER=.\audit
+)
+if not "%HOP_CONFIG_FOLDER%"=="" (
+ set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_CONFIG_FOLDER="%HOP_CONFIG_FOLDER%"
+)
+if not "%HOP_SHARED_JDBC_FOLDERS%"=="" (
+ set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_SHARED_JDBC_FOLDERS="%HOP_SHARED_JDBC_FOLDERS%"
+)
+if not "%HOP_PLUGIN_BASE_FOLDERS%"=="" (
+ set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_PLUGIN_BASE_FOLDERS="%HOP_PLUGIN_BASE_FOLDERS%"
+)
+if not "%HOP_PASSWORD_ENCODER_PLUGIN%"=="" (
+ set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_PASSWORD_ENCODER_PLUGIN=%HOP_PASSWORD_ENCODER_PLUGIN%
+)
+if not "%HOP_AES_ENCODER_KEY%"=="" (
+ set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AES_ENCODER_KEY=%HOP_AES_ENCODER_KEY%
+)
+if not "%HOP_AES_ENCODER_KEY_FILE%"=="" (
+ set HOP_OPTIONS=%HOP_OPTIONS%
-DHOP_AES_ENCODER_KEY_FILE=%HOP_AES_ENCODER_KEY_FILE%
+)
+
+set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_PLATFORM_OS=Windows
+set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_PLATFORM_RUNTIME=Run
+set HOP_OPTIONS=%HOP_OPTIONS% -DHOP_AUTO_CREATE_CONFIG=Y
+set HOP_OPTIONS=%HOP_OPTIONS% --add-opens
java.xml/jdk.xml.internal=ALL-UNNAMED --add-opens
java.base/java.lang=ALL-UNNAMED --add-opens
java.base/java.lang.invoke=ALL-UNNAMED --add-opens
java.base/java.lang.reflect=ALL-UNNAMED --add-opens
java.base/java.io=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED
--add-opens java.base/java.nio=ALL-UNNAMED --add-opens
java.base/java.util=ALL-UNNAMED --add-opens
java.base/java.util.concurrent=ALL-UNNAMED --add-opens
java.base/java.util.concur [...]
+
+REM ===[Collect command line
arguments...]======================================
+REM
+set _cmdline=%*
+
+:Run
+
+%_HOP_JAVA% -classpath %CLASSPATH% -Djava.library.path=%LIBSPATH%
%HOP_OPTIONS% org.apache.hop.hop.Hop %_cmdline%%
+
diff --git a/docker/integration-tests/integration-tests-beam-base.yaml
b/docker/integration-tests/integration-tests-beam-base.yaml
index fb62f84eca..eb2537f4a0 100644
--- a/docker/integration-tests/integration-tests-beam-base.yaml
+++ b/docker/integration-tests/integration-tests-beam-base.yaml
@@ -25,8 +25,12 @@ services:
build:
context: ../../.
dockerfile: docker/integration-tests/unit-tests-beam.Dockerfile
+ args:
+ HOP_SPARK_CLIENT_VERSION: ${HOP_SPARK_CLIENT_VERSION:-}
volumes:
- ../../integration-tests/:/files
environment:
- FLASK_ENV=docker
+ # Keep driver Spark client pack in sync with the fat jar (empty =
default pack)
+ - HOP_SPARK_CLIENT_VERSION=${HOP_SPARK_CLIENT_VERSION:-}
command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]
diff --git a/docker/integration-tests/integration-tests-spark.yaml
b/docker/integration-tests/integration-tests-spark.yaml
index 871732ff0c..8b689d9a2d 100644
--- a/docker/integration-tests/integration-tests-spark.yaml
+++ b/docker/integration-tests/integration-tests-spark.yaml
@@ -15,39 +15,50 @@
# specific language governing permissions and limitations
# under the License.
+# Beam Spark smoke tests: Hop (Java 21) + standalone Spark master/worker.
+# SPARK_VERSION is overridable for the multi-version matrix (see
run-spark-matrix.sh).
+# Hive/HDFS services are intentionally omitted from the default smoke path;
re-add via a
+# separate compose file when the Hive catalog tests are revived.
+
services:
- integration_test_spark:
+ # Service name must be a valid Spark RPC hostname (no underscores).
+ hop-spark-test:
extends:
file: integration-tests-beam-base.yaml
service: integration_test
depends_on:
- - spark
- - spark-worker
- - namenode
- - datanode
- - datanode2
- - datanode3
- - resourcemanager
- - nodemanager
- - hiveserver
- - hive-metastore
+ spark:
+ condition: service_healthy
+ spark-worker:
+ condition: service_started
links:
- spark
- spark-worker
- - namenode
- - datanode
- - datanode2
- - datanode3
- - resourcemanager
- - nodemanager
- - hiveserver
- - hive-metastore
+ # Hop is always Java 21. Spark workers also run Java 21 so they can
execute Hop
+ # bytecode from the fat jar. Advertise this container as the Spark driver
so
+ # workers on the compose network can open reverse connections.
+ environment:
+ - FLASK_ENV=docker
+ - SPARK_LOCAL_IP=0.0.0.0
+ # Match driver Spark client pack to cluster (and to fat jar); empty =
default pack
+ - HOP_SPARK_CLIENT_VERSION=${HOP_SPARK_CLIENT_VERSION:-}
+ # spark.ui.enabled=false: Hop ships jakarta servlet APIs that clash with
Spark's
+ # javax-based UI stack (jersey ServletContainer). UI is not needed for
IT smoke.
+ - HOP_OPTIONS=-Dspark.ui.enabled=false
-Dspark.driver.host=hop-spark-test -Dspark.driver.bindAddress=0.0.0.0
spark:
build:
context: ../../docker/integration-tests/spark/.
dockerfile: Dockerfile.master
+ args:
+ SPARK_VERSION: ${SPARK_VERSION:-3.5.8}
+ HADOOP_VERSION: ${HADOOP_VERSION:-3}
+ # Fast CDN by default (current patch); the Dockerfile falls back to
archive.apache.org
+ # for historical matrix versions that the CDN no longer carries.
+ BASE_URL: ${SPARK_BASE_URL:-https://dlcdn.apache.org/spark}
environment:
- INIT_DAEMON_STEP=setup_spark
+ # Advertise a stable compose-network hostname (not the random container
id)
+ - SPARK_MASTER_HOST=spark
ports:
- 8080
- 7077
@@ -55,80 +66,14 @@ services:
build:
context: ../../docker/integration-tests/spark/.
dockerfile: Dockerfile.worker
+ args:
+ SPARK_VERSION: ${SPARK_VERSION:-3.5.8}
+ HADOOP_VERSION: ${HADOOP_VERSION:-3}
+ BASE_URL: ${SPARK_BASE_URL:-https://dlcdn.apache.org/spark}
depends_on:
- - spark
+ spark:
+ condition: service_healthy
environment:
- "SPARK_MASTER=spark://spark:7077"
ports:
- 8081
- namenode:
- build:
- dockerfile: Dockerfile.hadoop
- context: ../../docker/integration-tests/spark/.
- hostname: namenode
- command: ["hdfs", "namenode"]
- ports:
- - 9870
- - 8020
- env_file:
- - ./spark/config
- environment:
- ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
- datanode:
- build:
- dockerfile: Dockerfile.hadoop
- context: ../../docker/integration-tests/spark/.
- command: ["hdfs", "datanode"]
- env_file:
- - ./spark/config
- datanode2:
- build:
- dockerfile: Dockerfile.hadoop
- context: ../../docker/integration-tests/spark/.
- command: ["hdfs", "datanode"]
- env_file:
- - ./spark/config
- datanode3:
- build:
- dockerfile: Dockerfile.hadoop
- context: ../../docker/integration-tests/spark/.
- command: ["hdfs", "datanode"]
- env_file:
- - ./spark/config
- resourcemanager:
- build:
- dockerfile: Dockerfile.hadoop
- context: ../../docker/integration-tests/spark/.
- hostname: resourcemanager
- command: ["yarn", "resourcemanager"]
- ports:
- - 8088
- env_file:
- - ./spark/config
- volumes:
- - ./test.sh:/opt/test.sh
- nodemanager:
- build:
- dockerfile: Dockerfile.hadoop
- context: ../../docker/integration-tests/spark/.
- command: ["yarn", "nodemanager"]
- env_file:
- - ./spark/config
- hiveserver:
- image: apache/hive:4.0.0-beta-1
- ports:
- - 10000
- - 10002
- depends_on:
- - hive-metastore
- environment:
- - SERVICE_NAME=hiveserver2
- - SERVICE_OPTS=-Dhive.metastore.uris=thrift://hive-metastore:9083
- volumes:
- - ./spark/hive-site.xml:/opt/hive/conf/hive-site.xml
- hive-metastore:
- image: apache/hive:4.0.0-beta-1
- ports:
- - 9083
- environment:
- - SERVICE_NAME=metastore
diff --git a/docker/integration-tests/spark/Dockerfile.master
b/docker/integration-tests/spark/Dockerfile.master
index 20f4149885..2caae28106 100644
--- a/docker/integration-tests/spark/Dockerfile.master
+++ b/docker/integration-tests/spark/Dockerfile.master
@@ -14,36 +14,54 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM alpine:3.10
-ENV ENABLE_INIT_DAEMON false
-ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon
-ENV INIT_DAEMON_STEP spark_master_init
+# Spark standalone master for Hop Beam integration tests.
+# SPARK_VERSION is parameterized so the same image definition can matrix-test
Spark 3.x lines.
+# Java 21 matches Hop's target bytecode (workers execute Hop transforms from
the fat jar).
+FROM eclipse-temurin:21-jre-jammy
-ENV BASE_URL=https://dlcdn.apache.org/spark/
-ENV SPARK_VERSION=3.4.1
-ENV HADOOP_VERSION=3
-ENV SPARK_MASTER_PORT 7077
-ENV SPARK_MASTER_WEBUI_PORT 8080
-ENV SPARK_MASTER_LOG /spark/logs
+ARG SPARK_VERSION=3.5.8
+ARG HADOOP_VERSION=3
+# dlcdn.apache.org serves the current patch of each active line at CDN speed;
archive.apache.org
+# keeps historical releases (needed for the multi-version matrix) but
throttles hard (~100 KB/s).
+# We try BASE_URL first and fall back to ARCHIVE_URL, so the default pin stays
fast while older
+# matrix versions (not on the CDN) still resolve.
+ARG BASE_URL=https://dlcdn.apache.org/spark
+ARG ARCHIVE_URL=https://archive.apache.org/dist/spark
+
+ENV ENABLE_INIT_DAEMON=false
+ENV INIT_DAEMON_BASE_URI=http://identifier/init-daemon
+ENV INIT_DAEMON_STEP=spark_master_init
+ENV SPARK_VERSION=${SPARK_VERSION}
+ENV HADOOP_VERSION=${HADOOP_VERSION}
+ENV SPARK_HOME=/spark
+ENV SPARK_MASTER_PORT=7077
+ENV SPARK_MASTER_WEBUI_PORT=8080
+ENV SPARK_MASTER_LOG=/spark/logs
+ENV PATH="${SPARK_HOME}/bin:${PATH}"
COPY ./scripts/wait-for-step.sh /
COPY ./scripts/execute-step.sh /
COPY ./scripts/finish-step.sh /
COPY ./scripts/master.sh /
-RUN apk add --no-cache curl bash openjdk11-jre nss libc6-compat coreutils
procps \
- && ln -s /lib64/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 \
- && chmod +x *.sh \
- && wget
${BASE_URL}/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
\
- && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
- && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark \
- && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
- && cd /
-
-#Give permission to execute scripts
-RUN chmod +x /wait-for-step.sh && chmod +x /execute-step.sh && chmod +x
/finish-step.sh && chmod +x /master.sh
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl bash procps \
+ && rm -rf /var/lib/apt/lists/* \
+ && chmod +x /wait-for-step.sh /execute-step.sh /finish-step.sh
/master.sh \
+ && SPARK_TGZ="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" \
+ && ( curl -fSL --connect-timeout 30 --retry 3 --retry-delay 5
--retry-all-errors \
+ --speed-limit 10240 --speed-time 60 \
+ "${BASE_URL}/spark-${SPARK_VERSION}/${SPARK_TGZ}" -o
/tmp/spark.tgz \
+ || curl -fSL --connect-timeout 30 --retry 5 --retry-delay 5
--retry-all-errors \
+ "${ARCHIVE_URL}/spark-${SPARK_VERSION}/${SPARK_TGZ}" -o
/tmp/spark.tgz ) \
+ && tar -xzf /tmp/spark.tgz -C / \
+ && mv /spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} /spark \
+ && rm /tmp/spark.tgz
EXPOSE 8080 7077 6066
+HEALTHCHECK --interval=5s --timeout=5s --start-period=30s --retries=24 \
+ CMD curl -fsS "http://127.0.0.1:8080/"; >/dev/null || exit 1
+
CMD ["/bin/bash", "/master.sh"]
diff --git a/docker/integration-tests/spark/Dockerfile.worker
b/docker/integration-tests/spark/Dockerfile.worker
index 56d5fffc18..eb6ec13116 100644
--- a/docker/integration-tests/spark/Dockerfile.worker
+++ b/docker/integration-tests/spark/Dockerfile.worker
@@ -15,35 +15,47 @@
# limitations under the License.
#
-FROM alpine:3.10
-
-ENV ENABLE_INIT_DAEMON false
-ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon
-ENV INIT_DAEMON_STEP spark_master_init
-
-ENV BASE_URL=https://dlcdn.apache.org/spark/
-ENV SPARK_VERSION=3.4.1
-ENV HADOOP_VERSION=3
-ENV SPARK_WORKER_WEBUI_PORT 8081
-ENV SPARK_WORKER_LOG /spark/logs
-ENV SPARK_MASTER "spark://spark:7077"
+# Spark standalone worker for Hop Beam integration tests.
+# Keep SPARK_VERSION in sync with Dockerfile.master (matrix-tested via compose
build args).
+# Java 21 matches Hop's target bytecode (workers execute Hop transforms from
the fat jar).
+FROM eclipse-temurin:21-jre-jammy
+
+ARG SPARK_VERSION=3.5.8
+ARG HADOOP_VERSION=3
+# Fast CDN first (current patch only), archive fallback for historical matrix
versions.
+# Keep in sync with Dockerfile.master.
+ARG BASE_URL=https://dlcdn.apache.org/spark
+ARG ARCHIVE_URL=https://archive.apache.org/dist/spark
+
+ENV ENABLE_INIT_DAEMON=false
+ENV INIT_DAEMON_BASE_URI=http://identifier/init-daemon
+ENV INIT_DAEMON_STEP=spark_master_init
+ENV SPARK_VERSION=${SPARK_VERSION}
+ENV HADOOP_VERSION=${HADOOP_VERSION}
+ENV SPARK_HOME=/spark
+ENV SPARK_WORKER_WEBUI_PORT=8081
+ENV SPARK_WORKER_LOG=/spark/logs
+ENV SPARK_MASTER=spark://spark:7077
+ENV PATH="${SPARK_HOME}/bin:${PATH}"
COPY ./scripts/wait-for-step.sh /
COPY ./scripts/execute-step.sh /
COPY ./scripts/finish-step.sh /
COPY ./scripts/worker.sh /
-RUN apk add --no-cache curl bash openjdk11-jre nss libc6-compat coreutils
procps \
- && ln -s /lib64/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 \
- && chmod +x *.sh \
- && wget
${BASE_URL}/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
\
- && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
- && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark \
- && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
- && cd /
-
-#Give permission to execute scripts
-RUN chmod +x /wait-for-step.sh && chmod +x /execute-step.sh && chmod +x
/finish-step.sh && chmod +x /worker.sh
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl bash procps \
+ && rm -rf /var/lib/apt/lists/* \
+ && chmod +x /wait-for-step.sh /execute-step.sh /finish-step.sh
/worker.sh \
+ && SPARK_TGZ="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" \
+ && ( curl -fSL --connect-timeout 30 --retry 3 --retry-delay 5
--retry-all-errors \
+ --speed-limit 10240 --speed-time 60 \
+ "${BASE_URL}/spark-${SPARK_VERSION}/${SPARK_TGZ}" -o
/tmp/spark.tgz \
+ || curl -fSL --connect-timeout 30 --retry 5 --retry-delay 5
--retry-all-errors \
+ "${ARCHIVE_URL}/spark-${SPARK_VERSION}/${SPARK_TGZ}" -o
/tmp/spark.tgz ) \
+ && tar -xzf /tmp/spark.tgz -C / \
+ && mv /spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} /spark \
+ && rm /tmp/spark.tgz
EXPOSE 8081
diff --git a/docker/integration-tests/unit-tests-beam.Dockerfile
b/docker/integration-tests/unit-tests-beam.Dockerfile
index e807016997..b1b97c14b0 100644
--- a/docker/integration-tests/unit-tests-beam.Dockerfile
+++ b/docker/integration-tests/unit-tests-beam.Dockerfile
@@ -22,10 +22,21 @@
# ~700MB file, so it is deliberately kept out of the shared base image
(unit-tests.Dockerfile) and
# built here instead. This image is only built/used when a project that
references the fat jar
# actually runs (see integration-tests-beam-base.yaml and
scripts/run-tests-docker.sh).
+#
+# HOP_SPARK_CLIENT_VERSION: when set, the fat jar embeds that Spark client pack
+# (lib/spark-clients/<version>/), matching hop-run's driver classpath
selection.
FROM hop-base-image
+ARG HOP_SPARK_CLIENT_VERSION=
+ENV HOP_SPARK_CLIENT_VERSION=${HOP_SPARK_CLIENT_VERSION}
+
# Runs as the (already configured) hop/jenkins user inherited from the base
image, which owns the
# Hop installation under ${DEPLOYMENT_PATH}/hop.
RUN cd ${DEPLOYMENT_PATH}/hop \
- && ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar
+ && if [ -n "${HOP_SPARK_CLIENT_VERSION}" ]; then \
+ ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar \
+ --spark-client-version="${HOP_SPARK_CLIENT_VERSION}"; \
+ else \
+ ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar; \
+ fi
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
index 0b61f1b47f..ec471eb3ed 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
@@ -49,9 +49,21 @@ Here is the documentation for the relevant plugins:
[[supportedversions]]
== What software versions are supported
+[NOTE]
+====
+*Java:* Hop requires *Java 21* for the Hop process (GUI, hop-run, hop-server)
and for the fat jar executed on Beam workers. Spark/Flink clusters must run a
JVM that can load that bytecode (Java 21).
+
+*Spark (measured):* With Hop 2.19 / Beam 2.74 on *Java 21*, only *Spark 3.5.x*
is supported for client-mode submit (`spark://…`). Spark 3.4 and earlier fail
on Java 21 with `NoSuchMethodException:
java.nio.DirectByteBuffer.<init>(long,int)`
([SPARK-42369](https://issues.apache.org/jira/browse/SPARK-42369), fixed in
Spark 3.5.0). Matching client packs to the cluster does not overcome that JDK
limit. See `integration-tests/spark/MATRIX-REPORT.md`. Beam upstream docs may
still mention 3.2.x [...]
+====
+
|===
|Hop version |Beam version |Spark version |Flink version
+|2.19.0
+|2.74.0
+|3.5.x (scala 2.12; pinned 3.5.8, matrix: 3.5.7 PASS)
+|1.19.x
+
|2.18.0
|2.71.0
|3.5.X (scala 2.12)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
index d296bd56b9..44784704cc 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
@@ -24,7 +24,31 @@ under the License.
== Beam Spark
-The Apache Spark Runner can be used to execute Beam pipelines using Apache
Spark version 3.1.
+The Apache Spark Runner can be used to execute Beam pipelines using Apache
Spark.
+
+IMPORTANT: Hop itself always runs on *Java 21* (driver and fat-jar bytecode).
Spark standalone clusters used with Hop must also provide a Java 21 runtime so
workers can execute Hop transforms.
+
+*Measured support (Hop 2.19 / Beam 2.74, client submit):* *Spark 3.5.x* only
(validated *3.5.8*). Hop requires *Java 21*; Spark did not handle the JDK 21
`DirectByteBuffer` constructor change until
[SPARK-42369](https://issues.apache.org/jira/browse/SPARK-42369) (fixed in
*Spark 3.5.0*). Therefore *Spark 3.4.x and earlier cannot run* with Hop on Java
21 — even with a matching client pack and fat jar.
+
+For `spark://master` client mode on 3.5.x, the **Spark client libraries on the
Hop driver and in the fat jar should still match the cluster patch/minor**
(default pack under `lib/spark-client/`, currently 3.5.8).
+
+=== Matching Spark client libraries (3.5.x)
+
+Hop installs a default Spark client pack under `lib/spark-client/` (beam
engine `spark.version`). Optional packs under `lib/spark-clients/<version>/`
are for aligning with a *3.5.x* cluster build, not for downgrading below 3.5.
+
+.Materialise / select a 3.5.x pack
+[source,bash]
+----
+./tools/spark-client-pack/materialize-pack.sh 3.5.8 /path/to/hop
+export HOP_SPARK_CLIENT_VERSION=3.5.8
+./hop-conf.sh --generate-fat-jar=/path/to/hop-spark-3.5.8.jar \
+ --spark-client-version=3.5.8
+./hop-run.sh -r spark -f your-pipeline.hpl
+----
+
+Point the pipeline run configuration *Fat jar* field at the version-matched
jar. Integration evidence: `integration-tests/spark/MATRIX-REPORT.md` and
`MATRIX-REPORT-MATCHED.md`.
+
+Beam's upstream documentation may still pin the Spark runner to an older line
(historically 3.2.x). Prefer the
xref:pipeline/beam/getting-started-with-beam.adoc#supportedversions[supported
versions table] and the matrix reports over assumptions.
The Spark Runner executes Beam pipelines on top of Apache Spark, providing:
diff --git a/integration-tests/scripts/run-spark-matched-matrix.sh
b/integration-tests/scripts/run-spark-matched-matrix.sh
new file mode 100755
index 0000000000..5912e7ffa5
--- /dev/null
+++ b/integration-tests/scripts/run-spark-matched-matrix.sh
@@ -0,0 +1,204 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Matched-pair Spark matrix: for each version V, use Spark client pack V +
cluster V.
+#
+# Unlike run-spark-matrix.sh (fixed Hop client 3.5.8 vs varying cluster), this
script:
+# 1. Materialises lib/spark-clients/<V>/ into the Hop install
+# 2. Rebuilds hop-base + hop-beam images so the fat jar embeds pack V
+# 3. Runs the smoke suite with HOP_SPARK_CLIENT_VERSION=V and SPARK_VERSION=V
+#
+# Prerequisites: assemblies/client package built (hop-client zip present).
+#
+# Usage:
+# ./integration-tests/scripts/run-spark-matched-matrix.sh KEEP_IMAGES=true
+# SPARK_VERSIONS="3.5.8 3.4.4"
./integration-tests/scripts/run-spark-matched-matrix.sh
+
+set -u
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+REPO_ROOT="$(cd "${CURRENT_DIR}/../.." >/dev/null 2>&1 && pwd)"
+REPORT_DIR="${CURRENT_DIR}/../surefire-reports"
+COMMITTED_REPORT="${CURRENT_DIR}/../spark/MATRIX-REPORT-MATCHED.md"
+MATRIX_REPORT="${REPORT_DIR}/spark-matched-matrix-report.md"
+RUN_TESTS="${CURRENT_DIR}/run-tests-docker.sh"
+MATERIALIZE="${REPO_ROOT}/tools/spark-client-pack/materialize-pack.sh"
+DOCKER_FILES_DIR="$(cd "${CURRENT_DIR}/../../docker/integration-tests"
>/dev/null 2>&1 && pwd)"
+HOP_HOME="${REPO_ROOT}/assemblies/client/target/hop"
+
+if [ -z "${SPARK_VERSIONS:-}" ]; then
+ # Default: current 3.5.x pin. Pre-3.5 fails on Java 21 (SPARK-42369) even
when matched.
+ # Override to re-document the ceiling: SPARK_VERSIONS="3.5.8 3.4.4"
+ SPARK_VERSIONS="3.5.8"
+fi
+
+FORWARD_ARGS=("$@")
+mkdir -p "${REPORT_DIR}"
+
+GENERATED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+BEAM_VERSION=$(
+ grep -m1 'apache-beam.version' "${REPO_ROOT}/pom.xml" |
+ sed -E 's/.*<apache-beam.version>([^<]+)<\/apache-beam.version>.*/\1/' ||
true
+)
+
+hadoop_version_for_spark() {
+ case "$1" in
+ 3.0.* | 3.1.* | 3.2.*) echo "3.2" ;;
+ *) echo "3" ;;
+ esac
+}
+
+evaluate_result() {
+ local report_file="$1"
+ RESULT="FAIL"
+ NOTES="—"
+ if [ ! -f "${report_file}" ]; then
+ NOTES="no surefire report"
+ return
+ fi
+ if grep -q '<failure' "${report_file}"; then
+ if grep -q 'environment_setup' "${report_file}"; then
+ NOTES="docker/environment failed"
+ elif grep -q 'InvalidClassException\|serialVersionUID' "${report_file}";
then
+ NOTES="serialVersionUID skew (client/cluster still mismatched?)"
+ elif grep -q 'DirectByteBuffer' "${report_file}"; then
+ NOTES="Java 21 DirectByteBuffer incompatibility"
+ elif grep -q 'stopped SparkContext' "${report_file}"; then
+ NOTES="stopped SparkContext"
+ elif grep -q 'ERROR:' "${report_file}"; then
+ NOTES="pipeline ERROR"
+ else
+ NOTES="test failed"
+ fi
+ return
+ fi
+ if ! grep -q '<testcase ' "${report_file}"; then
+ NOTES="empty report"
+ return
+ fi
+ RESULT="PASS"
+ if grep -q 'Beam pipeline execution has finished' "${report_file}"; then
+ NOTES="matched pack+cluster smoke finished"
+ else
+ NOTES="hop-run exit 0"
+ fi
+}
+
+# Ensure hop tree exists (same unzip as run-tests-docker)
+if [ ! -d "${HOP_HOME}" ]; then
+ echo "Extracting hop client zip into ${HOP_HOME} ..."
+ unzip -o -q "${REPO_ROOT}/assemblies/client/target/"*.zip -d
"${REPO_ROOT}/assemblies/client/target/"
+fi
+
+{
+ echo "# Hop Beam Spark *matched* client/cluster matrix"
+ echo
+ echo "Generated: ${GENERATED_AT}"
+ echo
+ echo "- Hop: \`2.19.0-SNAPSHOT\` · Beam: \`${BEAM_VERSION}\`"
+ echo "- Each cell: **Spark client pack V** = **cluster V** (Java 21 driver +
cluster)"
+ echo "- Suite: \`main-0001-test-spark-cluster\`"
+ echo
+ echo "| Client pack | Cluster | Result | Notes |"
+ echo "| --- | --- | --- | --- |"
+} >"${MATRIX_REPORT}"
+
+OVERALL_RC=0
+VERSION_TIMEOUT_SEC="${VERSION_TIMEOUT_SEC:-720}"
+
+for SPARK_VERSION in ${SPARK_VERSIONS}; do
+ echo "==========================================="
+ echo "Matched matrix: client ${SPARK_VERSION} + cluster ${SPARK_VERSION}"
+ echo "==========================================="
+
+ HADOOP_VERSION="$(hadoop_version_for_spark "${SPARK_VERSION}")"
+ VERSION_SAFE=${SPARK_VERSION//./_}
+ VERSION_REPORT="${REPORT_DIR}/surefile_spark_matched_${VERSION_SAFE}.xml"
+
+ # 1) Materialise pack into the Hop install that docker base image will COPY
+ bash "${MATERIALIZE}" "${SPARK_VERSION}" "${HOP_HOME}"
+
+ # 2) Force rebuild of base + beam images so fat jar embeds this pack
+ # (base image must pick up newly added lib/spark-clients/)
+ export HOP_SPARK_CLIENT_VERSION="${SPARK_VERSION}"
+ export SPARK_VERSION
+ export HADOOP_VERSION
+ export SKIP_SUREFIRE_CLEAN=true
+ export KEEP_IMAGES="${KEEP_IMAGES:-true}"
+
+ rm -f "${REPORT_DIR}/surefile_spark.xml"
+ rm -f "${REPORT_DIR}/failed_tests" "${REPORT_DIR}/passed_tests"
+
+ # run-tests-docker.sh will: unzip client zip, re-materialise pack, copy pack
→ lib/spark-client,
+ # invalidate/rebuild hop-base + hop-beam images, then run smoke. Do not
pre-build images here
+ # (a pre-build is wiped by unzip inside run-tests-docker).
+ set +e
+ timeout --signal=TERM --kill-after=60 "${VERSION_TIMEOUT_SEC}" \
+ bash "${RUN_TESTS}" \
+ PROJECT_NAME=spark \
+ SPARK_VERSION="${SPARK_VERSION}" \
+ HADOOP_VERSION="${HADOOP_VERSION}" \
+ HOP_SPARK_CLIENT_VERSION="${SPARK_VERSION}" \
+ KEEP_IMAGES="${KEEP_IMAGES:-true}" \
+ "${FORWARD_ARGS[@]}"
+ RC=$?
+ set -e
+
+ PROJECT_NAME=spark SPARK_VERSION="${SPARK_VERSION}"
HADOOP_VERSION="${HADOOP_VERSION}" \
+ HOP_SPARK_CLIENT_VERSION="${SPARK_VERSION}" \
+ docker compose -f "${DOCKER_FILES_DIR}/integration-tests-spark.yaml" down
--remove-orphans >/dev/null 2>&1 || true
+
+ if [ -f "${REPORT_DIR}/surefile_spark.xml" ]; then
+ cp -f "${REPORT_DIR}/surefile_spark.xml" "${VERSION_REPORT}"
+ fi
+
+ evaluate_result "${VERSION_REPORT}"
+ if [ ${RC} -eq 124 ] || [ ${RC} -eq 137 ]; then
+ RESULT="FAIL"
+ NOTES="timed out (${NOTES})"
+ fi
+ if [ "${RESULT}" != "PASS" ]; then
+ OVERALL_RC=1
+ fi
+
+ echo "| ${SPARK_VERSION} | ${SPARK_VERSION} | ${RESULT} | ${NOTES} |"
>>"${MATRIX_REPORT}"
+ echo "Matched result ${SPARK_VERSION}: ${RESULT} (${NOTES})"
+done
+
+{
+ echo
+ echo "## How to re-run"
+ echo
+ echo '```bash'
+ echo "./integration-tests/scripts/run-spark-matched-matrix.sh
KEEP_IMAGES=true"
+ echo "SPARK_VERSIONS=\"3.5.8 3.4.4\"
./integration-tests/scripts/run-spark-matched-matrix.sh"
+ echo '```'
+ echo
+ echo "## Notes"
+ echo
+ echo "- Client pack materialised by
\`tools/spark-client-pack/materialize-pack.sh\`."
+ echo "- Fat jar built with \`hop-conf --spark-client-version=V\` inside
hop-beam-image."
+ echo "- Driver uses \`HOP_SPARK_CLIENT_VERSION=V\` so classpath matches the
fat jar."
+ echo "- Compare with fixed-client results in \`MATRIX-REPORT.md\`."
+} >>"${MATRIX_REPORT}"
+
+cp -f "${MATRIX_REPORT}" "${COMMITTED_REPORT}"
+echo
+cat "${MATRIX_REPORT}"
+exit ${OVERALL_RC}
diff --git a/integration-tests/scripts/run-spark-matrix.sh
b/integration-tests/scripts/run-spark-matrix.sh
new file mode 100755
index 0000000000..a404d21295
--- /dev/null
+++ b/integration-tests/scripts/run-spark-matrix.sh
@@ -0,0 +1,255 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Run the Hop Beam Spark smoke suite against multiple Spark 3.x standalone
versions.
+# Hop and Beam stay fixed; only the cluster image SPARK_VERSION changes.
+#
+# Usage:
+# ./integration-tests/scripts/run-spark-matrix.sh
+# SPARK_VERSIONS="3.5.8 3.4.4"
./integration-tests/scripts/run-spark-matrix.sh
+# ./integration-tests/scripts/run-spark-matrix.sh KEEP_IMAGES=true
+#
+# Prerequisites: assemblies/client must already be built (same as
run-tests-docker.sh).
+
+set -u
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+REPO_ROOT="$(cd "${CURRENT_DIR}/../.." >/dev/null 2>&1 && pwd)"
+REPORT_DIR="${CURRENT_DIR}/../surefire-reports"
+# Committed snapshot of the last measured matrix (refreshed by this script)
+COMMITTED_REPORT="${CURRENT_DIR}/../spark/MATRIX-REPORT.md"
+MATRIX_REPORT="${REPORT_DIR}/spark-matrix-report.md"
+RUN_TESTS="${CURRENT_DIR}/run-tests-docker.sh"
+DOCKER_COMPOSE_FILE="$(cd "${CURRENT_DIR}/../../docker/integration-tests"
>/dev/null 2>&1 && pwd)/integration-tests-spark.yaml"
+
+# Representative patch releases for each Spark 3.x line.
+# Override with SPARK_VERSIONS="3.5.8 3.2.4" etc.
+# Note: Hop client mode bundles spark-core at plugins/engines/beam
spark.version.
+# Clusters on a different Spark minor typically fail with InvalidClassException
+# (ApplicationDescription serialVersionUID mismatch). That is still a measured
result.
+if [ -z "${SPARK_VERSIONS:-}" ]; then
+ SPARK_VERSIONS="3.2.4 3.3.4 3.4.4 3.5.8"
+fi
+
+# Map Spark line → Hadoop binary suffix used on archive.apache.org
+# 3.2.x ships as bin-hadoop3.2; 3.3+ use bin-hadoop3.
+hadoop_version_for_spark() {
+ case "$1" in
+ 3.0.* | 3.1.* | 3.2.*) echo "3.2" ;;
+ *) echo "3" ;;
+ esac
+}
+
+FORWARD_ARGS=("$@")
+
+mkdir -p "${REPORT_DIR}"
+
+HOP_VERSION="2.19.0-SNAPSHOT"
+BEAM_VERSION="unknown"
+if [ -f "${REPO_ROOT}/pom.xml" ]; then
+ BEAM_VERSION=$(
+ grep -m1 'apache-beam.version' "${REPO_ROOT}/pom.xml" |
+ sed -E 's/.*<apache-beam.version>([^<]+)<\/apache-beam.version>.*/\1/'
|| true
+ )
+fi
+if [ -f "${REPO_ROOT}/plugins/engines/beam/pom.xml" ]; then
+ SPARK_CLIENT_VERSION=$(
+ grep -m1 'spark.version' "${REPO_ROOT}/plugins/engines/beam/pom.xml" |
+ sed -E 's/.*<spark.version>([^<]+)<\/spark.version>.*/\1/' || true
+ )
+else
+ SPARK_CLIENT_VERSION="unknown"
+fi
+
+GENERATED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+# Evaluate a surefire report. Sets RESULT and NOTES.
+# Note: run-tests.sh increments failure counters inside a pipeline subshell, so
+# suite-level failures=/errors= attributes are often wrong. Trust <failure>
tags.
+evaluate_result() {
+ local report_file="$1"
+ RESULT="FAIL"
+ NOTES="—"
+
+ if [ ! -f "${report_file}" ]; then
+ NOTES="no surefire report produced"
+ return
+ fi
+
+ if grep -q '<failure' "${report_file}"; then
+ if grep -q 'environment_setup' "${report_file}"; then
+ NOTES="docker/environment failed to start"
+ elif grep -q 'InvalidClassException\|serialVersionUID' "${report_file}";
then
+ NOTES="Spark client/cluster version skew (serialVersionUID)"
+ elif grep -q 'NoClassDefFoundError' "${report_file}"; then
+ NOTES="NoClassDefFoundError (classpath)"
+ elif grep -q 'UnsupportedClassVersionError' "${report_file}"; then
+ NOTES="UnsupportedClassVersionError (Java mismatch)"
+ elif grep -q 'Invalid Spark URL' "${report_file}"; then
+ NOTES="Invalid Spark URL (driver hostname)"
+ elif grep -q 'Servlet class org.glassfish.jersey' "${report_file}"; then
+ NOTES="Spark UI servlet clash"
+ elif grep -q 'stopped SparkContext' "${report_file}"; then
+ NOTES="stopped SparkContext (often version skew)"
+ elif grep -q 'ERROR:' "${report_file}"; then
+ NOTES="pipeline ERROR (see surefire log)"
+ else
+ NOTES="test failed (see surefire log)"
+ fi
+ return
+ fi
+
+ if ! grep -q '<testcase ' "${report_file}"; then
+ NOTES="empty surefire report"
+ return
+ fi
+
+ RESULT="PASS"
+ if grep -q 'Beam pipeline execution has finished' "${report_file}"; then
+ NOTES="smoke pipeline finished"
+ else
+ NOTES="hop-run exit 0"
+ fi
+}
+
+{
+ echo "# Hop Beam Spark support matrix"
+ echo
+ echo "Generated: ${GENERATED_AT}"
+ echo
+ echo "- Hop: \`${HOP_VERSION}\`"
+ echo "- Beam: \`${BEAM_VERSION}\`"
+ echo "- Client Spark libraries (\`spark.version\`):
\`${SPARK_CLIENT_VERSION}\` (scala 2.12)"
+ echo "- Driver JVM: Java 21 (Hop requirement)"
+ echo "- Cluster JVM: Java 21 (matches Hop bytecode in fat jar)"
+ echo "- Suite: \`integration-tests/spark\` smoke
(\`main-0001-test-spark-cluster\`)"
+ echo "- Cluster: Spark standalone master + worker (parameterized
\`SPARK_VERSION\`)"
+ echo
+ echo "| Spark version | Result | Surefire report | Notes |"
+ echo "| --- | --- | --- | --- |"
+} >"${MATRIX_REPORT}"
+
+OVERALL_RC=0
+
+for SPARK_VERSION in ${SPARK_VERSIONS}; do
+ echo "==========================================="
+ echo "Matrix run: Spark ${SPARK_VERSION}"
+ echo "==========================================="
+
+ VERSION_SAFE=${SPARK_VERSION//./_}
+ VERSION_REPORT="${REPORT_DIR}/surefile_spark_${VERSION_SAFE}.xml"
+ HADOOP_VERSION="$(hadoop_version_for_spark "${SPARK_VERSION}")"
+
+ echo "Cluster binary: spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
+
+ # Drop the live surefile so a failed environment cannot inherit a prior PASS
report
+ rm -f "${REPORT_DIR}/surefile_spark.xml"
+ rm -f "${REPORT_DIR}/failed_tests" "${REPORT_DIR}/passed_tests"
+
+ # Per-version wall clock (seconds). Version-skew / Java mismatches can thrash
+ # executors for a long time without a hard timeout.
+ VERSION_TIMEOUT_SEC="${VERSION_TIMEOUT_SEC:-600}"
+
+ set +e
+ export SKIP_SUREFIRE_CLEAN=true
+ export SPARK_VERSION
+ export HADOOP_VERSION
+ export KEEP_IMAGES="${KEEP_IMAGES:-true}"
+ # shellcheck disable=SC2086
+ timeout --signal=TERM --kill-after=60 "${VERSION_TIMEOUT_SEC}" \
+ bash "${RUN_TESTS}" \
+ PROJECT_NAME=spark \
+ SPARK_VERSION="${SPARK_VERSION}" \
+ HADOOP_VERSION="${HADOOP_VERSION}" \
+ KEEP_IMAGES="${KEEP_IMAGES:-true}" \
+ "${FORWARD_ARGS[@]}"
+ RC=$?
+ set -e
+
+ if [ ${RC} -eq 124 ] || [ ${RC} -eq 137 ]; then
+ echo "Matrix run for Spark ${SPARK_VERSION} timed out after
${VERSION_TIMEOUT_SEC}s"
+ fi
+
+ # Always tear down this version's stack before the next rebuild
+ PROJECT_NAME=spark SPARK_VERSION="${SPARK_VERSION}"
HADOOP_VERSION="${HADOOP_VERSION}" \
+ docker compose -f "${DOCKER_COMPOSE_FILE}" down --remove-orphans
>/dev/null 2>&1 || true
+
+ if [ -f "${REPORT_DIR}/surefile_spark.xml" ]; then
+ cp -f "${REPORT_DIR}/surefile_spark.xml" "${VERSION_REPORT}"
+ fi
+
+ evaluate_result "${VERSION_REPORT}"
+
+ # Enrich notes from the matrix log / known environmental failures when
surefire is empty
+ if [ "${RESULT}" = "FAIL" ] && [ "${NOTES}" = "docker/environment failed to
start" ]; then
+ if docker logs "$(docker ps -aq --filter name=integration-tests-spark
2>/dev/null | head -1)" 2>/dev/null |
+ grep -q 'DirectByteBuffer'; then
+ NOTES="Spark does not start on Java 21 (DirectByteBuffer)"
+ fi
+ fi
+ if [ ${RC} -eq 124 ] || [ ${RC} -eq 137 ]; then
+ RESULT="FAIL"
+ NOTES="timed out after ${VERSION_TIMEOUT_SEC}s (${NOTES})"
+ fi
+
+ if [ "${RESULT}" != "PASS" ]; then
+ OVERALL_RC=1
+ fi
+ # Script exit code from docker is informational only (container often exits
0)
+ if [ ${RC} -ne 0 ] && [ ${RC} -ne 124 ] && [ ${RC} -ne 137 ] && [
"${RESULT}" = "PASS" ]; then
+ NOTES="${NOTES}; docker rc=${RC}"
+ fi
+
+ echo "| ${SPARK_VERSION} | ${RESULT} |
\`surefile_spark_${VERSION_SAFE}.xml\` | ${NOTES} |" >>"${MATRIX_REPORT}"
+ echo "Matrix result for Spark ${SPARK_VERSION}: ${RESULT} (docker_rc=${RC},
notes=${NOTES})"
+done
+
+{
+ echo
+ echo "## How to re-run"
+ echo
+ echo '```bash'
+ echo "# Full matrix"
+ echo "./integration-tests/scripts/run-spark-matrix.sh KEEP_IMAGES=true"
+ echo
+ echo "# Single version"
+ echo "SPARK_VERSION=3.5.8 ./integration-tests/scripts/run-tests-docker.sh
PROJECT_NAME=spark KEEP_IMAGES=true"
+ echo '```'
+ echo
+ echo "## Notes"
+ echo
+ echo "- Beam upstream docs still list Spark 3.2.x as the supported line;
this matrix is **empirical** for Hop pipelines on Beam ${BEAM_VERSION}."
+ echo "- Hive catalog workflows are deferred (\`optional-0002/0003-*.hwf\`)
and not part of this matrix."
+ echo "- **Client mode** (Hop GUI / hop-run → \`spark://master\`) uses Spark
libraries bundled with Hop (\`${SPARK_CLIENT_VERSION}\`)."
+ echo " Clusters on a different Spark minor almost always fail with
\`InvalidClassException\` / serialVersionUID mismatch on
\`ApplicationDescription\`."
+ echo " Supported client-mode line is therefore the Hop \`spark.version\`
minor (currently 3.5.x)."
+ echo "- Spark 3.2.x binaries use the \`bin-hadoop3.2\` artifact name; 3.3+
use \`bin-hadoop3\`."
+} >>"${MATRIX_REPORT}"
+
+# Refresh the committed project copy for docs / PR visibility
+cp -f "${MATRIX_REPORT}" "${COMMITTED_REPORT}"
+
+echo
+echo "Matrix report written to:"
+echo " ${MATRIX_REPORT}"
+echo " ${COMMITTED_REPORT}"
+echo
+cat "${MATRIX_REPORT}"
+
+exit ${OVERALL_RC}
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
index 07787d9630..ffac427059 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -36,11 +36,31 @@ for ARGUMENT in "$@"; do
GCP_KEY_FILE) GCP_KEY_FILE=${VALUE} ;;
KEEP_IMAGES) KEEP_IMAGES=${VALUE} ;;
CLIENT_UNZIP) CLIENT_UNZIP=${VALUE} ;;
+ SPARK_VERSION) SPARK_VERSION=${VALUE} ;;
+ HADOOP_VERSION) HADOOP_VERSION=${VALUE} ;;
+ SPARK_BASE_URL) SPARK_BASE_URL=${VALUE} ;;
+ HOP_SPARK_CLIENT_VERSION) HOP_SPARK_CLIENT_VERSION=${VALUE} ;;
*) ;;
esac
done
+# Default Spark standalone version for integration-tests/spark (overridable
for matrix runs)
+if [ -z "${SPARK_VERSION}" ]; then
+ SPARK_VERSION="3.5.8"
+fi
+if [ -z "${HADOOP_VERSION}" ]; then
+ HADOOP_VERSION="3"
+fi
+# Prefer the fast CDN (current patch only); the Dockerfile falls back to
archive.apache.org
+# for historical matrix versions the CDN no longer carries.
+if [ -z "${SPARK_BASE_URL}" ]; then
+ SPARK_BASE_URL="https://dlcdn.apache.org/spark";
+fi
+# Optional: match driver + fat-jar Spark client pack to a cluster minor (see
tools/spark-client-pack)
+export SPARK_VERSION HADOOP_VERSION SPARK_BASE_URL
+export HOP_SPARK_CLIENT_VERSION="${HOP_SPARK_CLIENT_VERSION:-}"
+
if [ -z "${PROJECT_NAME}" ]; then
PROJECT_NAME="*"
fi
@@ -83,8 +103,10 @@ if [ -z "${CLIENT_UNZIP}" ]; then
CLIENT_UNZIP="true"
fi
-# Cleanup surefire reports
-rm -rf "${CURRENT_DIR}"/../surefire-reports
+# Cleanup surefire reports (matrix runs set SKIP_SUREFIRE_CLEAN=true to keep
per-version copies)
+if [ "${SKIP_SUREFIRE_CLEAN:-false}" != "true" ]; then
+ rm -rf "${CURRENT_DIR}"/../surefire-reports
+fi
mkdir -p "${CURRENT_DIR}"/../surefire-reports/
chmod 777 "${CURRENT_DIR}"/../surefire-reports/
@@ -103,7 +125,48 @@ else
echo "Skipping client unzip (CLIENT_UNZIP=${CLIENT_UNZIP}, using existing
${HOP_DIR})"
fi
-# Build base image only once
+# Versioned Spark client packs are not in the client zip. Re-materialise after
unzip so
+# HOP_SPARK_CLIENT_VERSION=… finds lib/spark-clients/<ver>/ (includes
spark-streaming, etc.).
+# Also copy the selected pack into lib/spark-client/ so the default driver
classpath always
+# has spark-core + spark-streaming even if hop-run only loads
lib/spark-client/*.
+HOP_HOME_FOR_PACK="${CURRENT_DIR}/../../assemblies/client/target/hop"
+if [ -n "${HOP_SPARK_CLIENT_VERSION}" ]; then
+
MATERIALIZE_SCRIPT="${CURRENT_DIR}/../../tools/spark-client-pack/materialize-pack.sh"
+ if [ -f "${MATERIALIZE_SCRIPT}" ]; then
+ echo "Materialising Spark client pack ${HOP_SPARK_CLIENT_VERSION} into
${HOP_HOME_FOR_PACK}"
+ bash "${MATERIALIZE_SCRIPT}" "${HOP_SPARK_CLIENT_VERSION}"
"${HOP_HOME_FOR_PACK}"
+
PACK_DIR="${HOP_HOME_FOR_PACK}/lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}"
+ if [ -d "${PACK_DIR}" ] && [ -f
"${PACK_DIR}/spark-core_2.12-${HOP_SPARK_CLIENT_VERSION}.jar" ]; then
+ echo "Activating pack ${HOP_SPARK_CLIENT_VERSION} as lib/spark-client
(driver classpath)"
+ rm -rf "${HOP_HOME_FOR_PACK}/lib/spark-client"
+ mkdir -p "${HOP_HOME_FOR_PACK}/lib/spark-client"
+ cp -a "${PACK_DIR}/." "${HOP_HOME_FOR_PACK}/lib/spark-client/"
+ # Prove critical jars are present for the driver
+ ls -1 "${HOP_HOME_FOR_PACK}/lib/spark-client"/spark-core*.jar \
+ "${HOP_HOME_FOR_PACK}/lib/spark-client"/spark-streaming*.jar
+ else
+ echo "ERROR: Spark client pack incomplete at ${PACK_DIR}" >&2
+ ls -la "${PACK_DIR}" 2>/dev/null || true
+ exit 1
+ fi
+ else
+ echo "WARNING: ${MATERIALIZE_SCRIPT} not found; pack
${HOP_SPARK_CLIENT_VERSION} may be missing"
+ fi
+fi
+
+# Bust docker cache for the hop COPY layer when packs change
+if [ -d "${HOP_HOME_FOR_PACK}" ]; then
+ date -u +%Y-%m-%dT%H:%M:%SZ > "${HOP_HOME_FOR_PACK}/.spark-client-pack-stamp"
+fi
+
+# Drop stale base/beam images when using a versioned Spark pack so COPY hop
picks up jars
+if [ -n "${HOP_SPARK_CLIENT_VERSION}" ]; then
+ echo "Invalidating hop-base-image / hop-beam-image for Spark client pack
${HOP_SPARK_CLIENT_VERSION}"
+ docker rmi hop-beam-image 2>/dev/null || true
+ docker rmi hop-base-image 2>/dev/null || true
+fi
+
+# Build base image only once (must run AFTER pack materialise so jars are in
the image)
docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml build
--build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID}
--build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg
JENKINS_GID=${JENKINS_GID} --build-arg GCP_KEY_FILE=${GCP_KEY_FILE}
# The Hop fat jar (needed only by the Beam runners: spark/flink/gcp) is
expensive to build, so it
@@ -129,7 +192,12 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
# Built once per run, and only when such a project is actually enabled.
if [ "${BEAM_IMAGE_BUILT}" != "true" ] && grep -rqs "hop-fatjar.jar"
"$d" 2>/dev/null; then
echo "Project ${PROJECT_NAME} needs the Hop fat jar; building
hop-beam-image (once)."
- docker compose -f ${DOCKER_FILES_DIR}/integration-tests-beam-base.yaml
build
+ if [ -n "${HOP_SPARK_CLIENT_VERSION}" ]; then
+ echo "Spark client pack for fat jar: ${HOP_SPARK_CLIENT_VERSION}"
+ fi
+ HOP_SPARK_CLIENT_VERSION="${HOP_SPARK_CLIENT_VERSION}" \
+ docker compose -f
${DOCKER_FILES_DIR}/integration-tests-beam-base.yaml build \
+ --build-arg HOP_SPARK_CLIENT_VERSION="${HOP_SPARK_CLIENT_VERSION}"
EXECUTED_COMPOSE_FILES=("${EXECUTED_COMPOSE_FILES[@]}"
"${DOCKER_FILES_DIR}/integration-tests-beam-base.yaml")
BEAM_IMAGE_BUILT="true"
fi
@@ -139,7 +207,14 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml" ];
then
echo "Project compose exists."
EXECUTED_COMPOSE_FILES=("${EXECUTED_COMPOSE_FILES[@]}"
"${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml")
- PROJECT_NAME=${PROJECT_NAME} docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit
+ # Rebuild project images so SPARK_VERSION (and similar) build args
take effect
+ if [ "${PROJECT_NAME}" = "spark" ]; then
+ echo "Spark IT cluster version: ${SPARK_VERSION} (hadoop
${HADOOP_VERSION})"
+ PROJECT_NAME=${PROJECT_NAME} SPARK_VERSION=${SPARK_VERSION}
HADOOP_VERSION=${HADOOP_VERSION} SPARK_BASE_URL=${SPARK_BASE_URL} \
+ docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --build
--abort-on-container-exit
+ else
+ PROJECT_NAME=${PROJECT_NAME} docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit
+ fi
else
echo "Project compose does not exists."
PROJECT_NAME=${PROJECT_NAME} docker compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
diff --git a/integration-tests/spark/0001-test-spark-cluster.hpl
b/integration-tests/spark/0001-test-spark-cluster.hpl
index 1e86f3d71b..734b1222b5 100644
--- a/integration-tests/spark/0001-test-spark-cluster.hpl
+++ b/integration-tests/spark/0001-test-spark-cluster.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0001-test-flink-cluster</name>
+ <name>0001-test-spark-cluster</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
diff --git a/integration-tests/spark/MATRIX-REPORT-MATCHED.md
b/integration-tests/spark/MATRIX-REPORT-MATCHED.md
new file mode 100644
index 0000000000..249ef56a9d
--- /dev/null
+++ b/integration-tests/spark/MATRIX-REPORT-MATCHED.md
@@ -0,0 +1,45 @@
+# Hop Beam Spark *matched* client/cluster matrix
+
+Generated: 2026-07-10 (issue #7476)
+
+> **Pin moved 3.5.7 → 3.5.8** (current patch on `dlcdn.apache.org`; 3.5.7 is
now archive-only
+> and throttled). Same 3.5.x line, same conclusion. The measured row still
shows the last
+> **3.5.7** run — re-run `run-spark-matched-matrix.sh` to refresh it for 3.5.8.
+
+- Hop: `2.19.0-SNAPSHOT` · Beam: `2.74.0`
+- Each cell: **Spark client pack V** = **cluster V** (Java 21 driver + cluster)
+- Suite: `main-0001-test-spark-cluster`
+
+## Results
+
+| Client pack | Cluster | Result | Notes |
+| --- | --- | --- | --- |
+| 3.5.7 | 3.5.7 | **PASS** | Pack activated as `lib/spark-client`; smoke
finished |
+| 3.4.4 | 3.4.4 | **FAIL** | `NoSuchMethodException:
java.nio.DirectByteBuffer.<init>(long,int)` —
[SPARK-42369](https://issues.apache.org/jira/browse/SPARK-42369) fixed only
from **Spark 3.5.0** |
+
+## Interpretation
+
+Matching the client pack to the cluster **eliminates RPC serialVersionUID
skew**, but **cannot** make Spark 3.4 run on **Java 21**. Hop requires Java 21
for driver and workers (fat-jar bytecode), so **Spark ≥ 3.5.0 is the platform
floor**.
+
+Versioned packs remain valuable for:
+
+- Aligning Hop’s client jars with a **3.5.x** cluster patch level
+- Future matrix cells within 3.5.x (e.g. 3.5.7 vs 3.5.8)
+
+They are **not** a path back to Spark 3.4 / 3.3 / 3.2 while Hop stays on Java
21.
+
+## How to run
+
+```bash
+./integration-tests/scripts/run-spark-matched-matrix.sh KEEP_IMAGES=true
+```
+
+## Tooling
+
+| Piece | Path |
+| --- | --- |
+| Default pack (install) | `lib/spark-client/` |
+| Versioned packs | `lib/spark-clients/<version>/` |
+| Materialise pack | `tools/spark-client-pack/materialize-pack.sh` |
+| Fat jar flag | `hop-conf.sh --spark-client-version=V` |
+| Driver env | `HOP_SPARK_CLIENT_VERSION=V` |
diff --git a/integration-tests/spark/MATRIX-REPORT.md
b/integration-tests/spark/MATRIX-REPORT.md
new file mode 100644
index 0000000000..0ed515a60e
--- /dev/null
+++ b/integration-tests/spark/MATRIX-REPORT.md
@@ -0,0 +1,61 @@
+# Hop Beam Spark support matrix
+
+Generated: 2026-07-10 (issue #7476)
+
+> **Pin moved 3.5.7 → 3.5.8.** `dlcdn.apache.org` only serves the current
patch of each
+> active line, and once 3.5.8 shipped, 3.5.7 became archive-only (downloads at
~100 KB/s,
+> which is what made the docker build appear to hang). 3.5.8 is the same 3.5.x
line as 3.5.7
+> — both are
post-[SPARK-42369](https://issues.apache.org/jira/browse/SPARK-42369) — so the
+> conclusions below stand. The measured cells still show the last **3.5.7**
run; re-run
+> `run-spark-matrix.sh` to refresh them for 3.5.8.
+
+- Hop: `2.19.0-SNAPSHOT`
+- Beam: `2.74.0`
+- Default client Spark libraries (`spark.version`): `3.5.8` (scala 2.12)
+- Driver JVM: **Java 21** (Hop requirement)
+- Cluster JVM: **Java 21** (workers must load Hop fat-jar bytecode)
+- Suite: `integration-tests/spark` smoke (`main-0001-test-spark-cluster`)
+- Mode: Hop client submit (`BeamSparkPipelineEngine` → `spark://spark:7077` +
fat jar)
+
+## Results (fixed client 3.5.7 vs varying cluster)
+
+| Spark version | Cluster binary | Result | Notes |
+| --- | --- | --- | --- |
+| 3.2.4 | `bin-hadoop3.2` | **FAIL** | Master does not start on Java 21
(`DirectByteBuffer.<init>(long,int)`) |
+| 3.3.4 | `bin-hadoop3` | **FAIL** | Client/cluster skew
(`InvalidClassException` / serialVersionUID) |
+| 3.4.4 | `bin-hadoop3` | **FAIL** | Executor does not start on Java 21
(`DirectByteBuffer.<init>(long,int)`) |
+| 3.5.7 | `bin-hadoop3` | **PASS** | Smoke pipeline finished |
+
+## Results (matched client pack V + cluster V)
+
+| Client pack | Cluster | Result | Notes |
+| --- | --- | --- | --- |
+| 3.5.7 | 3.5.7 | **PASS** | After driver classpath / pack wiring fix |
+| 3.4.4 | 3.4.4 | **FAIL** | Same Java 21 `DirectByteBuffer` failure —
matching client pack does **not** help |
+| 3.3.4 | 3.3.4 | *(optional)* | Not required once 3.4 matched FAIL is
explained by Java 21 |
+
+## Conclusion
+
+**Hop on Java 21 is effectively limited to Spark 3.5.x** for Beam client-mode
submit.
+
+1. **Java 21 floor (dominant):** Spark’s unsafe `DirectByteBuffer` reflection
broke on JDK 21 until
[SPARK-42369](https://issues.apache.org/jira/browse/SPARK-42369) (*Fix
constructor for java.nio.DirectByteBuffer for Java 21+*), fixed in **Spark
3.5.0**. Hop requires Java 21 for the driver and for fat-jar workers, so the
cluster must also run Java 21. Spark **3.4.x and earlier cannot be used** on
that stack, even with a matching client pack / fat jar.
+2. **Client/cluster match:** When both sides are on 3.5.x, client libraries
and cluster minor must still align (default pack `3.5.8` with a matching
`3.5.8` cluster; `3.5.7`/`3.5.7` was validated previously). Versioned packs
(`tools/spark-client-pack`, `HOP_SPARK_CLIENT_VERSION`) remain useful for
**3.5.x patch** alignment, not for going back to 3.4.
+3. **Supported line:** **Spark 3.5.x** (pinned **3.5.8**; **3.5.7** measured)
+ Beam **2.74.0** + Hop **2.19** + **Java 21**.
+
+## How to re-run
+
+```bash
+# Default pin (recommended CI smoke)
+SPARK_VERSION=3.5.8 ./integration-tests/scripts/run-tests-docker.sh
PROJECT_NAME=spark KEEP_IMAGES=true
+
+# Fixed client vs varying cluster
+./integration-tests/scripts/run-spark-matrix.sh KEEP_IMAGES=true
+
+# Matched packs (documents Java 21 ceiling on pre-3.5)
+./integration-tests/scripts/run-spark-matched-matrix.sh KEEP_IMAGES=true
+```
+
+## Notes
+
+- Beam upstream docs may still mention Spark 3.2.x; that is not Hop’s measured
support on Java 21.
+- Hive catalog workflows remain deferred (`optional-0002/0003-*.hwf`).
diff --git a/integration-tests/spark/disabled.txt
b/integration-tests/spark/disabled.txt
deleted file mode 100644
index 7bf85a795a..0000000000
--- a/integration-tests/spark/disabled.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-<!--
-
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
--->
\ No newline at end of file
diff --git
a/integration-tests/spark/metadata/pipeline-run-configuration/spark.json
b/integration-tests/spark/metadata/pipeline-run-configuration/spark-local.json
similarity index 79%
copy from integration-tests/spark/metadata/pipeline-run-configuration/spark.json
copy to
integration-tests/spark/metadata/pipeline-run-configuration/spark-local.json
index d15ae3dcfb..af7fefefc6 100644
--- a/integration-tests/spark/metadata/pipeline-run-configuration/spark.json
+++
b/integration-tests/spark/metadata/pipeline-run-configuration/spark-local.json
@@ -2,7 +2,7 @@
"engineRunConfiguration": {
"BeamSparkPipelineEngine": {
"sparkMinReadTimeMillis": "",
- "sparkMaster": "spark://spark:7077",
+ "sparkMaster": "local[2]",
"sparkEnableSparkMetricSinks": false,
"streamingHopTransformsBufferSize": "",
"sparkCheckpointDir": "",
@@ -10,7 +10,7 @@
"userAgent": "Hop",
"sparkBatchIntervalMillis": "",
"pluginsToStage": "",
- "tempLocation": "/var/folders/v0/hjm6kpgd0076ncdkpf8rg3dw0000gn/T/",
+ "tempLocation": "/tmp",
"sparkCheckpointDurationMillis": "",
"fatJar": "/tmp/hop-fatjar.jar",
"sparkBundleSize": "",
@@ -21,6 +21,6 @@
}
},
"configurationVariables": [],
- "name": "spark",
- "description": ""
-}
\ No newline at end of file
+ "name": "spark-local",
+ "description": "Embedded Spark local[2] for driver-only smoke (no cluster)"
+}
diff --git
a/integration-tests/spark/metadata/pipeline-run-configuration/spark.json
b/integration-tests/spark/metadata/pipeline-run-configuration/spark.json
index d15ae3dcfb..b4ffb0475c 100644
--- a/integration-tests/spark/metadata/pipeline-run-configuration/spark.json
+++ b/integration-tests/spark/metadata/pipeline-run-configuration/spark.json
@@ -10,7 +10,7 @@
"userAgent": "Hop",
"sparkBatchIntervalMillis": "",
"pluginsToStage": "",
- "tempLocation": "/var/folders/v0/hjm6kpgd0076ncdkpf8rg3dw0000gn/T/",
+ "tempLocation": "/tmp",
"sparkCheckpointDurationMillis": "",
"fatJar": "/tmp/hop-fatjar.jar",
"sparkBundleSize": "",
@@ -22,5 +22,5 @@
},
"configurationVariables": [],
"name": "spark",
- "description": ""
-}
\ No newline at end of file
+ "description": "Beam Spark runner against the integration-test standalone
cluster"
+}
diff --git a/integration-tests/spark/main-0002-test-hive-connection.hwf
b/integration-tests/spark/optional-0002-test-hive-connection.hwf
similarity index 100%
rename from integration-tests/spark/main-0002-test-hive-connection.hwf
rename to integration-tests/spark/optional-0002-test-hive-connection.hwf
diff --git a/integration-tests/spark/main-0003-test-hive-write-read.hwf
b/integration-tests/spark/optional-0003-test-hive-write-read.hwf
similarity index 100%
rename from integration-tests/spark/main-0003-test-hive-write-read.hwf
rename to integration-tests/spark/optional-0003-test-hive-write-read.hwf
diff --git a/plugins/engines/beam/pom.xml b/plugins/engines/beam/pom.xml
index 912adbdb4b..6e7b8483fc 100644
--- a/plugins/engines/beam/pom.xml
+++ b/plugins/engines/beam/pom.xml
@@ -29,7 +29,7 @@
<name>Hop Plugins Engines Beam</name>
<properties>
- <spark.version>3.5.7</spark.version>
+ <spark.version>3.5.8</spark.version>
</properties>
<dependencyManagement>
@@ -1474,6 +1474,105 @@
</exclusion>
</exclusions>
</dependency>
+ <!--
+ Spark core/streaming are marked provided by beam-runners-spark-3.
Hop client-mode
+ submit (BeamSparkPipelineEngine → spark://master) needs them on the
driver classpath
+ and in the fat jar. Keep log4j/jackson-scala exclusions to avoid
clashing with Hop.
+ -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.12</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.11</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.13</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.12</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.11</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.13</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-constant</artifactId>
@@ -1549,75 +1648,5 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.11</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.13</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.lz4</groupId>
- <artifactId>lz4-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_2.12</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
diff --git a/plugins/engines/beam/src/assembly/assembly.xml
b/plugins/engines/beam/src/assembly/assembly.xml
index 5fe15267ce..c012216d5c 100644
--- a/plugins/engines/beam/src/assembly/assembly.xml
+++ b/plugins/engines/beam/src/assembly/assembly.xml
@@ -169,9 +169,35 @@
<exclude>io.dropwizard.metrics:*:jar</exclude>
<exclude>com.eclipsesource.minimal-json:minimal-json:jar</exclude>
<exclude>org.slf4j:slf4j-api:jar</exclude>
+ <!-- Spark client runtime lives in lib/spark-client (versioned
packs); keep Beam's own Spark fragments in lib/beam -->
+ <exclude>org.apache.spark:spark-core_2.12:jar</exclude>
+ <exclude>org.apache.spark:spark-streaming_2.12:jar</exclude>
+ <exclude>org.apache.spark:spark-launcher_2.12:jar</exclude>
+ <exclude>org.apache.spark:spark-kvstore_2.12:jar</exclude>
+
<exclude>org.apache.spark:spark-network-common_2.12:jar</exclude>
+
<exclude>org.apache.spark:spark-network-shuffle_2.12:jar</exclude>
+ <exclude>org.apache.spark:spark-unsafe_2.12:jar</exclude>
+ <exclude>org.apache.spark:spark-tags_2.12:jar</exclude>
</excludes>
<outputDirectory>lib/beam</outputDirectory>
</dependencySet>
+ <!-- Default Spark client pack (matches plugins/engines/beam
spark.version). Optional
+ additional packs can be materialised under
lib/spark-clients/<version>/ via
+ tools/spark-client-pack. Selected at runtime with
HOP_SPARK_CLIENT_VERSION. -->
+ <dependencySet>
+ <scope>runtime</scope>
+ <includes>
+ <include>org.apache.spark:spark-core_2.12:jar</include>
+ <include>org.apache.spark:spark-streaming_2.12:jar</include>
+ <include>org.apache.spark:spark-launcher_2.12:jar</include>
+ <include>org.apache.spark:spark-kvstore_2.12:jar</include>
+
<include>org.apache.spark:spark-network-common_2.12:jar</include>
+
<include>org.apache.spark:spark-network-shuffle_2.12:jar</include>
+ <include>org.apache.spark:spark-unsafe_2.12:jar</include>
+ <include>org.apache.spark:spark-tags_2.12:jar</include>
+ </includes>
+ <outputDirectory>lib/spark-client</outputDirectory>
+ </dependencySet>
<dependencySet>
<scope>runtime</scope>
<includes>
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/config/GenerateFatJarConfigPlugin.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/config/GenerateFatJarConfigPlugin.java
index c7c9a28dd3..52586c5c3a 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/config/GenerateFatJarConfigPlugin.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/config/GenerateFatJarConfigPlugin.java
@@ -17,6 +17,7 @@
package org.apache.hop.beam.config;
+import java.io.File;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.beam.gui.HopBeamGuiPlugin;
@@ -40,6 +41,14 @@ public class GenerateFatJarConfigPlugin implements
IConfigOptions {
"Specify the filename of the fat jar to generate from your current
software installation")
private String fatJarFilename;
+ @CommandLine.Option(
+ names = {"-scv", "--spark-client-version"},
+ description =
+ "Spark client pack version to embed (directory
lib/spark-clients/<version>). "
+ + "When omitted, uses HOP_SPARK_CLIENT_VERSION or the default
pack at lib/spark-client. "
+ + "Client and Spark cluster minor versions must match for
client-mode submit.")
+ private String sparkClientVersion;
+
@Override
public boolean handleOption(
ILogChannel log, IHasHopMetadataProvider hasHopMetadataProvider,
IVariables variables)
@@ -60,7 +69,30 @@ public class GenerateFatJarConfigPlugin implements
IConfigOptions {
String realFatJarFilename = variables.resolve(fatJarFilename);
log.logBasic("Generating a Hop fat jar file in : " + realFatJarFilename);
- List<String> installedJarFilenames =
HopBeamGuiPlugin.findInstalledJarFilenames();
+ String resolvedSparkClientVersion =
+ HopBeamGuiPlugin.resolveSparkClientVersion(
+ StringUtils.isNotEmpty(sparkClientVersion)
+ ? variables.resolve(sparkClientVersion)
+ : null);
+ File packDir =
HopBeamGuiPlugin.resolveSparkClientPackDir(resolvedSparkClientVersion);
+ if (StringUtils.isNotBlank(resolvedSparkClientVersion)) {
+ log.logBasic(
+ "Using Spark client pack version "
+ + resolvedSparkClientVersion
+ + " from "
+ + packDir.getPath());
+ if (!packDir.isDirectory()) {
+ throw new HopException(
+ "Spark client pack not found: "
+ + packDir.getPath()
+ + " — materialise it with
tools/spark-client-pack/materialize-pack.sh");
+ }
+ } else {
+ log.logBasic("Using default Spark client pack from " +
packDir.getPath());
+ }
+
+ List<String> installedJarFilenames =
+ HopBeamGuiPlugin.findInstalledJarFilenames(resolvedSparkClientVersion);
log.logBasic(
"Found " + installedJarFilenames.size() + " jar files to combine into
one fat jar file.");
@@ -88,4 +120,12 @@ public class GenerateFatJarConfigPlugin implements
IConfigOptions {
public void setFatJarFilename(String fatJarFilename) {
this.fatJarFilename = fatJarFilename;
}
+
+ public String getSparkClientVersion() {
+ return sparkClientVersion;
+ }
+
+ public void setSparkClientVersion(String sparkClientVersion) {
+ this.sparkClientVersion = sparkClientVersion;
+ }
}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
index 674e7d097d..02e01b7c87 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
@@ -207,28 +207,146 @@ public class HopBeamGuiPlugin {
}
}
+ /**
+ * Collect jar files from the current Hop installation for fat-jar
generation.
+ *
+ * <p>Uses the default Spark client pack ({@code lib/spark-client}), or the
versioned pack
+ * selected by system property / env {@code HOP_SPARK_CLIENT_VERSION} under
{@code
+ * lib/spark-clients/<version>}. Versioned packs under {@code
lib/spark-clients/} are never all
+ * included at once.
+ */
public static final List<String> findInstalledJarFilenames() {
+ return findInstalledJarFilenames(resolveSparkClientVersion(null));
+ }
+
+ /**
+ * @param sparkClientVersion Spark client pack version (e.g. {@code 3.5.8}),
or null/blank for the
+ * default pack at {@code lib/spark-client}
+ */
+ public static final List<String> findInstalledJarFilenames(String
sparkClientVersion) {
Set<File> jarFiles = new HashSet<>();
- jarFiles.addAll(FileUtils.listFiles(new File("lib"), new String[] {"jar"},
true));
+ boolean versionedPack = StringUtils.isNotBlank(sparkClientVersion);
+
+ // lib/ tree except spark client pack directories (handled separately
below)
+ collectJarsExcludingSparkClientPacks(new File("lib"), jarFiles,
versionedPack);
File libSwtFiles = new File("lib/swt/linux/x86_64");
- if (libSwtFiles.exists())
+ if (libSwtFiles.exists()) {
jarFiles.addAll(FileUtils.listFiles(libSwtFiles, new String[] {"jar"},
true));
+ }
- jarFiles.addAll(FileUtils.listFiles(new File("plugins"), new String[]
{"jar"}, true));
+ File pluginsDir = new File("plugins");
+ if (pluginsDir.isDirectory()) {
+ jarFiles.addAll(FileUtils.listFiles(pluginsDir, new String[] {"jar"},
true));
+ }
// If we are in the hop-web Docker container, we need to import the Hop
main JARs too for
- // Dataflow
- // (and for other runners probably too)
+ // Dataflow (and for other runners probably too)
File hopWebJars = new File("webapps/ROOT/WEB-INF/lib");
- if (hopWebJars.exists())
+ if (hopWebJars.exists()) {
jarFiles.addAll(FileUtils.listFiles(hopWebJars, new String[] {"jar"},
true));
+ }
+
+ // Selected Spark client pack only (never all versioned packs at once)
+ File sparkClientDir = resolveSparkClientPackDir(sparkClientVersion);
+ if (sparkClientDir.isDirectory()) {
+ jarFiles.addAll(FileUtils.listFiles(sparkClientDir, new String[]
{"jar"}, false));
+ }
+
+ // Final guard: when using a versioned pack, drop any spark-* jar outside
that pack
+ // (avoids Beam fragments under lib/beam mixing serialVersionUID /
version-info).
+ if (versionedPack) {
+ String packPath;
+ try {
+ packPath = sparkClientDir.getCanonicalPath();
+ } catch (Exception e) {
+ packPath = sparkClientDir.getAbsolutePath();
+ }
+ final String packPrefix = packPath;
+ jarFiles.removeIf(
+ f -> {
+ String n = f.getName();
+ // Spark distribution jars are named spark-*.jar; Beam's runner is
beam-runners-spark-*
+ if (!n.startsWith("spark-") || !n.endsWith(".jar")) {
+ return false;
+ }
+ try {
+ return !f.getCanonicalPath().startsWith(packPrefix);
+ } catch (Exception e) {
+ return !f.getAbsolutePath().startsWith(packPrefix);
+ }
+ });
+ }
List<String> jarFilenames = new ArrayList<>();
jarFiles.forEach(file -> jarFilenames.add(file.toString()));
return jarFilenames;
}
+ /**
+ * Resolve which Spark client pack version to use. Explicit argument wins,
then system property
+ * {@code HOP_SPARK_CLIENT_VERSION}, then env of the same name.
+ */
+ public static String resolveSparkClientVersion(String explicitVersion) {
+ if (StringUtils.isNotBlank(explicitVersion)) {
+ return explicitVersion.trim();
+ }
+ String prop = System.getProperty("HOP_SPARK_CLIENT_VERSION");
+ if (StringUtils.isNotBlank(prop)) {
+ return prop.trim();
+ }
+ String env = System.getenv("HOP_SPARK_CLIENT_VERSION");
+ if (StringUtils.isNotBlank(env)) {
+ return env.trim();
+ }
+ return null;
+ }
+
+ /** Directory for the default pack or {@code lib/spark-clients/<version>}. */
+ public static File resolveSparkClientPackDir(String sparkClientVersion) {
+ if (StringUtils.isBlank(sparkClientVersion)) {
+ return new File("lib/spark-client");
+ }
+ return new File("lib/spark-clients/" + sparkClientVersion.trim());
+ }
+
+ /**
+ * @param versionedPack when true, also skip {@code spark-*.jar} under
{@code lib/beam} so Beam's
+ * fixed Spark fragments cannot mix with an alternate client pack
+ */
+ private static void collectJarsExcludingSparkClientPacks(
+ File dir, Set<File> jarFiles, boolean versionedPack) {
+ if (dir == null || !dir.isDirectory()) {
+ return;
+ }
+ String name = dir.getName();
+ File parent = dir.getParentFile();
+ // Skip lib/spark-client and lib/spark-clients entirely
+ if (parent != null
+ && "lib".equals(parent.getName())
+ && ("spark-client".equals(name) || "spark-clients".equals(name))) {
+ return;
+ }
+ File[] children = dir.listFiles();
+ if (children == null) {
+ return;
+ }
+ for (File child : children) {
+ if (child.isDirectory()) {
+ collectJarsExcludingSparkClientPacks(child, jarFiles, versionedPack);
+ } else if (child.isFile() && child.getName().endsWith(".jar")) {
+ // Versioned packs own all spark-* jars; drop Beam's spark fragments
from lib/beam
+ if (versionedPack
+ && child.getName().startsWith("spark-")
+ && parent != null
+ && "beam".equals(parent.getName())) {
+ continue;
+ }
+ jarFiles.add(child);
+ }
+ }
+ }
+
/**
* Add a toolbar item just above the pipeline graph to quickly visit the
execution of a Dataflow
* pipeline in the GCP console.
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java
index c8917c31d0..a931909c09 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java
@@ -26,17 +26,28 @@ import org.apache.hop.core.variables.DescribedVariable;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.pipeline.engine.IPipelineEngine;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-@Disabled("Temp disable until we migrate to spark 4")
+/**
+ * Embedded Spark local-mode smoke test for the Beam Spark pipeline engine.
+ *
+ * <p>Requires spark-core on the test classpath (see hop-engines-beam pom).
Hop runs on Java 21;
+ * Spark 3.5.x is used for local mode. Cluster multi-version coverage lives in
{@code
+ * integration-tests/spark} / {@code run-spark-matrix.sh}.
+ */
class BeamSparkPipelineEngineTest extends BeamBasePipelineEngineTest {
@Test
void testSparkPipelineEngine() throws Exception {
+ // Spark UI pulls in a shaded Jetty + jersey stack that conflicts with
Hop's servlet APIs
+ // on the unit-test classpath. The pipeline does not need the UI.
+ System.setProperty("spark.ui.enabled", "false");
+ System.setProperty("spark.driver.host", "127.0.0.1");
+ System.setProperty("spark.driver.bindAddress", "127.0.0.1");
BeamSparkPipelineRunConfiguration configuration = new
BeamSparkPipelineRunConfiguration();
- configuration.setSparkMaster("local");
+ configuration.setSparkMaster("local[2]");
+ configuration.setTempLocation(System.getProperty("java.io.tmpdir"));
configuration.setEnginePluginId("BeamSparkPipelineEngine");
PipelineRunConfiguration pipelineRunConfiguration =
new PipelineRunConfiguration(
diff --git a/pom.xml b/pom.xml
index fc6e04269d..eb865cf2f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -509,6 +509,7 @@
<!-- External Integration Test Exclusion -->
<exclude>**/integration-tests/**/*.txt</exclude>
+ <exclude>**/integration-tests/**/*.md</exclude>
<exclude>**/integration-tests/**/*.csv</exclude>
<exclude>**/integration-tests/**/*.avro</exclude>
<exclude>**/integration-tests/**/*.xls</exclude>
diff --git a/tools/spark-client-pack/materialize-pack.sh
b/tools/spark-client-pack/materialize-pack.sh
new file mode 100755
index 0000000000..608206421d
--- /dev/null
+++ b/tools/spark-client-pack/materialize-pack.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Materialise a Spark client pack for Hop under lib/spark-clients/<version>/.
+#
+# Usage:
+# ./tools/spark-client-pack/materialize-pack.sh 3.4.4
+# ./tools/spark-client-pack/materialize-pack.sh 3.4.4 /path/to/hop
+# SPARK_CLIENT_VERSIONS="3.3.4 3.4.4 3.5.8"
./tools/spark-client-pack/materialize-pack.sh
+#
+# Then generate a matching fat jar:
+# HOP_SPARK_CLIENT_VERSION=3.4.4 ./hop-conf.sh
--generate-fat-jar=/tmp/hop-spark-3.4.4.jar \
+# --spark-client-version=3.4.4
+#
+# And run Hop with the same pack on the driver classpath:
+# HOP_SPARK_CLIENT_VERSION=3.4.4 ./hop-run.sh ...
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
+PACK_POM="${SCRIPT_DIR}/pom.xml"
+
+# Default Hop install to materialise into (client assembly extract or live
install)
+HOP_HOME_RAW="${2:-${HOP_HOME:-${REPO_ROOT}/assemblies/client/target/hop}}"
+# Always absolute so mvn -DoutputDirectory and existence checks are
cwd-independent
+HOP_HOME="$(cd "${HOP_HOME_RAW}" 2>/dev/null && pwd || true)"
+if [ -z "${HOP_HOME}" ]; then
+ mkdir -p "${HOP_HOME_RAW}"
+ HOP_HOME="$(cd "${HOP_HOME_RAW}" && pwd)"
+fi
+
+if [ -n "${1:-}" ] && [[ "${1}" != *"="* ]]; then
+ VERSIONS="$1"
+elif [ -n "${SPARK_CLIENT_VERSIONS:-}" ]; then
+ VERSIONS="${SPARK_CLIENT_VERSIONS}"
+else
+ VERSIONS="3.5.8"
+fi
+
+MVN="${REPO_ROOT}/mvnw"
+if [ ! -x "${MVN}" ]; then
+ MVN="mvn"
+fi
+
+# Only keep the same artifact set the beam assembly puts in lib/spark-client
+keep_pack_jars() {
+ local dir="$1"
+ local version="$2"
+ # Remove jars that are not part of the curated pack (transitive noise)
+ find "${dir}" -maxdepth 1 -type f -name '*.jar' | while read -r jar; do
+ base="$(basename "${jar}")"
+ case "${base}" in
+ spark-core_2.12-"${version}".jar | \
+ spark-streaming_2.12-"${version}".jar | \
+ spark-launcher_2.12-"${version}".jar | \
+ spark-kvstore_2.12-"${version}".jar | \
+ spark-network-common_2.12-"${version}".jar | \
+ spark-network-shuffle_2.12-"${version}".jar | \
+ spark-unsafe_2.12-"${version}".jar | \
+ spark-tags_2.12-"${version}".jar | \
+ spark-common-utils_2.12-"${version}".jar | \
+ spark-sql-api_2.12-"${version}".jar)
+ ;;
+ *)
+ rm -f "${jar}"
+ ;;
+ esac
+ done
+}
+
+for VERSION in ${VERSIONS}; do
+ OUT_DIR="${HOP_HOME}/lib/spark-clients/${VERSION}"
+ echo "==========================================="
+ echo "Materialising Spark client pack ${VERSION}"
+ echo " pom: ${PACK_POM}"
+ echo " out: ${OUT_DIR}"
+ echo "==========================================="
+
+ mkdir -p "${OUT_DIR}"
+ rm -f "${OUT_DIR}"/*.jar
+
+ "${MVN}" -f "${PACK_POM}" \
+ -Dspark.client.version="${VERSION}" \
+ dependency:copy-dependencies \
+ -DoutputDirectory="${OUT_DIR}" \
+ -DincludeScope=runtime \
+ -q
+
+ keep_pack_jars "${OUT_DIR}" "${VERSION}"
+
+ echo "Pack contents:"
+ ls -1 "${OUT_DIR}" | sed 's/^/ /'
+
+ if [ ! -f "${OUT_DIR}/spark-core_2.12-${VERSION}.jar" ]; then
+ echo "ERROR: spark-core_2.12-${VERSION}.jar missing from pack" >&2
+ exit 1
+ fi
+done
+
+echo "Done. Use HOP_SPARK_CLIENT_VERSION=<version> with hop-run / hop-conf
--spark-client-version."
diff --git a/tools/spark-client-pack/pom.xml b/tools/spark-client-pack/pom.xml
new file mode 100644
index 0000000000..ec36d0e8bc
--- /dev/null
+++ b/tools/spark-client-pack/pom.xml
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.hop.tools</groupId>
+ <artifactId>spark-client-pack</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+ <name>Hop Spark client pack materializer</name>
+ <description>
+ Resolves Spark client runtime jars for a given spark.client.version into
an output directory.
+ Used to build versioned packs under hop/lib/spark-clients/<version>/.
+ </description>
+
+ <properties>
+ <spark.client.version>3.5.8</spark.client.version>
+ <spark.scala.binary>2.12</spark.scala.binary>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${spark.scala.binary}</artifactId>
+ <version>${spark.client.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${spark.scala.binary}</artifactId>
+ <version>${spark.client.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>3.8.1</version>
+ </plugin>
+ </plugins>
+ </build>
+</project>
