This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 067c403812 Docs: version attributes in antora.yml for Java, Beam,
Spark and Flink, used where pages mean the current version. fixes #3297 (#9241)
067c403812 is described below
commit 067c4038126ebc99c19fbaf25d647c226203dafe
Author: Adalennis <[email protected]>
AuthorDate: Wed Nov 4 11:03:18 2026 +0100
Docs: version attributes in antora.yml for Java, Beam, Spark and Flink,
used where pages mean the current version. fixes #3297 (#9241)
* Docs: version attributes in antora.yml for Java, Beam, Spark and Flink,
used where pages mean the current version. #3297
* Version attributes: documented in writing-documentation, pom comments,
release check, after review. #3297
---
docs/hop-dev-manual/antora.yml | 3 ++
.../pages/apache-release/creating-a-release.adoc | 2 ++
.../modules/ROOT/pages/build-and-test.adoc | 2 +-
.../modules/ROOT/pages/docker-build.adoc | 2 +-
.../modules/ROOT/pages/hopweb/index.adoc | 6 ++--
.../ROOT/pages/marketplace-publish-plugin.adoc | 2 +-
.../modules/ROOT/pages/writing-documentation.adoc | 22 +++++++++++++
docs/hop-user-manual/antora.yml | 7 +++++
.../modules/ROOT/pages/docker-container.adoc | 2 +-
.../getting-started/hop-download-install.adoc | 2 +-
.../scheduling-workflows-and-pipelines.adoc | 2 +-
.../ROOT/pages/installation-configuration.adoc | 2 +-
.../pages/pipeline/beam/beam-samples-flink.adoc | 2 +-
.../pipeline/beam/getting-started-with-beam.adoc | 2 +-
.../pipeline/beam/running-the-beam-samples.adoc | 2 +-
.../beam-flink-pipeline-engine.adoc | 2 +-
.../beam-spark-pipeline-engine.adoc | 14 ++++-----
.../native-spark-pipeline-engine.adoc | 2 +-
.../spark/cluster-mapping-walkthrough.adoc | 2 +-
.../ROOT/pages/pipeline/spark/databricks.adoc | 6 ++--
.../spark/getting-started-with-native-spark.adoc | 12 ++++----
.../ROOT/pages/pipeline/spark/lakehouse.adoc | 2 +-
.../pipeline/spark/local-standalone-cluster.adoc | 36 +++++++++++-----------
.../modules/ROOT/pages/pipeline/spark/yarn.adoc | 14 ++++-----
.../pipeline/transforms/beamsnowflakeinput.adoc | 2 +-
.../pipeline/transforms/beamsnowflakeoutput.adoc | 2 +-
.../pipeline/transforms/beamsplunkoutput.adoc | 2 +-
.../pages/pipeline/transforms/textfileinput.adoc | 2 +-
.../hop-user-manual/modules/ROOT/pages/sizing.adoc | 4 +--
.../field-metadata/date-format-symbols.adoc | 2 +-
.../field-metadata/number-format-symbols.adoc | 2 +-
.../modules/ROOT/pages/supported-jvms.adoc | 2 +-
plugins/engines/beam/pom.xml | 2 ++
plugins/engines/spark/pom.xml | 1 +
pom.xml | 2 ++
35 files changed, 106 insertions(+), 67 deletions(-)
diff --git a/docs/hop-dev-manual/antora.yml b/docs/hop-dev-manual/antora.yml
index 0612ccd5cb..aa04483e95 100644
--- a/docs/hop-dev-manual/antora.yml
+++ b/docs/hop-dev-manual/antora.yml
@@ -21,3 +21,6 @@ version: 2.21.0
prerelease: true
nav:
- modules/ROOT/nav.adoc
+asciidoc:
+ attributes:
+ java-version: '21'
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/apache-release/creating-a-release.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/apache-release/creating-a-release.adoc
index 2fd34bbb4a..600654528f 100644
---
a/docs/hop-dev-manual/modules/ROOT/pages/apache-release/creating-a-release.adoc
+++
b/docs/hop-dev-manual/modules/ROOT/pages/apache-release/creating-a-release.adoc
@@ -401,6 +401,8 @@ Update following files and set the correct documentation
version and name
Remove the `prerelease` and `display_version` attributes
+Check that the version attributes under `asciidoc.attributes` in both files
still match the poms, see
xref:writing-documentation.adoc#_version_attributes[Version attributes].
+
== [[CreatingAKey]]Creating a key
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/build-and-test.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/build-and-test.adoc
index 131dc0d6ed..aa211caa07 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/build-and-test.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/build-and-test.adoc
@@ -20,7 +20,7 @@ under the License.
== What you need
-* A JDK 21. The build targets Java 21 and the tests are skipped on anything
older.
+* A JDK {java-version}. The build targets Java {java-version} and the tests
are skipped on anything older.
* Maven 3.6.3 or newer — or nothing at all, and use the bundled wrapper
`./mvnw`.
[source,bash]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/docker-build.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/docker-build.adoc
index dc51b69b9e..a4976929af 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/docker-build.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/docker-build.adoc
@@ -170,7 +170,7 @@ The following image stages can be specified with the
`--images` argument:
|`client`
|`hop`
-|Main Hop client/server image. Based on Alpine Linux with OpenJDK 21. Can run
pipelines, workflows, and Hop Server.
+|Main Hop client/server image. Based on Alpine Linux with OpenJDK
{java-version}. Can run pipelines, workflows, and Hop Server.
|`web`
|`hop-web`
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/hopweb/index.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/hopweb/index.adoc
index 0852517fbe..2f00a34e85 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/hopweb/index.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/hopweb/index.adoc
@@ -30,7 +30,7 @@ WARNING: Hop Web is built with
https://eclipse.dev/rap/demos/[Eclipse RAP], and
Before starting, ensure you have:
-* **Java 21** installed and available in your PATH
+* **Java {java-version}** installed and available in your PATH
* **Apache Hop** built from source (run `mvn clean install -DskipTests` in the
Hop source directory)
* **Apache Tomcat 10** downloaded and extracted
@@ -53,7 +53,7 @@ Additional info in
xref:hopweb/developer-guide.adoc#_building_the_hop_web_docker
We'll use Apache Tomcat in this example. If you use another application
server, the process should be similar.
-Hop 2.x is built with Java 21, so you'll need to
https://tomcat.apache.org/download-10.cgi[download the latest Tomcat 10] (or
use the official Tomcat 10 Docker image).
+This version of Hop is built with Java {java-version}, so you'll need to
https://tomcat.apache.org/download-10.cgi[download the latest Tomcat 10] (or
use the official Tomcat 10 Docker image).
Copy or extract the following files from your Hop build, where
`$CATALINA_HOME` is your Tomcat installation folder.
@@ -235,7 +235,7 @@ image:hop-web.png[Hop Web, width="90%"]
=== Tomcat won't start
* **Check Tomcat logs**: `$CATALINA_HOME/logs/catalina.out` for errors
-* **Verify Java 21**: Run `java -version` (should show version 17)
+* **Verify Java {java-version}**: Run `java -version` (should show version
{java-version})
* **Check port availability**: `lsof -i :8080` (Linux/Mac) - port 8080 must be
free
=== Permission denied errors
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/marketplace-publish-plugin.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/marketplace-publish-plugin.adoc
index 03a02a8c6e..e73cc220f5 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/marketplace-publish-plugin.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/marketplace-publish-plugin.adoc
@@ -66,7 +66,7 @@ TIP: Apache Hop’s own optional plugins use `groupId`
`org.apache.hop` and are
== Prerequisites
-* Java 21 and Maven 3.6.3+
+* Java {java-version} and Maven 3.6.3+
* A Hop version to *compile against* (Maven property, for example
`hop.version=2.19.0`)
* A Hop client for smoke-testing install (same major.minor line you support)
* A Maven repository (Sonatype Nexus OSS/Pro, Artifactory, etc.):
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/writing-documentation.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/writing-documentation.adoc
index 262193fd9b..854fbbef88 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/writing-documentation.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/writing-documentation.adoc
@@ -65,6 +65,28 @@ The path is relative to the user manual on the website, and
it is what the help
Anything a plugin developer needs — rather than a user — belongs in this
manual instead.
+== Version attributes
+
+The versions of Java, Apache Beam, Spark and Flink that Hop uses are
attributes in the `antora.yml` of the manual, so a page doesn't repeat the
number:
+
+[options="header",cols="1,1,2"]
+|===
+|Attribute |Manual |Comes from
+|`+{java-version}+` |user and dev manual |`target.jdk.version` in the root
`pom.xml`
+|`+{beam-version}+` |user manual |`apache-beam.version` in the root `pom.xml`
+|`+{beam-spark-version}+` |user manual |`spark.version` in
`plugins/engines/beam/pom.xml` (the Spark of the Beam Spark runner)
+|`+{spark-version}+` |user manual |`spark.version` in
`plugins/engines/spark/pom.xml` (the native Spark engine)
+|`+{flink-version}+` |user manual |the `beam-runners-flink-<version>`
dependency in `plugins/engines/beam/pom.xml`
+|===
+
+Use an attribute where the text means the version Hop uses now: `+Hop requires
Java {java-version}+`.
+Keep the number where it shouldn't change with an upgrade: a past version
(`since 2.15.0`), a measurement or a log example, a table with a row per
release, or the behaviour of one specific version of a library.
+A code block only replaces attributes when it has `subs="attributes+"`:
`++[source,bash,subs="attributes+"]++`.
+
+When a pull request changes one of these versions in a pom, it also updates
the attribute in the `antora.yml` files listed above, so the `next`
documentation shows the new version.
+A comment next to each of them in the poms says so.
+A new Java version also changes the JRE that `docker/Dockerfile` installs,
which xref:docker-build.adoc[Building the Docker images] and the Docker
container page of the user manual describe with `+{java-version}+`.
+
== Previewing the documentation locally
The manuals are published by the Antora playbooks in the
https://github.com/apache/hop-website[apache/hop-website] repository, not this
one.
diff --git a/docs/hop-user-manual/antora.yml b/docs/hop-user-manual/antora.yml
index eb16d16d84..32d0cf7a7e 100644
--- a/docs/hop-user-manual/antora.yml
+++ b/docs/hop-user-manual/antora.yml
@@ -22,3 +22,10 @@ prerelease: true
display_version: 2.21.0 (pre-release)
nav:
- modules/ROOT/nav.adoc
+asciidoc:
+ attributes:
+ java-version: '21'
+ beam-version: '2.74.0'
+ beam-spark-version: '3.5.8'
+ spark-version: '4.1.3'
+ flink-version: '1.19'
diff --git a/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
b/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
index 0d2530e111..55d9a57324 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
@@ -37,7 +37,7 @@ For the Hop Web GUI image (`apache/hop-web`), including which
folders to persist
== Operating system
The docker container runs a minimal Linux system called
https://hub.docker.com/_/alpine[Alpine].
-OpenJDK version 21 is then used to execute Apache Hop.
+OpenJDK version {java-version} is then used to execute Apache Hop.
The Linux user used to execute in the container is `hop` and the group is
`hop` as well.
== Container Folder Structure
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-download-install.adoc
b/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-download-install.adoc
index a06ebcf90b..390ec212e2 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-download-install.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-download-install.adoc
@@ -44,7 +44,7 @@ OpenJDK Runtime Environment (build 21.0.10)
OpenJDK 64-Bit Server VM (build 21.0.10, mixed mode, sharing)
----
-If that command is not found, or reports an older version, install a Java 21
runtime first.
+If that command is not found, or reports an older version, install a Java
{java-version} runtime first.
See xref:supported-jvms.adoc[Supported Java Versions] for which runtimes are
known to work, and which one to avoid.
== Download and unzip
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/scheduling-workflows-and-pipelines.adoc
b/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/scheduling-workflows-and-pipelines.adoc
index 2e74791757..ff81c6d1f6 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/scheduling-workflows-and-pipelines.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/scheduling-workflows-and-pipelines.adoc
@@ -101,7 +101,7 @@ They are CI systems first; use them when that is already
where the work is trigg
For *hop-run on a host*:
-* A Hop client on that host (the same archive you unzip for Hop Gui), and Java
21 on `PATH` or in `HOP_JAVA_HOME` / `JAVA_HOME`.
+* A Hop client on that host (the same archive you unzip for Hop Gui), and Java
{java-version} on `PATH` or in `HOP_JAVA_HOME` / `JAVA_HOME`.
See xref:installation-configuration.adoc[Installation and configuration].
* The project and a lifecycle environment registered in `HOP_CONFIG_FOLDER`,
so `-e prod` is enough.
See xref:projects/projects-environments.adoc[Projects and environments] and
the xref:hop-server/deploy-git-checkout.adoc[git checkout] setup.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/installation-configuration.adoc
b/docs/hop-user-manual/modules/ROOT/pages/installation-configuration.adoc
index 277fd2f339..e1c78c74ab 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/installation-configuration.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/installation-configuration.adoc
@@ -55,7 +55,7 @@ Apache Hop is known to work well on the following widely used
Java Runtimes:
* https://www.microsoft.com/openjdk[Microsoft OpenJDK] (OpenJDK builds for
Windows, MacOS and Linux).
* https://openjdk.java.net/install/[OpenJDK Java Runtime]
-Apache Hop works well with these 64-bit java runtimes for version 21.
+Apache Hop works well with these 64-bit java runtimes for version
{java-version}.
Other Java Runtimes may work but haven't been used and tested as extensively
as the Oracle and OpenJDK JREs, so you may be pioneering. Feel free to open a
https://hop.apache.org/community/tools/#GitHub-Issues[GitHub ticket] if you run
into issues, but please mention your JRE and version.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/beam-samples-flink.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/beam-samples-flink.adoc
index 82a11f6dc3..e7029a8179 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/beam-samples-flink.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/beam-samples-flink.adoc
@@ -27,7 +27,7 @@ under the License.
Check the
xref:pipeline/beam/getting-started-with-beam.adoc#supportedversions[Supported
Versions] on the Getting Started With Beam page to find the latest supported
Flink version for your Hop version.
-For example, for Hop 1.2, the latest currently supported version is 1.13. Make
sure to download Flink for a recent and JDK 8 compatible Scale version. For
Flink 1.3.6, this is Scala 2.11.
+For example, this version of Hop supports Flink {flink-version}: download a
Flink {flink-version}.x release.
Download your selected Flink version and unzip to a convenient location.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
index f52c66ed68..a1f451bee1 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/getting-started-with-beam.adoc
@@ -52,7 +52,7 @@ Here is the documentation for the relevant plugins:
[NOTE]
====
-*Java:* Hop requires *Java 21* for the Hop process (GUI, hop-run, hop-server)
and for the fat jar executed on Beam workers. Spark/Flink clusters must run a
JVM that can load that bytecode (Java 21).
+*Java:* Hop requires *Java {java-version}* for the Hop process (GUI, hop-run,
hop-server) and for the fat jar executed on Beam workers. Spark/Flink clusters
must run a JVM that can load that bytecode (Java {java-version}).
*Spark (measured):* With Hop 2.19 / Beam 2.74 on *Java 21*, only *Spark 3.5.x*
is supported for client-mode submit (`spark://…`). Spark 3.4 and earlier fail
on Java 21 with `NoSuchMethodException:
java.nio.DirectByteBuffer.<init>(long,int)`
([SPARK-42369](https://issues.apache.org/jira/browse/SPARK-42369), fixed in
Spark 3.5.0). Matching client packs to the cluster does not overcome that JDK
limit. See `integration-tests/spark/MATRIX-REPORT.md`. Beam upstream docs may
still mention 3.2.x [...]
====
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/running-the-beam-samples.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/running-the-beam-samples.adoc
index 459900f407..4266e0eed0 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/running-the-beam-samples.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/beam/running-the-beam-samples.adoc
@@ -30,7 +30,7 @@ The steps on this page and the detail pages for Spark, Flink
and Google Dataflow
=== Java
-You'll already have Java installed to run Apache Hop. Both Apache Hop and Beam
require a Java 21 environment.
+You'll already have Java installed to run Apache Hop. Both Apache Hop and Beam
require a Java {java-version} environment.
Double-check your java version with the `java -version` command. Your output
should look similar to the one below.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.adoc
index dce4a9944f..599d0f5367 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.adoc
@@ -22,7 +22,7 @@ under the License.
== Beam Flink
-This runner allows you to run Hop pipelines on https://flink.apache.org[Apache
Flink] version 1.19. See
xref:pipeline/beam/getting-started-with-beam.adoc[Getting started with Apache
Beam] for the Beam and Flink versions of each Hop release.
+This runner allows you to run Hop pipelines on https://flink.apache.org[Apache
Flink] version {flink-version}. See
xref:pipeline/beam/getting-started-with-beam.adoc[Getting started with Apache
Beam] for the Beam and Flink versions of each Hop release.
The Flink runner supports two modes: Local Direct Flink Runner and Flink
Runner.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
index c7d08b62c1..c0603ab9d5 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc
@@ -25,23 +25,23 @@ under the License.
The Apache Spark Runner can be used to execute Beam pipelines using Apache
Spark.
-IMPORTANT: Hop itself always runs on *Java 21* (driver and fat-jar bytecode).
Spark standalone clusters used with Hop must also provide a Java 21 runtime so
workers can execute Hop transforms.
+IMPORTANT: Hop itself always runs on *Java {java-version}* (driver and fat-jar
bytecode). Spark standalone clusters used with Hop must also provide a Java
{java-version} runtime so workers can execute Hop transforms.
*Measured support (Hop 2.19 / Beam 2.74, client submit):* *Spark 3.5.x* only
(validated *3.5.8*). Hop requires *Java 21*; Spark did not handle the JDK 21
`DirectByteBuffer` constructor change until
[SPARK-42369](https://issues.apache.org/jira/browse/SPARK-42369) (fixed in
*Spark 3.5.0*). Therefore *Spark 3.4.x and earlier cannot run* with Hop on Java
21 — even with a matching client pack and fat jar.
-For `spark://master` client mode on 3.5.x, the **Spark client libraries on the
Hop driver and in the fat jar should still match the cluster patch/minor**
(default pack under `lib/spark-client/`, currently 3.5.8).
+For `spark://master` client mode on 3.5.x, the **Spark client libraries on the
Hop driver and in the fat jar should still match the cluster patch/minor**
(default pack under `lib/spark-client/`, currently {beam-spark-version}).
=== Matching Spark client libraries (3.5.x)
Hop installs a default Spark client pack under `lib/spark-client/` (beam
engine `spark.version`). Optional packs under `lib/spark-clients/<version>/`
are for aligning with a *3.5.x* cluster build, not for downgrading below 3.5.
.Materialise / select a 3.5.x pack
-[source,bash]
+[source,bash,subs="attributes+"]
----
-./tools/spark-client-pack/materialize-pack.sh 3.5.8 /path/to/hop
-export HOP_SPARK_CLIENT_VERSION=3.5.8
-./hop-conf.sh --generate-fat-jar=/path/to/hop-spark-3.5.8.jar \
- --spark-client-version=3.5.8
+./tools/spark-client-pack/materialize-pack.sh {beam-spark-version} /path/to/hop
+export HOP_SPARK_CLIENT_VERSION={beam-spark-version}
+./hop-conf.sh --generate-fat-jar=/path/to/hop-spark-{beam-spark-version}.jar \
+ --spark-client-version={beam-spark-version}
./hop-run.sh -r spark -f your-pipeline.hpl
----
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc
index 9b270756e5..5fb341fce3 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc
@@ -38,7 +38,7 @@
image::pipeline/spark/native-spark-pipeline-engine-run-configurtion-editor.png[N
=== Requirements
-* Java 21
+* Java {java-version}
* Spark 4.1.x (bundled under `plugins/engines/spark/lib` for `+local[*]+`)
=== Options
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
index 6156f2771a..7015b8df4d 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
@@ -307,7 +307,7 @@ Host data under `/tmp/spark-demo-dist/hop-data` is left in
place so you can keep
|`HOP_DATA` must be the **shared** volume path (`file:///data/hop-data`). Do
not point Dataset paths at package `PROJECT_HOME`.
|ClassNotFoundException / wrong Spark
-|Fat jar must be **`native-provided`**. Cluster image must be Spark **4.1.x**
+ **Java 21**. Do not mix Beam Spark 3 fat jars.
+|Fat jar must be **`native-provided`**. Cluster image must be Spark **4.1.x**
+ **Java {java-version}**. Do not mix Beam Spark 3 fat jars.
|Driver cannot reverse-connect workers
|Submit script sets `spark.driver.host=spark` (master hostname on the compose
network). Adjust if you change service names.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc
index 611a82012d..79cbb1f10d 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc
@@ -88,7 +88,7 @@ all-purpose cluster** or a classic **job cluster**
(`new_cluster` in the Jobs AP
* **Free tier / “cannot create a cluster”** — many Free or UI-first workspaces
never expose a classic cluster id. Deploy & run then has nothing valid to put
in `existing_cluster_id`. Putting the literal string `new_cluster` into that
field without a nested object produces errors such as _Cluster new_cluster does
not exist_ (the API treated it as a cluster id).
* **Serverless-only error** — `Only serverless compute is supported in the
workspace` means classic `new_cluster` / existing all-purpose clusters are
blocked. Serverless needs job-level `environments` and task `environment_key`,
and **must not** send task-level `libraries` the same way classic jobs do.
-* **Spark 4.1 / Java 21 style workloads** — for Hop’s **native** Spark 4.x
engine and a fat jar built for modern runtimes, provision a workspace that
**allows classic clusters** (e.g. AWS Premium workspace) and a DBR / job
cluster configuration aligned with your Spark and JVM expectations. Do not
assume Free/serverless will match local `native-provided` fat-jar testing.
+* **Spark 4.1 / Java {java-version} style workloads** — for Hop’s **native**
Spark 4.x engine and a fat jar built for modern runtimes, provision a workspace
that **allows classic clusters** (e.g. AWS Premium workspace) and a DBR / job
cluster configuration aligned with your Spark and JVM expectations. Do not
assume Free/serverless will match local `native-provided` fat-jar testing.
* **New workspace for classic** — if the current workspace is permanently
serverless-oriented, creating a **new** classic-capable workspace (cloud
account + tier that allows classic compute) is often simpler than fighting
Free-tier product limits.
* [[lesson-fat-jar-library-uri]]**Fat jar library URI must change when jar
bytes change (Dedicated / long-lived cluster “caching crapout”)** — overwriting
a **fixed** Volume path such as `/Volumes/…/hop-native.jar` (even with Files
API `?overwrite=true` and a fresh `.sha256` sidecar) is **not** enough for
Databricks to always run the new classes. A long-lived **Dedicated** (or other
all-purpose) cluster often keeps **previously installed job libraries** on the
driver/executor classpath ( [...]
**Volume object is minutes old and matches local `sha256sum`**, but
MainSpark driver log still shows **old behavior** (e.g. missing `>>>>>> Spark
project package distribution build: pkg-dist-…` right after `Initializing Hop`,
or obsolete package-distribution log strings such as `cluster-shared … (no
SparkFiles)`). Hop Deploy & run therefore uploads a **content-addressed**
library path `hop-native-<sha12>.jar` and points the job’s `libraries[].jar` at
that URI so each content change is [...]
@@ -340,7 +340,7 @@ The **Cluster / compute** field accepts one of:
|Literal **`serverless`**
|Job: `environments` with `environment_key` + `spec.client` +
`java_dependencies`; task: `environment_key` only (no cluster fields, no task
`libraries`)
-|Serverless-only workspaces. JAR-on-serverless is a different product surface
(runtime limits, library rules). Prefer classic for Hop native Spark 4.x / Java
21 style deploy until you explicitly target serverless.
+|Serverless-only workspaces. JAR-on-serverless is a different product surface
(runtime limits, library rules). Prefer classic for Hop native Spark 4.x / Java
{java-version} style deploy until you explicitly target serverless.
|===
=== Classic job cluster (`new_cluster`) defaults
@@ -501,7 +501,7 @@ Keep `m5.xlarge` (or similar) and supply a full
`new_cluster` JSON that includes
|Do not put the word `new_cluster` as an existing cluster id without the
nested object. Use cluster field **`new_cluster`** so Hop emits a real
`new_cluster` block.
|Only serverless compute is supported
-|Workspace is serverless-only. Use classic-capable workspace/tier for
MainSpark JAR, or `serverless` compute mode (different job shape; not the
default for Hop native Spark 4.x / Java 21 style deploy).
+|Workspace is serverless-only. Use classic-capable workspace/tier for
MainSpark JAR, or `serverless` compute mode (different job shape; not the
default for Hop native Spark 4.x / Java {java-version} style deploy).
|===
[[faq-exit-marks-failed]]
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
index 2ca4429486..b939b16f43 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
@@ -74,7 +74,7 @@ The *native* engine skips Beam entirely:
|Beam pipelines on Spark (batch *and* streaming)
|Spark line
-|https://spark.apache.org/docs/4.1.3/[4.1.x] (Scala 2.13)
+|https://spark.apache.org/docs/{spark-version}/[4.1.x] (Scala 2.13)
|3.5.x (Scala 2.12; see Beam support matrix)
|Programming model
@@ -123,9 +123,9 @@ You still design pipelines *visually* in Hop. Spark’s role
is the distributed
[NOTE]
====
-*Java:* Hop requires *Java 21* for the GUI, `hop-run`, and the fat jar
executed on the Spark driver/executors. Your Spark cluster must also run a JVM
that can load that bytecode (Java 21).
+*Java:* Hop requires *Java {java-version}* for the GUI, `hop-run`, and the fat
jar executed on the Spark driver/executors. Your Spark cluster must also run a
JVM that can load that bytecode (Java {java-version}).
-*Spark:* The native engine is built and tested against *Spark 4.1.x*
(currently *4.1.3* in the Hop source tree, Scala *2.13*). For `+local[*]+`
runs, Spark libraries are packaged under `plugins/engines/spark/lib`, so you do
*not* need a separate Spark install on your laptop.
+*Spark:* The native engine is built and tested against *Spark 4.1.x*
(currently *{spark-version}* in the Hop source tree, Scala *2.13*). For
`+local[*]+` runs, Spark libraries are packaged under
`plugins/engines/spark/lib`, so you do *not* need a separate Spark install on
your laptop.
====
[options="header",cols="1,1,2,2"]
@@ -149,7 +149,7 @@ Do *not* mix this engine with Beam’s Spark 3.5 fat-jar
packs or an accidental
* Apache Hop *2.19+* with the `hop-engines-spark` plugin. It is not part of
the default client download: install it with `./hop marketplace install
hop-engines-spark` (see xref:hop-tools/hop-marketplace.adoc[Hop Marketplace])
and restart Hop.
* For cluster runs, also the `hop-engines-beam` plugin: the fat jar builder
ships with it.
-* *Java 21*.
+* *Java {java-version}*.
* Familiarity with Hop pipelines, metadata, and run configurations.
* For cluster submit:
** A *Spark 4.1.x* distribution (`SPARK_HOME` pinned to that tree).
@@ -231,7 +231,7 @@ Same run configuration, no GUI:
=== Build a fat jar for cluster submit
-On a real cluster you must *not* embed a second copy of Spark inside the
application jar. The cluster already provides Spark 4.1. Embedding Spark leads
to dual classpaths and failures on Java 21.
+On a real cluster you must *not* embed a second copy of Spark inside the
application jar. The cluster already provides Spark 4.1. Embedding Spark leads
to dual classpaths and failures on Java {java-version}.
Use the special fat-jar token *`native-provided`*: Hop and plugins go into the
jar; *Spark stays out*.
@@ -701,7 +701,7 @@ image::pipeline/spark/spark-native-0099-complex.png[0099
complex sample,width=95
== Checklist
-* [ ] Java 21; Hop 2.19+ with `plugins/engines/spark`
+* [ ] Java {java-version}; Hop 2.19+ with `plugins/engines/spark`
* [ ] Run configuration engine = *Native Spark* (not Beam Spark)
* [ ] First success on `+local[*]+` with Spark file input → transform → Spark
file output
* [ ] Execution information location attached if you need history and samples
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
index 2d7330342f..69e7075f2b 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
@@ -56,7 +56,7 @@ It assumes you already have a working *Native Spark* pipeline
run configuration
| Coordinate / version
| Spark (Hop native engine)
-| `spark-sql_2.13` *4.1.3* (Scala *2.13*)
+| `spark-sql_2.13` *{spark-version}* (Scala *2.13*)
| Delta Lake
| `io.delta:delta-spark_4.1_2.13:*4.3.1*`
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/local-standalone-cluster.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/local-standalone-cluster.adoc
index 2bf48bcb8b..47f57fda4b 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/local-standalone-cluster.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/local-standalone-cluster.adoc
@@ -32,7 +32,7 @@ Do this after your first `+local[*]+` run (see
xref:pipeline/spark/getting-start
== What you need
* Linux or macOS. Spark's start scripts are shell scripts; on Windows use WSL.
-* Java 21, both for Hop and for the Spark processes. Spark 4.1 also runs on
Java 17, but the Hop classes in the fat jar need 21.
+* Java {java-version}, both for Hop and for the Spark processes. Spark 4.1
also runs on Java 17, but the Hop classes in the fat jar need {java-version}.
* About 10 GB of free disk: 600 MB for Spark, 900 MB for the fat jar, and room
for executor copies of that jar, about 3.5 GB per run (see <<disk-space>>).
* A Hop client with the native Spark engine and the Apache Beam engine plugins
(next section).
@@ -53,26 +53,26 @@ The Beam plugin also adds the `beam` folder to the
`samples` project. The exampl
== 2. Start a Spark standalone cluster
-The cluster runs the same Spark version the engine is built with: *4.1.3*,
Scala 2.13.
+The cluster runs the same Spark version the engine is built with:
*{spark-version}*, Scala 2.13.
=== Download Spark
-[source,bash]
+[source,bash,subs="attributes+"]
----
mkdir -p ~/spark && cd ~/spark
-curl -fLO
https://dlcdn.apache.org/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz
-curl -fLO
https://downloads.apache.org/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz.sha512
-sha512sum -c spark-4.1.3-bin-hadoop3.tgz.sha512 # macOS: shasum -a 512 -c ...
-tar xzf spark-4.1.3-bin-hadoop3.tgz
+curl -fLO
https://dlcdn.apache.org/spark/spark-{spark-version}/spark-{spark-version}-bin-hadoop3.tgz
+curl -fLO
https://downloads.apache.org/spark/spark-{spark-version}/spark-{spark-version}-bin-hadoop3.tgz.sha512
+sha512sum -c spark-{spark-version}-bin-hadoop3.tgz.sha512 # macOS: shasum -a
512 -c ...
+tar xzf spark-{spark-version}-bin-hadoop3.tgz
----
-If the `dlcdn` link no longer works, the release is kept at
`https://archive.apache.org/dist/spark/spark-4.1.3/`.
+If the `dlcdn` link no longer works, the release is kept at
`https://archive.apache.org/dist/spark/spark-{spark-version}/`.
=== Configure the cluster
-Create `~/spark/spark-4.1.3-bin-hadoop3/conf/spark-env.sh`. The start scripts
read it, so the settings apply every time you start or stop the cluster.
+Create `~/spark/spark-{spark-version}-bin-hadoop3/conf/spark-env.sh`. The
start scripts read it, so the settings apply every time you start or stop the
cluster.
-[source,bash]
+[source,bash,subs="attributes+"]
----
# Master on this machine only
SPARK_MASTER_HOST=localhost
@@ -88,18 +88,18 @@ SPARK_LOCAL_DIRS=$HOME/spark/local
# Remove the folders of finished applications after an hour, checked every 10
minutes
SPARK_WORKER_OPTS="-Dspark.worker.cleanup.enabled=true
-Dspark.worker.cleanup.interval=600 -Dspark.worker.cleanup.appDataTtl=3600"
-# Uncomment if java on your PATH is not Java 21
-# JAVA_HOME=/path/to/java-21
+# Uncomment if java on your PATH is not Java {java-version}
+# JAVA_HOME=/path/to/java-{java-version}
----
Why `SPARK_LOCAL_DIRS` and `SPARK_WORKER_OPTS` matter: every executor copies
the fat jar into `SPARK_LOCAL_DIRS` (default `/tmp`) and again into the
worker's `work` folder. See <<disk-space>>.
=== Start the master and the workers
-[source,bash]
+[source,bash,subs="attributes+"]
----
mkdir -p ~/spark/local
-export SPARK_HOME=~/spark/spark-4.1.3-bin-hadoop3
+export SPARK_HOME=~/spark/spark-{spark-version}-bin-hadoop3
$SPARK_HOME/sbin/start-master.sh
$SPARK_HOME/sbin/start-worker.sh spark://localhost:7077
----
@@ -270,9 +270,9 @@ Like Hop GUI, Hop Server keeps the Spark application it
started running until Ho
== Stop the cluster
-[source,bash]
+[source,bash,subs="attributes+"]
----
-export SPARK_HOME=~/spark/spark-4.1.3-bin-hadoop3
+export SPARK_HOME=~/spark/spark-{spark-version}-bin-hadoop3
$SPARK_HOME/sbin/stop-worker.sh
$SPARK_HOME/sbin/stop-master.sh
----
@@ -285,7 +285,7 @@ $SPARK_HOME/sbin/stop-master.sh
Every executor copies the fat jar twice:
* into `SPARK_LOCAL_DIRS` while it fetches it. With the default `/tmp`, a few
parallel copies of a 900 MB jar can fill a small or quota-limited `/tmp` (for
example a `tmpfs`). The executors then fail with `Disk quota exceeded` or `No
space left on device`, and Spark keeps starting new executors that fail the
same way, so the pipeline never ends. Stop it, set `SPARK_LOCAL_DIRS` to a
folder on disk, and restart the workers.
-* into the worker's `work/<application id>/<executor id>/` folder, where it
stays after the application ends. Four executors use about 3.5 GB per run. The
`spark.worker.cleanup.*` settings in `spark-env.sh` remove those folders some
time after an application has finished. You can also delete
`~/spark/spark-4.1.3-bin-hadoop3/work/app-*` yourself while no application is
running.
+* into the worker's `work/<application id>/<executor id>/` folder, where it
stays after the application ends. Four executors use about 3.5 GB per run. The
`spark.worker.cleanup.*` settings in `spark-env.sh` remove those folders some
time after an application has finished. You can also delete
`~/spark/spark-{spark-version}-bin-hadoop3/work/app-*` yourself while no
application is running.
[[troubleshooting]]
== Troubleshooting
@@ -336,4 +336,4 @@ Every executor copies the fat jar twice:
* xref:pipeline/spark/cluster-mapping-walkthrough.adoc[Cluster demo
walk-through] (Docker, `spark-submit`)
* xref:pipeline/spark/yarn.adoc[Run native Spark pipelines on YARN]
* xref:hop-tools/hop-marketplace.adoc[Hop Marketplace]
-* https://spark.apache.org/docs/4.1.3/spark-standalone.html[Spark standalone
mode] (Apache Spark documentation)
+* https://spark.apache.org/docs/{spark-version}/spark-standalone.html[Spark
standalone mode] (Apache Spark documentation)
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/yarn.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/yarn.adoc
index 9d667a62e0..3f3f9cfed4 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/yarn.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/yarn.adoc
@@ -34,7 +34,7 @@ Set up a
xref:pipeline/spark/local-standalone-cluster.adoc[local standalone clus
* Hop 2.20 or later with the `hop-engines-spark` and `hop-engines-beam`
plugins (see xref:pipeline/spark/local-standalone-cluster.adoc[Local standalone
cluster], steps 1 and 3), and a `native-provided` fat jar.
* The Hadoop client configuration of the cluster on the Hop machine: a folder
with `core-site.xml`, `hdfs-site.xml` and `yarn-site.xml`. On a cluster edge
node this is often `/etc/hadoop/conf`.
* Network access from the Hop machine to the YARN ResourceManager and to HDFS
(namenode and datanodes), and from the YARN nodes back to the Hop machine: the
executors connect to the Spark driver, which is Hop.
-* Java 21 on the YARN nodes, for the Spark processes in the containers. If the
nodes run an older Java by default, point the containers at a Java 21
installation (see below).
+* Java {java-version} on the YARN nodes, for the Spark processes in the
containers. If the nodes run an older Java by default, point the containers at
a Java {java-version} installation (see below).
* Data that the YARN nodes can read and write, for example on HDFS. The
executors cannot read files on the Hop machine.
Kerberos-secured clusters are not covered by this page.
@@ -67,15 +67,15 @@ Create a *Pipeline run configuration* with *Engine type*
*Native Spark pipeline
|`spark.ui.enabled=false` +
`spark.submit.deployMode=client` +
`spark.driver.host=<address of the Hop machine>` +
-`spark.executorEnv.JAVA_HOME=<Java 21 on the nodes>` +
-`spark.yarn.appMasterEnv.JAVA_HOME=<Java 21 on the nodes>`
-|`spark.driver.host` must be an address of the Hop machine that the YARN nodes
can reach. The two `JAVA_HOME` lines are only needed when the nodes' default
Java is older than 21.
+`spark.executorEnv.JAVA_HOME=<Java {java-version} on the nodes>` +
+`spark.yarn.appMasterEnv.JAVA_HOME=<Java {java-version} on the nodes>`
+|`spark.driver.host` must be an address of the Hop machine that the YARN nodes
can reach. The two `JAVA_HOME` lines are only needed when the nodes' default
Java is older than {java-version}.
|===
What Hop does with this on YARN:
* It passes every property in the `*-site.xml` files to Spark as
`spark.hadoop.<property>`. A `spark.hadoop.*` line in *Spark config* overrides
the same property from the files.
-* It sets `spark.yarn.jars` to its own Spark runtime: the jars in
`plugins/engines/spark/lib` plus the SLF4J and Jackson jars from `lib/core`,
about 285 MB. Spark uploads them to the YARN staging folder on HDFS for every
run, so the containers run exactly the Spark version Hop runs (4.1.3). If you
set `spark.yarn.jars` or `spark.yarn.archive` in *Spark config* yourself, they
must point at Spark 4.1.3 for Scala 2.13 and include those SLF4J and Jackson
jars.
+* It sets `spark.yarn.jars` to its own Spark runtime: the jars in
`plugins/engines/spark/lib` plus the SLF4J and Jackson jars from `lib/core`,
about 285 MB. Spark uploads them to the YARN staging folder on HDFS for every
run, so the containers run exactly the Spark version Hop runs
({spark-version}). If you set `spark.yarn.jars` or `spark.yarn.archive` in
*Spark config* yourself, they must point at Spark {spark-version} for Scala
2.13 and include those SLF4J and Jackson jars.
Use HDFS paths (`hdfs:///data/...`) or other storage all nodes can reach in
Spark File transforms; see xref:pipeline/spark/paths-and-filesystems.adoc[Paths
and file systems].
@@ -161,7 +161,7 @@ Stop the cluster with `docker compose -f
docker/integration-tests/integration-te
|The Hop machine cannot reach the HDFS datanodes to upload the Spark jars.
Check the datanode addresses the namenode hands out and the network between the
Hop machine and the datanodes.
|`The YARN application has already ended! It might have been killed or the
Application Master may have failed to start`
-|Look at the logs of the application's first container in the YARN UI.
`UnsupportedClassVersionError: org/apache/spark/deploy/yarn/ExecutorLauncher
has been compiled by a more recent version of the Java Runtime` means the nodes
run an older Java: set `spark.executorEnv.JAVA_HOME` and
`spark.yarn.appMasterEnv.JAVA_HOME` to a Java 21 installation on the nodes.
+|Look at the logs of the application's first container in the YARN UI.
`UnsupportedClassVersionError: org/apache/spark/deploy/yarn/ExecutorLauncher
has been compiled by a more recent version of the Java Runtime` means the nodes
run an older Java: set `spark.executorEnv.JAVA_HOME` and
`spark.yarn.appMasterEnv.JAVA_HOME` to a Java {java-version} installation on
the nodes.
|`NoClassDefFoundError: org/slf4j/LoggerFactory` in the first container
|`spark.yarn.jars` or `spark.yarn.archive` in *Spark config* leave out SLF4J
or Jackson. Remove them to use Hop's default, or add those jars.
@@ -179,4 +179,4 @@ Stop the cluster with `docker compose -f
docker/integration-tests/integration-te
* xref:pipeline/spark/getting-started-with-native-spark.adoc[Getting started
with the native Spark engine]
*
xref:pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc[Native
Spark pipeline engine] (option reference)
* xref:pipeline/spark/paths-and-filesystems.adoc[Paths and file systems on
native Spark]
-* https://spark.apache.org/docs/4.1.3/running-on-yarn.html[Running Spark on
YARN] (Apache Spark documentation)
+* https://spark.apache.org/docs/{spark-version}/running-on-yarn.html[Running
Spark on YARN] (Apache Spark documentation)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeinput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeinput.adoc
index 2030598351..461d69c909 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeinput.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeinput.adoc
@@ -24,7 +24,7 @@ limitations under the License.
== Description
-Read rows from Snowflake with Apache Beam SnowflakeIO 2.74.0. Snowflake copies
the table or query result to a GCS external stage, and Hop turns each CSV row
into a Hop row. This transform is a source: it does not accept incoming hops.
+Read rows from Snowflake with Apache Beam SnowflakeIO {beam-version}.
Snowflake copies the table or query result to a GCS external stage, and Hop
turns each CSV row into a Hop row. This transform is a source: it does not
accept incoming hops.
The read is bounded. SnowflakeIO rejects a streaming read, and this transform
does not offer one. Declared Beam compatibility is not a remote-runner
certification. No test in this release runs a pipeline against Snowflake or GCS.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeoutput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeoutput.adoc
index 2ad65d1905..d49d035d03 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeoutput.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsnowflakeoutput.adoc
@@ -24,7 +24,7 @@ limitations under the License.
== Description
-Write one bounded Hop collection to a Snowflake table with Apache Beam
SnowflakeIO 2.74.0. Hop formats each row as CSV values, SnowflakeIO stages
those files in GCS, and Snowflake copies them into the table. Connect exactly
one incoming transform. This is a terminal sink; it does not produce downstream
rows.
+Write one bounded Hop collection to a Snowflake table with Apache Beam
SnowflakeIO {beam-version}. Hop formats each row as CSV values, SnowflakeIO
stages those files in GCS, and Snowflake copies them into the table. Connect
exactly one incoming transform. This is a terminal sink; it does not produce
downstream rows.
Snowpipe is not used. An unbounded collection is rejected. Declared Beam
compatibility is not a remote-runner certification. No test in this release
runs a pipeline against Snowflake or GCS.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsplunkoutput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsplunkoutput.adoc
index ac863a7f21..cbf22d7c69 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsplunkoutput.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamsplunkoutput.adoc
@@ -24,7 +24,7 @@ limitations under the License.
== Description
-Publish one Splunk HTTP Event Collector (HEC) event for each incoming Hop row
using Apache Beam SplunkIO 2.74.0. Connect exactly one incoming transform. This
is a terminal sink; it does not produce downstream rows. There is no Splunk
input transform: SplunkIO is write-only.
+Publish one Splunk HTTP Event Collector (HEC) event for each incoming Hop row
using Apache Beam SplunkIO {beam-version}. Connect exactly one incoming
transform. This is a terminal sink; it does not produce downstream rows. There
is no Splunk input transform: SplunkIO is write-only.
The HEC token is sent as `Authorization: Splunk <token>`. The request URL is
the configured base URL plus `/services/collector/event`. Declared Beam
compatibility is not a remote-runner certification.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textfileinput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textfileinput.adoc
index 92f1869d89..54e303eaee 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textfileinput.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textfileinput.adoc
@@ -67,7 +67,7 @@ Otherwise, the filename is skipped.
The Text File Input transform can search for files by wildcard in the form of
a regular expression.
Regular expressions are more sophisticated than using '*' and '?' wildcards.
-The expressions use the Java syntax, described in
https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/regex/Pattern.html[java.util.regex.Pattern]:
+The expressions use the Java syntax, described in
https://docs.oracle.com/en/java/javase/{java-version}/docs/api/java.base/java/util/regex/Pattern.html[java.util.regex.Pattern]:
* The expression has to match the whole file name, as if it started with `^`
and ended with `$`. `userdata` only selects a file named `userdata`, not
`userdata.txt`.
* The expression is matched against the file name only, not against its
folder, also when sub folders are included.
diff --git a/docs/hop-user-manual/modules/ROOT/pages/sizing.adoc
b/docs/hop-user-manual/modules/ROOT/pages/sizing.adoc
index f417f267ac..3833006f24 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/sizing.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/sizing.adoc
@@ -22,7 +22,7 @@ under the License.
Hop does not reserve a fixed amount of memory or disk for a pipeline.
What you need depends on which tool you start, which plugins you keep, and
whether a transform holds rows or passes them through.
-xref:installation-configuration.adoc[Installation and configuration] is the
floor for a default client: Java 21, one CPU core and 4 GB of RAM.
+xref:installation-configuration.adoc[Installation and configuration] is the
floor for a default client: Java {java-version}, one CPU core and 4 GB of RAM.
This page is the next step.
It covers a stripped `hop-run` on a small device, a workstation for day-to-day
design, and batch pipelines that run on Spark or Beam.
@@ -226,7 +226,7 @@ A pipeline that is larger than the workstation should not
be forced through the
Pick a run configuration that hands the data to another engine:
* xref:pipeline/spark/getting-started-with-native-spark.adoc[Native Spark] for
batch pipelines on Spark 4.1.
- The cluster JVM must be able to run Java 21, the same as Hop.
+ The cluster JVM must be able to run Java {java-version}, the same as Hop.
* xref:pipeline/beam/getting-started-with-beam.adoc[Apache Beam] when you need
Spark 3.5, Flink, Dataflow, streaming or windows.
The native Spark and Beam engines are optional plugins.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/date-format-symbols.adoc
b/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/date-format-symbols.adoc
index c665fe19f9..87553394f0 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/date-format-symbols.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/date-format-symbols.adoc
@@ -16,7 +16,7 @@ under the License.
////
Date and timestamp format masks follow Java `SimpleDateFormat`.
-See the
https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/text/SimpleDateFormat.html[Java
21 SimpleDateFormat] documentation for the full pattern language.
+See the
https://docs.oracle.com/en/java/javase/{java-version}/docs/api/java.base/java/text/SimpleDateFormat.html[Java
{java-version} SimpleDateFormat] documentation for the full pattern language.
[options="header"]
|===
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/number-format-symbols.adoc
b/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/number-format-symbols.adoc
index beb820d264..11c4b2011a 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/number-format-symbols.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/snippets/field-metadata/number-format-symbols.adoc
@@ -16,7 +16,7 @@ under the License.
////
Number format masks follow Java `DecimalFormat`.
-See the
https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/text/DecimalFormat.html[Java
21 DecimalFormat] documentation for the full pattern language.
+See the
https://docs.oracle.com/en/java/javase/{java-version}/docs/api/java.base/java/text/DecimalFormat.html[Java
{java-version} DecimalFormat] documentation for the full pattern language.
[options="header"]
|===
diff --git a/docs/hop-user-manual/modules/ROOT/pages/supported-jvms.adoc
b/docs/hop-user-manual/modules/ROOT/pages/supported-jvms.adoc
index 2651e37534..6cd10f2151 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/supported-jvms.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/supported-jvms.adoc
@@ -30,7 +30,7 @@ TIP: Take a look at the
xref:installation-configuration.adoc[Installation] page
== Supported JVMs
-Apache Hop works well with these 64-bit java runtimes for version 21.
+Apache Hop works well with these 64-bit java runtimes for version
{java-version}.
* https://www.java.com/[Oracle Java Runtime]
* https://www.microsoft.com/openjdk[Microsoft OpenJDK] (OpenJDK builds for
Windows, MacOS and Linux).
diff --git a/plugins/engines/beam/pom.xml b/plugins/engines/beam/pom.xml
index 2381717597..37b8ed2220 100644
--- a/plugins/engines/beam/pom.xml
+++ b/plugins/engines/beam/pom.xml
@@ -29,6 +29,7 @@
<name>Hop Plugins Engines Beam</name>
<properties>
+ <!-- Also update beam-spark-version in docs/hop-user-manual/antora.yml
-->
<spark.version>3.5.8</spark.version>
</properties>
@@ -364,6 +365,7 @@
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
+ <!-- Also update flink-version in docs/hop-user-manual/antora.yml
-->
<artifactId>beam-runners-flink-1.19</artifactId>
<exclusions>
<exclusion>
diff --git a/plugins/engines/spark/pom.xml b/plugins/engines/spark/pom.xml
index bbf093d86d..4d84b67ecc 100644
--- a/plugins/engines/spark/pom.xml
+++ b/plugins/engines/spark/pom.xml
@@ -40,6 +40,7 @@
<lakehouse.connectors.dir>${project.basedir}/target/lakehouse-connectors</lakehouse.connectors.dir>
<scala.binary.version>2.13</scala.binary.version>
<spark.parquet.version>1.16.0</spark.parquet.version>
+ <!-- Also update spark-version in docs/hop-user-manual/antora.yml -->
<spark.version>4.1.3</spark.version>
<spark.version.prefix>4.1</spark.version.prefix>
</properties>
diff --git a/pom.xml b/pom.xml
index d657508415..eb36022735 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,6 +99,7 @@
<properties>
<!-- One version of each library in the client assembly. hop-libs uses
the same properties. -->
<animal-sniffer-annotations.version>1.27</animal-sniffer-annotations.version>
+ <!-- Also update beam-version in docs/hop-user-manual/antora.yml -->
<apache-beam.version>2.74.0</apache-beam.version>
<api-common.version>2.62.0</api-common.version>
<assembly_appendId>false</assembly_appendId>
@@ -206,6 +207,7 @@
<spotless.skip>false</spotless.skip>
<stax2-api.version>4.3.2</stax2-api.version>
<swtbot.version>4.3.0</swtbot.version>
+ <!-- Also update java-version in docs/hop-user-manual/antora.yml and
docs/hop-dev-manual/antora.yml, and the JRE in docker/Dockerfile -->
<target.jdk.version>21</target.jdk.version>
<tm4e.core.version>0.17.2</tm4e.core.version>
<ui.test.argLine></ui.test.argLine>