This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 1a84051451 minor spark upgrade, fixes #7672 (#7673)
1a84051451 is described below
commit 1a840514514d95a5702e38049536c25bd0018872
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Jul 29 13:06:37 2026 +0200
minor spark upgrade, fixes #7672 (#7673)
---
.dockerignore | 3 +
docker/build-hop-images.sh | 73 +++-
.../integration-tests-spark-native-cluster.yaml | 4 +-
docker/unified.Dockerfile | 60 ++-
.../spark/cluster-mapping-walkthrough.adoc | 2 +-
.../spark/getting-started-with-native-spark.adoc | 4 +-
.../ROOT/pages/pipeline/spark/lakehouse.adoc | 2 +-
lib/pom.xml | 80 +---
plugins/databases/clickhouse/pom.xml | 3 +-
plugins/engines/spark/README.md | 455 ---------------------
plugins/engines/spark/pom.xml | 31 +-
plugins/misc/marketplace/README.md | 1 +
pom.xml | 2 +-
tools/install-wave1-plugins.sh | 8 +
14 files changed, 163 insertions(+), 565 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index ada15024a7..d96baeafb8 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -60,3 +60,6 @@
!lib/**
!lib-jdbc/
!lib-jdbc/**
+# Build helpers used inside the unified Dockerfile (marketplace-optional
plugin install)
+!tools/
+!tools/**
diff --git a/docker/build-hop-images.sh b/docker/build-hop-images.sh
index e9e73f9f86..8f04949bc9 100755
--- a/docker/build-hop-images.sh
+++ b/docker/build-hop-images.sh
@@ -40,6 +40,9 @@
# --progress <mode> Docker build output mode: auto (default),
plain (verbose), tty (compact)
# --builder <type> Builder type: full (Maven, default) or fast
(pre-built artifacts)
# --skip-fat-jar Skip building the fat jar (auto-detected
based on images, only needed for dataflow/web-beam)
+# --with-optional-plugins Install the marketplace-optional (Wave 1)
plugins into the image
+# (auto-enabled for dataflow/web-beam, which
need the Beam engine)
+# --no-optional-plugins Never install the marketplace-optional
plugins
# --no-cache Build without using cache
# -h, --help Show this help message
#
@@ -68,6 +71,12 @@
# # Manually skip fat jar generation (saves time if not building
dataflow/web-beam)
# ./build-hop-images.sh --images web,client --skip-fat-jar
#
+# # Ship the marketplace-optional (Wave 1) plugins inside the server image
+# ./build-hop-images.sh --images client --with-optional-plugins
+#
+# # Same, reusing the plugin zips from a local build instead of rebuilding
them
+# ./build-hop-images.sh --images client --with-optional-plugins --builder
fast
+#
################################################################################
set -e
@@ -113,6 +122,7 @@ MAVEN_THREADS="1C" # Maven thread count (1C, 2C, or
specific number like 4)
BUILD_PROGRESS="auto" # Docker build progress output (auto, plain, tty)
BUILDER_TYPE="full" # Builder type: full (Maven build) or fast (pre-built
artifacts)
SKIP_FAT_JAR="auto" # Whether to skip fat jar generation (auto, true, false)
+OPTIONAL_PLUGINS="auto" # Install marketplace-optional (Wave 1) plugins
(auto, true, false)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
@@ -204,6 +214,19 @@ needs_fat_jar() {
return 1
}
+# Function to determine whether a stage needs the marketplace-optional (Wave
1) plugins.
+# The Beam engine is one of them and it owns --generate-fat-jar, so anything
built around the
+# fat jar can only work with the optional plugins installed.
+stage_needs_optional_plugins() {
+ local stage_name="$1"
+
+ if [[ "$stage_name" == "dataflow"* ]] || [[ "$stage_name" == "web-beam"*
]]; then
+ return 0
+ fi
+
+ return 1
+}
+
# Function to detect version from pom.xml
detect_version() {
if [ -f "$PROJECT_ROOT/pom.xml" ]; then
@@ -320,6 +343,14 @@ parse_args() {
SKIP_FAT_JAR="true"
shift
;;
+ --with-optional-plugins)
+ OPTIONAL_PLUGINS="true"
+ shift
+ ;;
+ --no-optional-plugins)
+ OPTIONAL_PLUGINS="false"
+ shift
+ ;;
--no-cache)
USE_CACHE="false"
shift
@@ -407,14 +438,41 @@ build_image() {
if [[ "$skip_fat_jar_value" == "true" ]]; then
print_info "Skipping fat jar generation (not needed for $stage_name)"
fi
-
+
+ # Determine whether the marketplace-optional plugins go into this specific
image
+ local optional_plugins_value="false"
+ if [[ "$OPTIONAL_PLUGINS" == "true" ]]; then
+ optional_plugins_value="true"
+ elif [[ "$OPTIONAL_PLUGINS" == "auto" ]]; then
+ # Auto-detect: only images that need the Beam engine get them
+ if stage_needs_optional_plugins "$stage_name"; then
+ optional_plugins_value="true"
+ fi
+ fi
+
+ # The full builder already has the whole reactor; the fast builder needs
the flavor that
+ # also picks up the locally built plugin zips (see builder-fast-plugins in
the Dockerfile)
+ local effective_builder="$BUILDER_TYPE"
+ if [[ "$optional_plugins_value" == "true" ]]; then
+ print_info "Installing marketplace-optional (Wave 1) plugins into
$stage_name"
+ if [[ "$BUILDER_TYPE" == "fast" ]]; then
+ effective_builder="fast-plugins"
+ if ! ls "$PROJECT_ROOT"/plugins/*/*/target/*.zip &> /dev/null; then
+ print_error "No plugin zips found under plugins/*/*/target/"
+ print_error "Run a local build first, or drop
--with-optional-plugins"
+ exit 1
+ fi
+ fi
+ fi
+
# Build arguments
local build_args=(
"--build-arg" "HOP_BUILD_FROM_SOURCE=$SOURCE_TYPE"
"--build-arg" "TARGET_IMAGE=$stage_name"
"--build-arg" "MAVEN_THREADS=$maven_threads"
- "--build-arg" "BUILDER_TYPE=$BUILDER_TYPE"
+ "--build-arg" "BUILDER_TYPE=$effective_builder"
"--build-arg" "SKIP_FAT_JAR=$skip_fat_jar_value"
+ "--build-arg" "INCLUDE_OPTIONAL_PLUGINS=$optional_plugins_value"
"--progress" "$BUILD_PROGRESS"
"--file" "$SCRIPT_DIR/unified.Dockerfile"
"--target" "$stage_name"
@@ -528,7 +586,16 @@ build_images() {
else
echo "Fat jar: enabled"
fi
-
+
+ # Show marketplace-optional plugin status
+ if [[ "$OPTIONAL_PLUGINS" == "auto" ]]; then
+ echo "Wave1 plugins: auto (only for dataflow/web-beam, which need the
Beam engine)"
+ elif [[ "$OPTIONAL_PLUGINS" == "true" ]]; then
+ echo "Wave1 plugins: enabled (all marketplace-optional plugins)"
+ else
+ echo "Wave1 plugins: disabled"
+ fi
+
echo "Push: $PUSH"
echo "Cache: $USE_CACHE"
echo ""
diff --git
a/docker/integration-tests/integration-tests-spark-native-cluster.yaml
b/docker/integration-tests/integration-tests-spark-native-cluster.yaml
index bb9750bfd8..d94d7eb4e8 100644
--- a/docker/integration-tests/integration-tests-spark-native-cluster.yaml
+++ b/docker/integration-tests/integration-tests-spark-native-cluster.yaml
@@ -57,7 +57,7 @@ services:
context: ./spark
dockerfile: Dockerfile.master
args:
- SPARK_VERSION: ${SPARK_VERSION:-4.1.2}
+ SPARK_VERSION: ${SPARK_VERSION:-4.1.3}
HADOOP_VERSION: ${HADOOP_VERSION:-3}
BASE_URL: ${SPARK_BASE_URL:-https://dlcdn.apache.org/spark}
hostname: spark
@@ -89,7 +89,7 @@ services:
context: ./spark
dockerfile: Dockerfile.worker
args:
- SPARK_VERSION: ${SPARK_VERSION:-4.1.2}
+ SPARK_VERSION: ${SPARK_VERSION:-4.1.3}
HADOOP_VERSION: ${HADOOP_VERSION:-3}
BASE_URL: ${SPARK_BASE_URL:-https://dlcdn.apache.org/spark}
depends_on:
diff --git a/docker/unified.Dockerfile b/docker/unified.Dockerfile
index bb63c4a410..037a3915e5 100644
--- a/docker/unified.Dockerfile
+++ b/docker/unified.Dockerfile
@@ -116,6 +116,23 @@ COPY ./docker/resources/ /build/docker/resources/
#
# These will be extracted and prepared in Stage 3
+################################################################################
+# Stage 2b': Fast Builder + marketplace-optional plugins
+################################################################################
+# Same as builder-fast, plus everything Stage 3 needs to install the optional
("Wave 1")
+# plugins from the local reactor output: the helper scripts, the registry they
read, and the
+# plugin zips themselves. Docker COPY flattens globs, so the zips land in a
single directory
+# and Stage 3 resolves them by file name through HOP_PLUGIN_ZIP_DIR.
+#
+# This is a separate flavor on purpose: the zip glob fails the build when
nothing matches, and
+# a plain "fast" build must not start requiring plugins/**/target/*.zip to be
present.
+FROM builder-fast AS builder-fast-plugins
+
+COPY ./tools/ /build/tools/
+COPY
./plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
\
+
/build/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
+COPY ./plugins/*/*/target/*.zip /build/optional-plugin-zips/
+
################################################################################
# Stage 2c: Builder Selector
################################################################################
@@ -172,16 +189,49 @@ RUN echo "=== Extracting assemblies ===" && \
echo "WARNING: Plugins assembly not found, will use built plugins
directly"; \
fi
+# Step 1b: Install the marketplace-optional ("Wave 1") plugins into the
extracted client
+#
+# These plugins are released to Maven but deliberately kept out of
hop-client.zip, so they are
+# absent from the images unless requested. The list is read from
optional-plugins.yaml, the same
+# source of truth Jenkins and the integration tests use
(tools/install-wave1-plugins.sh).
+#
+# Note: the Beam engine is one of these plugins and it owns the
--generate-fat-jar option, so the
+# dataflow and web-beam targets require this to be enabled
(build-hop-images.sh does that for you).
+#
+# Works with both builders: "full" resolves the zips from the reactor it just
built, "fast" from
+# the flat directory it copied out of the local plugins/**/target/ folders
(HOP_PLUGIN_ZIP_DIR).
+ARG INCLUDE_OPTIONAL_PLUGINS=false
+RUN if [ "${INCLUDE_OPTIONAL_PLUGINS}" = "true" ]; then \
+ echo "=== Installing marketplace-optional plugins ===" && \
+ if [ ! -f /build/tools/install-wave1-plugins.sh ]; then \
+ echo "ERROR: tools/install-wave1-plugins.sh is not present in this
builder." && \
+ exit 1; \
+ fi && \
+ HOP_VERSION="$(ls /build/assemblies/client/target/hop-client-*.zip |
head -1 | sed -e 's#.*/hop-client-##' -e 's#\.zip$##')" && \
+ echo "Hop version: ${HOP_VERSION}" && \
+ export HOP_VERSION && \
+ export HOP_PLUGIN_ZIP_DIR=/build/optional-plugin-zips && \
+ bash /build/tools/install-wave1-plugins.sh
/build/assemblies/client/target/hop; \
+ else \
+ echo "=== Skipping marketplace-optional plugins
(INCLUDE_OPTIONAL_PLUGINS=false) ==="; \
+ fi
+
# Step 2: Generate fat jar for dataflow template (only if needed)
ARG SKIP_FAT_JAR=false
RUN if [ "${SKIP_FAT_JAR}" = "false" ]; then \
echo "=== Generating fat jar ===" && \
- if [ -f /build/assemblies/client/target/hop/hop-conf.sh ]; then \
- /build/assemblies/client/target/hop/hop-conf.sh \
- --generate-fat-jar=/tmp/hop-fatjar.jar; \
- else \
+ if [ ! -f /build/assemblies/client/target/hop/hop-conf.sh ]; then \
echo "ERROR: hop-conf.sh not found" && exit 1; \
- fi; \
+ fi && \
+ # --generate-fat-jar is contributed by the Beam engine plugin, which
is marketplace-optional.
+ # Without it hop-conf silently produces nothing usable, so fail loudly
instead.
+ if [ ! -d /build/assemblies/client/target/hop/plugins/engines/beam ];
then \
+ echo "ERROR: the Beam engine plugin is missing, so the fat jar
cannot be generated." && \
+ echo " Rebuild with --with-optional-plugins
(INCLUDE_OPTIONAL_PLUGINS=true)." && \
+ exit 1; \
+ fi && \
+ /build/assemblies/client/target/hop/hop-conf.sh \
+ --generate-fat-jar=/tmp/hop-fatjar.jar; \
else \
echo "=== Skipping fat jar generation (not needed) ===" && \
touch /tmp/hop-fatjar.jar; \
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
index 99a705fb43..6dc166acec 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/cluster-mapping-walkthrough.adoc
@@ -129,7 +129,7 @@ Compose mounts:
If you used a non-default `DIST_DIR` in `prepare-dist.sh`, set both
`HOP_DIST_DIR` and `HOP_DATA_HOST_DIR=${HOP_DIST_DIR}/hop-data` for compose.
* Master UI: http://localhost:8080 (two workers should register).
-* Images use the existing `docker/integration-tests/spark` Dockerfiles with
**`SPARK_VERSION=4.1.2`** (override with env if needed).
+* Images use the existing `docker/integration-tests/spark` Dockerfiles with
**`SPARK_VERSION=4.1.3`** (override with env if needed).
* After a run you can inspect **on the host** without `docker exec`, for
example:
[source,bash]
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
index cdea5d0ed5..f0fa23c38f 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc
@@ -67,7 +67,7 @@ The *native* engine skips Beam entirely:
|Beam pipelines on Spark (batch *and* streaming)
|Spark line
-|https://spark.apache.org/docs/4.1.2/[4.1.x] (Scala 2.13)
+|https://spark.apache.org/docs/4.1.3/[4.1.x] (Scala 2.13)
|3.5.x (Scala 2.12; see Beam support matrix)
|Programming model
@@ -115,7 +115,7 @@ You still design pipelines *visually* in Hop. Spark’s role
is the distributed
====
*Java:* Hop requires *Java 21* for the GUI, `hop-run`, and the fat jar
executed on the Spark driver/executors. Your Spark cluster must also run a JVM
that can load that bytecode (Java 21).
-*Spark:* The native engine is built and tested against *Spark 4.1.x*
(currently *4.1.2* in the Hop source tree, Scala *2.13*). For `local[*]` runs,
Spark libraries are packaged under `plugins/engines/spark/lib`, so you do *not*
need a separate Spark install on your laptop.
+*Spark:* The native engine is built and tested against *Spark 4.1.x*
(currently *4.1.3* in the Hop source tree, Scala *2.13*). For `local[*]` runs,
Spark libraries are packaged under `plugins/engines/spark/lib`, so you do *not*
need a separate Spark install on your laptop.
====
[options="header",cols="1,1,2,2"]
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
index 66bd1a883c..7421e8a490 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/lakehouse.adoc
@@ -55,7 +55,7 @@ It assumes you already have a working *Native Spark* pipeline
run configuration
| Coordinate / version
| Spark (Hop native engine)
-| `spark-sql_2.13` *4.1.2* (Scala *2.13*)
+| `spark-sql_2.13` *4.1.3* (Scala *2.13*)
| Delta Lake
| `io.delta:delta-spark_4.1_2.13:*4.3.1*`
diff --git a/lib/pom.xml b/lib/pom.xml
index 32acd80150..7332b7eb4c 100644
--- a/lib/pom.xml
+++ b/lib/pom.xml
@@ -93,7 +93,7 @@
<kaml.version>0.104.0</kaml.version>
<kotlin.version>1.9.25</kotlin.version>
<log4j.version>2.25.4</log4j.version>
- <lz4-java.version>1.11.0</lz4-java.version>
+ <lz4-java.version>1.11.1</lz4-java.version>
<maven-resolver-provider.version>3.9.9</maven-resolver-provider.version>
<maven-resolver.version>1.9.22</maven-resolver.version>
<msal4j-persistence.version>1.3.0</msal4j-persistence.version>
@@ -341,7 +341,7 @@
<version>${juniversalchardet.version}</version>
</dependency>
<dependency>
- <groupId>org.lz4</groupId>
+ <groupId>at.yawk.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>${lz4-java.version}</version>
</dependency>
@@ -452,80 +452,10 @@
</dependency>
<dependency>
<groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-buffer</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-base</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-http</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-http2</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-resolver</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-classes-epoll</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-common</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-handler</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-handler-proxy</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-epoll</artifactId>
- <version>${netty.version}</version>
- <classifier>linux-x86_64</classifier>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-kqueue</artifactId>
- <version>${netty.version}</version>
- <classifier>osx-x86_64</classifier>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-unix-common</artifactId>
+ <artifactId>netty-bom</artifactId>
<version>${netty.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
diff --git a/plugins/databases/clickhouse/pom.xml
b/plugins/databases/clickhouse/pom.xml
index 4b62f71b49..f8abd5009b 100644
--- a/plugins/databases/clickhouse/pom.xml
+++ b/plugins/databases/clickhouse/pom.xml
@@ -29,7 +29,7 @@
<name>Hop Plugins Databases ClickHouse</name>
<properties>
- <hop.jdbc.driver1>com.clickhouse:clickhouse-jdbc</hop.jdbc.driver1>
+
<hop.libcore.exclude1>com.clickhouse:clickhouse-jdbc</hop.libcore.exclude1>
</properties>
<dependencyManagement>
@@ -48,7 +48,6 @@
<dependency>
<groupId>com.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
- <!-- below is only needed when all you want is a shaded jar -->
<classifier>all</classifier>
<scope>provided</scope>
</dependency>
diff --git a/plugins/engines/spark/README.md b/plugins/engines/spark/README.md
deleted file mode 100644
index 25b455d5c7..0000000000
--- a/plugins/engines/spark/README.md
+++ /dev/null
@@ -1,455 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Native Apache Spark pipeline engine
-
-Implements [issue #7486](https://github.com/apache/hop/issues/7486): a Hop
-`IPipelineEngine` that runs batch pipelines on **Apache Spark 4.1.x** without
-Apache Beam.
-
-## Design notes (Phase 0 decisions)
-
-| Topic | Choice |
-|-------|--------|
-| Spark version | `4.1.2` (`spark-sql_2.13`), independent of Beam’s Spark 3.x |
-| Row model | Spark `Row` + `StructType` (`HopSparkRowConverter`) |
-| Hop transforms | `mapPartitions` via `HopMapPartitionsFn` (init once per
partition) |
-| Metadata on the wire | Transform XML + `JsonRowMeta` +
`SerializableMetadataProvider` JSON |
-| Deployment v1 | `local[*]` and standalone `spark://`; fat-jar field for
cluster |
-| Streaming / windows | Out of scope — use Beam Spark |
-| Stateful transforms | Native handlers: Memory Group By, Merge Join, Unique
Rows, Sort Rows; sorted Group By remains banned |
-
-## Classpath
-
-- **Compile / local run:** Spark 4.1 is packaged under
`plugins/engines/spark/lib`
- so Hop can start `local[*]` without a separate Spark install. The plugin must
- also ship **hadoop-client-api/runtime** and **log4j**: Spark 4 loads
- `org.apache.spark.sql.classic.SparkSession` reflectively and fails with
- *“Cannot find a SparkSession implementation on the Classpath”* when those
- classes are missing (the real `ClassNotFoundException` is swallowed).
-- **Fat jar tokens (not real pack directories under `lib/spark-clients/`):**
-
- | Token | Use for | Spark runtime |
- |-------|---------|----------------|
- | `native` | hop-run client / embed Spark | Ships Spark 4 from
`plugins/engines/spark/lib` |
- | **`native-provided`** | **`spark-submit` on a cluster** | **Cluster
provides Spark** (do not embed) |
-
- **Do not** embed Spark inside the app jar for `spark-submit`. That mixes two
- Spark copies on the driver classpath and fails on Java 21 (e.g.
- `DirectByteBuffer` / wrong `Running Spark version`).
-
- ```bash
- # From a Hop install — fat jar for spark-submit:
- ./hop-conf.sh --generate-fat-jar=/tmp/hop-native-spark4-submit.jar \
- --spark-client-version=native-provided
- ```
-
-- **spark-submit (cluster):** Use driver main class
- `org.apache.hop.spark.run.MainSpark` with a **native-provided** fat jar:
-
- ```bash
- # Critical: pin SPARK_HOME to this distro. If SPARK_HOME points at an older
- # install (e.g. /opt/spark -> 3.3.0), even "./bin/spark-submit" from a 4.1.x
- # tree re-executes ${SPARK_HOME}/bin/spark-class and loads the wrong Spark.
- export SPARK_HOME=~/Downloads/spark-4.1.3-bin-hadoop3
-
- # Export project metadata to JSON, then:
- "${SPARK_HOME}/bin/spark-submit" \
- --master spark://spark:7077 \
- --class org.apache.hop.spark.run.MainSpark \
- /tmp/hop-native-spark4-submit.jar \
- /path/to/pipeline.hpl \
- /path/to/metadata.json \
- YourNativeSparkRunConfig
- ```
-
- Positional args: `pipeline.hpl`, `metadata.json`, `runConfigName`, optional
- env config file. Named form also works: `--HopPipelinePath=...`,
- `--HopMetadataPath=...`, `--HopRunConfigurationName=...`,
- `--HopConfigFile=...`.
-
- The run configuration must use the **native** Spark pipeline engine (not
- Beam). Prefer leaving **master blank** so `spark-submit --master` wins, or
set
- master to `spark://spark:7077`. Leave **fatJar empty** under submit.
- Paths must be readable on the driver; data paths must be reachable from
workers.
-- After rebuilding the plugin, reinstall into your Hop distro (or rebuild
- `assemblies/client`) so `plugins/engines/spark/lib` is updated.
-
-## Execution information location
-
-Attach an **execution information location** (and optionally an execution data
-profile) on the native Spark **pipeline run configuration**, same as
Local/Beam.
-
-The driver will:
-
-1. Load and initialize the location at prepare time
-2. `registerExecution` when the pipeline starts
-3. Periodically `updateExecutionState` for the pipeline and each transform
- component (metrics come from Spark accumulators / `EngineMetrics`)
-4. Write a final state and `close` the location when the pipeline finishes,
- fails, or is stopped
-
-Notes:
-
-- **State updates** (`updateExecutionState`) run on the **driver** from
- `EngineMetrics` (all transform types).
-- **Row sampling** (`registerData`) runs on **executors** for generic
- mapPartitions transforms when the run configuration also names an
- **execution data profile** (First/Last/Random rows, etc.). Extra GUI
- samplers on the engine are serialized into the closure like Beam.
-- File locations used for sampling must be **writable from every executor**
- (shared FS / object store). Local-only paths only receive driver state.
-- For `spark-submit`, export location + data profile into the metadata JSON.
-
-## Module layout
-
-- `engines/` — `SparkPipelineEngine`, run configuration, capabilities
-- `pipeline/` — Hop → Spark converter and transform handlers
-- `pipeline/handler/` — native shuffle handlers + generic mapPartitions
-- `core/` — `HopMapPartitionsFn`, row conversion, executor-side Hop init
-- `run/` — `MainSpark` for spark-submit
-
-## Native handlers (Phase 3)
-
-| Plugin id | Spark API | Notes |
-|-----------|-----------|--------|
-| `MemoryGroupBy` | `groupBy` + agg | SUM, AVG, MIN, MAX, COUNTs, FIRST/LAST,
STDDEV, percentile, concat variants |
-| `MergeJoin` | `Dataset.join` | INNER / LEFT / RIGHT / FULL OUTER |
-| `Unique` | `dropDuplicates` (+ count via groupBy) | Reject-to-error not
supported |
-| `SortRows` | `orderBy` | Optional unique-on-keys |
-| `GroupBy` (sorted) | — | **Banned** — use Memory Group By |
-| `SparkFileInput` | `spark.read.format(...).load` | CSV/Parquet/JSON/ORC/text
|
-| `SparkFileOutput` | `df.write.format(...).save` | Save modes, optional
coalesce/partitionBy |
-
-## Classic Hop I/O (POC)
-
-`HopMapPartitionsFn` sets partition-scoped variables for classic writers:
-
-| Variable | Value |
-|----------|--------|
-| `${Internal.Transform.ID}` | `{transformName}-{partitionId}` |
-| `${Internal.Transform.CopyNr}` / `BundleNr` | partition id |
-
-It also enables `beamContext` so Text File Output auto-appends `_id_bundle`
like Beam.
-`dependencies.xml` stages `textfile`, `getfilenames`, and `excel` for
mapPartitions POCs
-(Get File Names → Text File Input, small Excel lookups, etc.). Pure sources
still run on
-one partition so files are not listed/read N times.
-
-## File I/O transforms
-
-Dedicated **Spark File Input** / **Spark File Output** transforms (not Hop
Text File Input)
-provide a clean Dataset API mapping:
-
-- **Input:** path, format, header/separator/quote (CSV), optional field schema
or `inferSchema`
-- **Output:** path, format, save mode (Overwrite/Append/Ignore/ErrorIfExists),
header, coalesce,
- partition-by columns
-
-**Paths are Spark/Hadoop URIs** (e.g. local path, `hdfs://…`, `s3a://…`), not
Hop VFS schemes
-(`s3://`, named MinIO `minio://`, …). Dataset `load`/`save` never call
`HopVfs`. Classic
-mapPartitions transforms still use Hop VFS. See user manual: *Paths and file
systems on native
-Spark*.
-
-Optional **path scheme map** on the run configuration (`from=to` lines, e.g.
`s3=s3a`) rewrites
-URI schemes for Spark File / Lake PATH only via `SparkPathDialect.toSparkUri`.
`SparkPathDialect`
-also appends a hint when a known Hop-only scheme still appears in I/O errors
after mapping.
-
-## Project package (nested Simple Mapping / Pipeline Executor)
-
-**Specifically for Native Spark execution** (not File → Export current project
to zip).
-
-```bash
-# GUI: Tools → Export project package for Native Spark…
-# CLI:
-./hop-conf.sh -j my-project --export-spark-project=/tmp/my-project-spark.zip
-
-# spark-submit
---class org.apache.hop.spark.run.MainSpark ... \
- --HopProjectPackage=/tmp/my-project-spark.zip \
- --HopPipelinePath=pipelines/run.hpl \
- --HopRunConfigurationName=YourNativeSparkRunConfig
-```
-
-`SparkProjectPackage` + `SparkPipelineEngine`:
-
-1. Driver stages the zip and sets `HOP_SPARK_PROJECT_PACKAGE`
-2. Session start: `SparkContext.addFile` ships the zip to **all executors**
(`SparkFiles`)
-3. Mini-pipeline init: `SparkFiles.get` → extract under `java.io.tmpdir` →
`PROJECT_HOME`
-
-Do **not** use package `PROJECT_HOME` as the root for Spark Dataset data paths.
-
-## Cluster walk-through (Docker)
-
-Sample project: `src/samples/spark-demo`
-Compose:
`docker/integration-tests/integration-tests-spark-native-cluster.yaml` (Spark
**4.1.2**)
-
-```bash
-./plugins/engines/spark/src/samples/spark-demo/scripts/prepare-dist.sh
-# artifacts default to /tmp/spark-demo-dist (override DIST_DIR / HOP_DIST_DIR)
-# shared data plane: /tmp/spark-demo-dist/hop-data → /data/hop-data
(host-visible)
-docker compose -f
docker/integration-tests/integration-tests-spark-native-cluster.yaml \
- up -d --build --scale spark-worker=2
-docker compose -f
docker/integration-tests/integration-tests-spark-native-cluster.yaml \
- exec spark /opt/hop-samples/spark-demo/scripts/spark-submit-demo.sh
-# Workflow Executor demo:
-docker compose -f
docker/integration-tests/integration-tests-spark-native-cluster.yaml \
- exec spark /opt/hop-samples/spark-demo/scripts/spark-submit-workflows-demo.sh
-# Host inspect: ls /tmp/spark-demo-dist/hop-data/out
/tmp/spark-demo-dist/hop-data/executions
-```
-
-See user manual: *Cluster demo walk-through*.
-
-Writes run as Spark actions during graph build; the output handler registers
an empty leaf
-Dataset so a later engine `count()` does not re-write files.
-
-## Lakehouse connectors (Delta Lake / Apache Iceberg)
-
-**Open table format** support on the native Spark engine (ACID tables,
catalogs, time travel,
-MERGE, maintenance). Connectors **ship in the default** `hop-engines-spark`
plugin assembly:
-
-| Path | Contents |
-|------|----------|
-| `plugins/engines/spark/lib/delta/` | Delta Lake 4.3.1 tree |
-| `plugins/engines/spark/lib/iceberg/` | Iceberg Spark runtime 1.11.0 |
-
-**User documentation (Antora):**
-`docs/hop-user-manual/.../pipeline/spark/lakehouse.adoc` plus transform pages
-`spark-lake-table-*.adoc` and `metadata-types/spark-catalog.adoc`.
-**Integration tests:** `integration-tests/lakehouse` (Delta) and
-`integration-tests/iceberg` (Iceberg) — PATH Input/Output, overwrite/time
travel, MERGE.
-
-| Pin | Coordinate |
-|-----|------------|
-| Spark | `spark-sql_2.13:4.1.2` (existing) |
-| Delta Lake | `io.delta:delta-spark_4.1_2.13:4.3.1` |
-| Apache Iceberg | `org.apache.iceberg:iceberg-spark-runtime-4.1_2.13:1.11.0` |
-
-The `_4.1_2.13` suffix means **Spark 4.1 / Scala 2.13**, not the product major
version.
-
-### Classloader (why not a sibling plugin)
-
-`SparkPipelineEngine` sets TCCL to the **Spark engine plugin classloader**,
which loads:
-
-1. JARs under `plugins/engines/spark/lib/` (recursive — includes `lib/delta`,
`lib/iceberg`)
-2. Folders from `plugins/engines/spark/dependencies.xml`
-3. The engine plugin jar
-
-A sibling plugin under `plugins/engines/spark-delta` is a **separate**
classloader and is
-**not** visible to `SparkSession` / DataSource resolution. Connectors must
live **inside**
-the engine lib tree (as shipped) or on the spark-submit driver/executor
classpath.
-
-### hop-run / GUI `local[*]`
-
-No manual overlay is required on a normal Hop install. After rebuilding the
plugin:
-
-```bash
-./mvnw -pl plugins/engines/spark -am package
-# zip / client assembly unpacks lib/delta and lib/iceberg under
plugins/engines/spark/
-```
-
-`target/lakehouse-connectors/` is a frozen jar list for docs verification
(generated every build).
-
-### spark-submit
-
-Use a **native-provided** fat jar so Spark is not embedded twice. Lakehouse
jars under
-`plugins/engines/spark/lib/{delta,iceberg}` are **included** in that fat jar
(they are not
-filtered out like Spark/Scala/Hadoop client). Optional `--packages` only if
you strip them:
-
-```bash
-export SPARK_HOME=/path/to/spark-4.1.x-bin-hadoop3
-
-"${SPARK_HOME}/bin/spark-submit" \
- --master spark://spark:7077 \
- --conf
spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
\
- --conf
spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog
\
- --class org.apache.hop.spark.run.MainSpark \
- /tmp/hop-native-provided.jar \
- /path/to/pipeline.hpl \
- /path/to/metadata.json \
- YourNativeSparkRunConfig
-```
-
-When both Delta and Iceberg are used, keep **Delta** on `spark_catalog` and
register Iceberg
-catalogs under **other** names (e.g. `lake`).
-
-### Session conf matrix (spike / future transforms)
-
-| When | Conf |
-|------|------|
-| Any Delta use | `spark.sql.extensions` includes
`io.delta.sql.DeltaSparkSessionExtension` |
-| Any Delta use |
`spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog`
|
-| Any Iceberg use | `spark.sql.extensions` includes
`org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions` |
-| Iceberg TABLE | `spark.sql.catalog.<name>=…` (+ type/warehouse/uri) |
-
-### Spike findings (PR 1 — 2026-07, Spark 4.1.2 + Delta 4.3.1 + Iceberg 1.11.0)
-
-Recorded by `SparkLakehouseConnectorTest` with `-Plakehouse`:
-
-| Check | Result |
-|-------|--------|
-| `Class.forName(DeltaSparkSessionExtension)` with connector on CL | OK |
-| PATH write/read with extension **and** DeltaCatalog | OK (required gate) |
-| PATH write/read with extension **only** (no DeltaCatalog) | **Fails** with
`DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG` |
-| Iceberg extension on CL | OK |
-| Iceberg bare `format("iceberg").save/load(path)` (no catalog conf) |
**Fails** — defaults toward `HiveCatalog` / needs HMS jars |
-| Iceberg `writeTo(hop_iceberg.\`file:///path\`)` + SQL read (PR 3) | **OK** |
-| Iceberg Hadoop warehouse + `writeTo("local.db.t")` + `spark.table` | OK
(named-table style) |
-| Iceberg TABLE `writeTo` vs `saveAsTable` | **writeTo for PATH**; formal
TABLE lock still PR 5 / KD-24 |
-
-**Tiered probe default (updated by spike + PR 3):**
-
-- Always require connector classes on the classpath.
-- **Any Delta use (including PATH-only)** requires session-registered
- `DeltaSparkSessionExtension` **and**
`spark.sql.catalog.spark_catalog=…DeltaCatalog`.
-- **Any Iceberg PATH use** requires Iceberg extensions + Hadoop catalog
`hop_iceberg`
- (auto-registered on hop-run; can be set at runtime on active sessions).
-- hop-run always applies full Delta/Iceberg conf when those formats are
present.
-
-### Frozen jar list (shipped under `lib/delta` + `lib/iceberg`)
-
-Also staged into `target/lakehouse-connectors/` on every build for docs
verification.
-
-| Jar | Role |
-|-----|------|
-| `delta-spark_4.1_2.13-4.3.1.jar` | Delta Spark SQL / DataSource |
-| `delta-storage-4.3.1.jar` | Delta storage layer |
-| `delta-kernel-api-4.3.1.jar` | Delta Kernel API |
-| `delta-kernel-defaults-4.3.1.jar` | Delta Kernel defaults |
-| `delta-kernel-unitycatalog-4.3.1.jar` | Transitive from delta-spark 4.3.1 |
-| `unitycatalog-client-0.5.0.jar` | Transitive (UC-related) |
-| `unitycatalog-hadoop-0.5.0.jar` | Transitive (UC-related) |
-| `iceberg-spark-runtime-4.1_2.13-1.11.0.jar` | Shaded Iceberg Spark runtime |
-
-### Probe API
-
-`org.apache.hop.spark.table.SparkLakeConnectorProbe` verifies connector
classes are loadable
-and throws `HopException` with reinstall / `--packages` guidance when missing.
-`LakeSessionPlan` scans active lake transforms before `SparkSession` build:
probes the
-classpath and applies Delta extension + `DeltaCatalog` on new hop-run sessions.
-
-### Lake Table transforms (PR 2–3 — Delta + Iceberg PATH)
-
-| Plugin id | Role |
-|-----------|------|
-| `SparkLakeTableInput` | Read Delta or Iceberg PATH tables |
-| `SparkLakeTableOutput` | Write Delta or Iceberg PATH; **default save mode
ErrorIfExists** |
-
-| Format | PATH read | PATH write |
-|--------|-----------|------------|
-| **Delta** | `spark.read.format("delta").load(path)` |
`format("delta").mode(…).save(path)` |
-| **Iceberg** | SQL `SELECT * FROM hop_iceberg.\`file:///…\`` |
`writeTo(hop_iceberg.\`uri\`).using("iceberg")` |
-
-Iceberg PATH uses a built-in Hadoop catalog named **`hop_iceberg`** (not
`spark_catalog`, so it
-can co-exist with Delta’s `DeltaCatalog`). Bare `format("iceberg").load(path)`
is **not** used —
-on Spark 4.1 + Iceberg 1.11 it defaults toward HiveCatalog and fails without
HMS jars.
-
-hop-run auto-registers:
-
-| When | Session conf |
-|------|----------------|
-| Any Delta | extensions + `spark.sql.catalog.spark_catalog=…DeltaCatalog` |
-| Any Iceberg | extensions + `spark.sql.catalog.hop_iceberg=…SparkCatalog`
(type=hadoop, warehouse under tmp) |
-
-Classic **Spark File Input/Output** are unchanged (csv/parquet/json/orc/text
only).
-
-### Time travel (PR 4)
-
-Lake Table Input supports:
-
-| UI type | Delta option | Iceberg SQL |
-|---------|--------------|-------------|
-| `NONE` | (current) | (current) |
-| `VERSION` | `versionAsOf` | `VERSION AS OF <snapshot_id>` |
-| `TIMESTAMP` | `timestampAsOf` | `TIMESTAMP AS OF TIMESTAMP '…'` |
-
-Example: write twice with Overwrite, then set Time travel = VERSION and
Version = `0` (Delta)
-or the first snapshot id from `{table}.snapshots` (Iceberg).
-
-### Spark Catalog metadata + TABLE mode (PR 5)
-
-**Metadata type:** `SparkCatalog` (category Connections) — Spark SQL catalog
conf for Iceberg
-Hadoop/REST (or custom). Fields: catalog name, type, warehouse/uri, optional
credential, extra
-`key=value` lines → `spark.sql.catalog.<name>.*`.
-
-Lake Table transforms in **TABLE** mode:
-
-| Field | Purpose |
-|-------|---------|
-| Identifier mode | `TABLE` |
-| Table identifier | e.g. `lake.db.orders` |
-| Spark Catalog metadata name | Hop metadata entry to register (optional if
catalog already on session) |
-
-| Format | TABLE read | TABLE write |
-|--------|------------|-------------|
-| **Iceberg** (primary) | `SELECT * FROM catalog.ns.table` (+ time travel
clauses) | `writeTo(id).using("iceberg")` |
-| **Delta** (advanced) | `format("delta").table(id)` |
`format("delta").saveAsTable(id)` |
-
-`LakeSessionPlan` loads referenced `SparkCatalog` entries and applies them
when building the
-session (hop-run) or registers them on an active session when possible
(submit).
-
-For spark-submit, export project metadata JSON including `SparkCatalog`
definitions, or pass
-equivalent `--conf spark.sql.catalog.<name>=…`.
-
-### MERGE (PR 6)
-
-**Spark Lake Table Merge** — single upstream hop supplies source rows; target
is PATH or TABLE.
-
-| Field | Notes |
-|-------|--------|
-| Merge condition | ON clause, aliases `t` (target) and `s` (source) |
-| When matched | `UPDATE_ALL` / `DELETE` / `NONE` |
-| When not matched | `INSERT_ALL` / `NONE` |
-| Not matched by source | optional `DELETE` (Delta; Iceberg depends on
version) |
-| Raw MERGE SQL | advanced override (empty default) |
-
-Target SQL ids: Delta PATH → `delta.\`path\``; Iceberg PATH →
`hop_iceberg.\`uri\``; TABLE → multi-part id.
-
-Metrics are log-only (merge completion), not Dataset row counts.
-
-### Maintenance (PR 7)
-
-**Spark Lake Table Maintenance** — zero-input action (no hop required).
Destructive ops require
-`acknowledgeDestructive`.
-
-| Operation | Delta | Iceberg |
-|-----------|-------|---------|
-| OPTIMIZE | `OPTIMIZE delta.\`path\` [ZORDER BY …]` | `CALL
cat.system.rewrite_data_files` |
-| VACUUM | `VACUUM … RETAIN n HOURS` (**n required**) | n/a → use
EXPIRE_SNAPSHOTS |
-| EXPIRE_SNAPSHOTS | n/a | `CALL … expire_snapshots(retain_last => N)` |
-| REWRITE_MANIFESTS | n/a | `CALL … rewrite_manifests` |
-| DELETE_WHERE | `DELETE FROM … WHERE …` (**WHERE required**) | same |
-
-Metrics are log-only (expect 0 in/out in the GUI).
-
-
-### RAT / license
-
-Delta Lake and Apache Iceberg are Apache-2.0 licensed and redistributed under
-`plugins/engines/spark/lib/{delta,iceberg}/`. Re-check transitive licenses
when pins change.
-Unity Catalog client jars ship as Delta transitives (Apache-2.0).
-
-## Build
-
-```bash
-# Unit + lakehouse ITs (connectors are default runtime deps)
-./mvnw -pl plugins/engines/spark -am test
-
-# Package plugin zip (includes lib/delta + lib/iceberg)
-./mvnw -pl plugins/engines/spark -am package
-
-# -Plakehouse is a no-op alias retained for older scripts
-./mvnw -pl plugins/engines/spark -am test -Plakehouse
-```
diff --git a/plugins/engines/spark/pom.xml b/plugins/engines/spark/pom.xml
index 03fe995bf9..3985641885 100644
--- a/plugins/engines/spark/pom.xml
+++ b/plugins/engines/spark/pom.xml
@@ -31,26 +31,30 @@
<description>Native Apache Spark 4.x pipeline engine for Hop (issue
#7486)</description>
<properties>
- <!-- Lakehouse connectors shipped in default engines-spark assembly
under lib/{delta,iceberg}.
- Artifact suffix _4.1_2.13 = Spark 4.1 / Scala 2.13, not product
version. -->
<delta.version>4.3.1</delta.version>
- <!-- Matches Spark 4.1.2 parent; needed for local[*] unit tests only
-->
<hadoop.client.version>3.4.2</hadoop.client.version>
- <!-- Do not stage Jackson into plugins/engines/spark/lib (Hop parent
CL has it).
- Dual ObjectMapper classes under HopURLClassLoader break HopJson
on local[*]. -->
<hop.pluginlib.exclude1>jackson-core</hop.pluginlib.exclude1>
<hop.pluginlib.exclude2>jackson-databind</hop.pluginlib.exclude2>
<hop.pluginlib.exclude3>jackson-annotations</hop.pluginlib.exclude3>
<iceberg.version>1.11.0</iceberg.version>
- <!-- Staged jar freeze list for docs / verification (not a second
install path). -->
<lakehouse.connectors.dir>${project.basedir}/target/lakehouse-connectors</lakehouse.connectors.dir>
<scala.binary.version>2.13</scala.binary.version>
- <!-- Pin Spark 4.1.x independently of Beam's Spark 3.x dependency -->
- <spark.version>4.1.2</spark.version>
- <!-- Spark major.minor used in Delta/Iceberg Maven artifact names (4.1
from 4.1.2). -->
+ <spark.version>4.1.3</spark.version>
<spark.version.prefix>4.1</spark.version.prefix>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-libs</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<!-- Spark MetricsServlet / Jetty needs servlet API (SingleThreadModel
→ use 5.0) -->
<dependency>
@@ -69,15 +73,6 @@
<artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.client.version}</version>
</dependency>
-
- <!-- Spark 4.1 — packaged under plugins/engines/spark/lib for local[*].
- Hadoop client + Log4j MUST stay on the plugin classpath: Spark 4
looks up
- org.apache.spark.sql.classic.SparkSession reflectively and fails
with
- "Cannot find a SparkSession implementation on the Classpath" when
those
- transitive classes are missing (ClassNotFoundException is
swallowed).
- slf4j is provided by Hop; keep it excluded to avoid dual bindings.
- Jackson core is also provided by Hop lib/core — dual copies under
the plugin
- classloader cause LinkageError on HopJson.newMapper() (sampling).
-->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
diff --git a/plugins/misc/marketplace/README.md
b/plugins/misc/marketplace/README.md
index 4b09e2b425..90f016de89 100644
--- a/plugins/misc/marketplace/README.md
+++ b/plugins/misc/marketplace/README.md
@@ -235,6 +235,7 @@ Do not hardcode plugin lists elsewhere. Everything below
reads this registry:
| `full-client-env.yaml` | Generated from this registry during the marketplace
build (Maven `generate-resources`) |
| Local publish to Nexus/Artifactory |
`docker/marketplace-nexus/publish-marketplace-plugins.sh` |
| Unpack zips into an install (IT images) | `tools/install-wave1-plugins.sh`
via `tools/list-marketplace-plugins.sh` |
+| Docker images | `docker/build-hop-images.sh --with-optional-plugins` runs
`tools/install-wave1-plugins.sh` inside the builder stage |
Each entry needs at least:
diff --git a/pom.xml b/pom.xml
index c608a24967..6c34d85254 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,7 +154,7 @@
<maven.compiler.target>${target.jdk.version}</maven.compiler.target>
<minimalMavenBuildVersion>3.6.3</minimalMavenBuildVersion>
<mockito-core.version>5.22.0</mockito-core.version>
- <netty.version>4.2.15.Final</netty.version>
+ <netty.version>4.2.16.Final</netty.version>
<objenesis.version>3.5</objenesis.version>
<opentelemetry.version>1.62.0</opentelemetry.version>
<org.eclipse.platform.version>3.132.0</org.eclipse.platform.version>
diff --git a/tools/install-wave1-plugins.sh b/tools/install-wave1-plugins.sh
index 71994e756b..56869a781e 100755
--- a/tools/install-wave1-plugins.sh
+++ b/tools/install-wave1-plugins.sh
@@ -17,12 +17,17 @@
#
# Requires plugin modules to have been packaged (*.zip under
plugins/**/target/).
#
+# HOP_PLUGIN_ZIP_DIR: optional flat directory of plugin zips, searched by file
name when the
+# reactor layout is not available. Docker COPY flattens globs, so the "fast"
image builder
+# hands the zips over in one directory instead of
plugins/<category>/<name>/target/.
+#
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
INSTALL_DIR="${1:-${ROOT}/assemblies/client/target/hop}"
VERSION="${HOP_VERSION:-2.19.0-SNAPSHOT}"
LIST_SCRIPT="${ROOT}/tools/list-marketplace-plugins.sh"
+ZIP_DIR="${HOP_PLUGIN_ZIP_DIR:-}"
if [[ ! -d "${INSTALL_DIR}" ]]; then
echo "Hop install not found: ${INSTALL_DIR}" >&2
@@ -48,6 +53,9 @@ for entry in ${PLUGINS[@]+"${PLUGINS[@]}"}; do
id="${entry%%|*}"
rel="${entry#*|}"
zip="${ROOT}/${rel}"
+ if [[ ! -f "${zip}" && -n "${ZIP_DIR}" && -f "${ZIP_DIR}/${rel##*/}" ]]; then
+ zip="${ZIP_DIR}/${rel##*/}"
+ fi
if [[ ! -f "${zip}" ]]; then
echo "SKIP (not built): ${id} — ${rel}"
skipped=$((skipped + 1))
