This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new af615ad34b some extra cleanup, fixes #7721 (#7722)
af615ad34b is described below
commit af615ad34b463393811c0e6e026848ce5ddc3336
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Jul 31 17:39:19 2026 +0200
some extra cleanup, fixes #7721 (#7722)
---
assemblies/client/pom.xml | 16 +-
assemblies/client/src/assembly/assembly.xml | 5 +-
assemblies/plugins/pom.xml | 13 +-
assemblies/static/src/main/resources/hop-conf.bat | 7 -
assemblies/static/src/main/resources/hop-conf.sh | 10 --
.../static/src/main/resources/hop-encrypt.bat | 7 -
.../static/src/main/resources/hop-encrypt.sh | 10 --
.../static/src/main/resources/hop-gui-nolog.bat | 7 -
assemblies/static/src/main/resources/hop-gui.bat | 7 -
assemblies/static/src/main/resources/hop-gui.sh | 10 --
.../static/src/main/resources/hop-import.bat | 7 -
assemblies/static/src/main/resources/hop-import.sh | 10 --
assemblies/static/src/main/resources/hop-run.bat | 7 -
assemblies/static/src/main/resources/hop-run.sh | 10 --
.../static/src/main/resources/hop-search.bat | 7 -
assemblies/static/src/main/resources/hop-search.sh | 10 --
.../static/src/main/resources/hop-server.bat | 7 -
assemblies/static/src/main/resources/hop-server.sh | 10 --
assemblies/static/src/main/resources/hop.bat | 7 -
.../pages/pipeline/transforms/detectlanguage.adoc | 15 +-
.../pages/pipeline/transforms/stanfordnlp.adoc | 36 +++--
plugins/engines/beam/pom.xml | 34 +----
.../beam/src/assembly/assembly-beam-runtime.xml | 154 --------------------
.../engines/beam/src/assembly/assembly-libs.xml | 161 ---------------------
plugins/engines/beam/src/assembly/assembly.xml | 41 +++++-
.../org/apache/hop/beam/gui/HopBeamGuiPlugin.java | 11 +-
.../marketplace/catalog/OptionalPluginCatalog.java | 2 +-
.../apache/hop/marketplace/optional-plugins.yaml | 6 +
.../marketplace/install/PluginInstallerTest.java | 2 +-
plugins/transforms/detectlanguage/pom.xml | 5 -
plugins/transforms/stanfordnlp/pom.xml | 7 +
31 files changed, 116 insertions(+), 525 deletions(-)
diff --git a/assemblies/client/pom.xml b/assemblies/client/pom.xml
index b99efff000..abad987b67 100644
--- a/assemblies/client/pom.xml
+++ b/assemblies/client/pom.xml
@@ -61,17 +61,9 @@
<version>${project.version}</version>
<type>zip</type>
</dependency>
- <!-- Slim Beam-related libs only: lib/spark-client + shared lib/core
extras.
- Beam SDKs/runners ship with the marketplace plugin under
plugins/engines/beam/lib-beam.
- Use -Pbeam to bake the full Beam plugin (+ lib-beam) into
hop-client (IT / fat Beam builds). -->
- <dependency>
- <groupId>org.apache.hop</groupId>
- <artifactId>hop-engines-beam</artifactId>
- <version>${project.version}</version>
- <classifier>libs</classifier>
- <type>zip</type>
- </dependency>
- <!-- hop-engines-spark is a marketplace-optional plugin (not bundled
in the client). -->
+ <!-- hop-engines-beam and hop-engines-spark are marketplace-optional
plugins (not bundled
+ in the client). Use -Pbeam / -Pfull to bake the full Beam plugin
— including its
+ lib/core and lib/spark-client — into hop-client (IT / fat Beam
builds). -->
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-ui</artifactId>
@@ -144,7 +136,7 @@
</dependencies>
<!--
- Opt-in Beam engine + SDKs in hop-client (plugins/engines/beam including
lib-beam).
+ Opt-in Beam engine + SDKs in hop-client (plugin jar plus its libs in
lib/core).
Analogous to HOP_SPARK_CLIENT_VERSION=native for Spark: default lean
client has no Beam
runners on the classpath; use -Pbeam for IT / local full Beam installs.
diff --git a/assemblies/client/src/assembly/assembly.xml
b/assemblies/client/src/assembly/assembly.xml
index 58b15cad02..c5b6534a64 100644
--- a/assemblies/client/src/assembly/assembly.xml
+++ b/assemblies/client/src/assembly/assembly.xml
@@ -40,9 +40,8 @@
<include>org.apache.hop:hop-core:zip</include>
<include>org.apache.hop:hop-engine:zip</include>
<include>org.apache.hop:hop-ui:zip</include>
- <!-- Slim: spark-client + shared lib/core extras (no Beam
SDKs) -->
- <include>org.apache.hop:hop-engines-beam:zip:libs</include>
- <!-- Full Beam plugin + lib-beam only when dependency present
(-Pbeam) -->
+ <!-- Beam is a marketplace plugin; bundled only when the
dependency is
+ present (-Pbeam / -Pfull). It brings its own lib/core
libs and lib/spark-client. -->
<include>org.apache.hop:hop-engines-beam:zip</include>
<include>org.apache.hop:hop-ui-rcp:zip</include>
</includes>
diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml
index 5198265e75..ff455aabc6 100644
--- a/assemblies/plugins/pom.xml
+++ b/assemblies/plugins/pom.xml
@@ -908,12 +908,7 @@
<version>${project.version}</version>
<type>zip</type>
</dependency>
- <dependency>
- <groupId>org.apache.hop</groupId>
- <artifactId>hop-transform-detectlanguage</artifactId>
- <version>${project.version}</version>
- <type>zip</type>
- </dependency>
+ <!-- marketplace optional: hop-transform-detectlanguage -->
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-detectlastrow</artifactId>
@@ -1717,6 +1712,12 @@
<version>${project.version}</version>
<type>zip</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-transform-detectlanguage</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-drools</artifactId>
diff --git a/assemblies/static/src/main/resources/hop-conf.bat
b/assemblies/static/src/main/resources/hop-conf.bat
index 12726246c8..fea7732864 100644
--- a/assemblies/static/src/main/resources/hop-conf.bat
+++ b/assemblies/static/src/main/resources/hop-conf.bat
@@ -29,13 +29,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git a/assemblies/static/src/main/resources/hop-conf.sh
b/assemblies/static/src/main/resources/hop-conf.sh
index 67dd91794e..3c3257dd3a 100755
--- a/assemblies/static/src/main/resources/hop-conf.sh
+++ b/assemblies/static/src/main/resources/hop-conf.sh
@@ -104,16 +104,6 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
-# Beam SDKs/runners live with the optional Beam plugin (not lib/beam).
-# Only on the system classpath when marketplace (or -Pbeam) has installed them.
-if [ -d "plugins/engines/beam/lib-beam" ]; then
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
-fi
-if [ -d "plugins/engines/beam" ]; then
- # Hop Beam plugin jars at the plugin root (exclude subdirs via non-recursive
glob is fine for /*.jar intent;
- # shell * includes jars and may pick non-jars; match historical lib/* style)
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
-fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.config.HopConfig "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-encrypt.bat
b/assemblies/static/src/main/resources/hop-encrypt.bat
index 66b37dea1e..0c605ec0fe 100644
--- a/assemblies/static/src/main/resources/hop-encrypt.bat
+++ b/assemblies/static/src/main/resources/hop-encrypt.bat
@@ -30,13 +30,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git a/assemblies/static/src/main/resources/hop-encrypt.sh
b/assemblies/static/src/main/resources/hop-encrypt.sh
index 365b2e72d6..e67e2ab8a9 100755
--- a/assemblies/static/src/main/resources/hop-encrypt.sh
+++ b/assemblies/static/src/main/resources/hop-encrypt.sh
@@ -104,16 +104,6 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
-# Beam SDKs/runners live with the optional Beam plugin (not lib/beam).
-# Only on the system classpath when marketplace (or -Pbeam) has installed them.
-if [ -d "plugins/engines/beam/lib-beam" ]; then
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
-fi
-if [ -d "plugins/engines/beam" ]; then
- # Hop Beam plugin jars at the plugin root (exclude subdirs via non-recursive
glob is fine for /*.jar intent;
- # shell * includes jars and may pick non-jars; match historical lib/* style)
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
-fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -classpath "${CLASSPATH}"
org.apache.hop.encryption.HopEncrypt "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-gui-nolog.bat
b/assemblies/static/src/main/resources/hop-gui-nolog.bat
index 042384cd8e..b9297fa6bd 100644
--- a/assemblies/static/src/main/resources/hop-gui-nolog.bat
+++ b/assemblies/static/src/main/resources/hop-gui-nolog.bat
@@ -27,13 +27,6 @@ if not "%HOP_WINDOWS_SHELL_ENCODING%"=="" chcp
%HOP_WINDOWS_SHELL_ENCODING%
set LIBSPATH=lib\core
set CLASSPATH=lib\core\*;lib\swt\win64\*
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
set _temphelp=0
if [%1]==[help] set _temphelp=1
diff --git a/assemblies/static/src/main/resources/hop-gui.bat
b/assemblies/static/src/main/resources/hop-gui.bat
index 7fb165def5..7c00cd500c 100755
--- a/assemblies/static/src/main/resources/hop-gui.bat
+++ b/assemblies/static/src/main/resources/hop-gui.bat
@@ -32,13 +32,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git a/assemblies/static/src/main/resources/hop-gui.sh
b/assemblies/static/src/main/resources/hop-gui.sh
index 188ec8af38..1df8915e91 100755
--- a/assemblies/static/src/main/resources/hop-gui.sh
+++ b/assemblies/static/src/main/resources/hop-gui.sh
@@ -123,16 +123,6 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
-# Beam SDKs/runners live with the optional Beam plugin (not lib/beam).
-# Only on the system classpath when marketplace (or -Pbeam) has installed them.
-if [ -d "plugins/engines/beam/lib-beam" ]; then
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
-fi
-if [ -d "plugins/engines/beam" ]; then
- # Hop Beam plugin jars at the plugin root (exclude subdirs via non-recursive
glob is fine for /*.jar intent;
- # shell * includes jars and may pick non-jars; match historical lib/* style)
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
-fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.ui.hopgui.HopGui "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-import.bat
b/assemblies/static/src/main/resources/hop-import.bat
index 5a4b91e88c..210f1acbda 100644
--- a/assemblies/static/src/main/resources/hop-import.bat
+++ b/assemblies/static/src/main/resources/hop-import.bat
@@ -30,13 +30,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git a/assemblies/static/src/main/resources/hop-import.sh
b/assemblies/static/src/main/resources/hop-import.sh
index 5ee373c1ae..fc65d4ebd7 100755
--- a/assemblies/static/src/main/resources/hop-import.sh
+++ b/assemblies/static/src/main/resources/hop-import.sh
@@ -104,16 +104,6 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
-# Beam SDKs/runners live with the optional Beam plugin (not lib/beam).
-# Only on the system classpath when marketplace (or -Pbeam) has installed them.
-if [ -d "plugins/engines/beam/lib-beam" ]; then
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
-fi
-if [ -d "plugins/engines/beam" ]; then
- # Hop Beam plugin jars at the plugin root (exclude subdirs via non-recursive
glob is fine for /*.jar intent;
- # shell * includes jars and may pick non-jars; match historical lib/* style)
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
-fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.imp.HopImport "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-run.bat
b/assemblies/static/src/main/resources/hop-run.bat
index 9c4e215227..c4c370abf1 100755
--- a/assemblies/static/src/main/resources/hop-run.bat
+++ b/assemblies/static/src/main/resources/hop-run.bat
@@ -32,13 +32,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git a/assemblies/static/src/main/resources/hop-run.sh
b/assemblies/static/src/main/resources/hop-run.sh
index 83c03b7f5f..8b29434593 100755
--- a/assemblies/static/src/main/resources/hop-run.sh
+++ b/assemblies/static/src/main/resources/hop-run.sh
@@ -104,16 +104,6 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
-# Beam SDKs/runners live with the optional Beam plugin (not lib/beam).
-# Only on the system classpath when marketplace (or -Pbeam) has installed them.
-if [ -d "plugins/engines/beam/lib-beam" ]; then
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
-fi
-if [ -d "plugins/engines/beam" ]; then
- # Hop Beam plugin jars at the plugin root (exclude subdirs via non-recursive
glob is fine for /*.jar intent;
- # shell * includes jars and may pick non-jars; match historical lib/* style)
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
-fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.run.HopRun "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-search.bat
b/assemblies/static/src/main/resources/hop-search.bat
index 35571bb464..4526845a1b 100644
--- a/assemblies/static/src/main/resources/hop-search.bat
+++ b/assemblies/static/src/main/resources/hop-search.bat
@@ -29,13 +29,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git a/assemblies/static/src/main/resources/hop-search.sh
b/assemblies/static/src/main/resources/hop-search.sh
index 607f53261e..a16d884a8b 100755
--- a/assemblies/static/src/main/resources/hop-search.sh
+++ b/assemblies/static/src/main/resources/hop-search.sh
@@ -104,16 +104,6 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
-# Beam SDKs/runners live with the optional Beam plugin (not lib/beam).
-# Only on the system classpath when marketplace (or -Pbeam) has installed them.
-if [ -d "plugins/engines/beam/lib-beam" ]; then
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
-fi
-if [ -d "plugins/engines/beam" ]; then
- # Hop Beam plugin jars at the plugin root (exclude subdirs via non-recursive
glob is fine for /*.jar intent;
- # shell * includes jars and may pick non-jars; match historical lib/* style)
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
-fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.search.HopSearch "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-server.bat
b/assemblies/static/src/main/resources/hop-server.bat
index d47d761bf0..e6194a0d01 100644
--- a/assemblies/static/src/main/resources/hop-server.bat
+++ b/assemblies/static/src/main/resources/hop-server.bat
@@ -32,13 +32,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git a/assemblies/static/src/main/resources/hop-server.sh
b/assemblies/static/src/main/resources/hop-server.sh
index d732505d4d..f32919b2ba 100755
--- a/assemblies/static/src/main/resources/hop-server.sh
+++ b/assemblies/static/src/main/resources/hop-server.sh
@@ -109,16 +109,6 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
-# Beam SDKs/runners live with the optional Beam plugin (not lib/beam).
-# Only on the system classpath when marketplace (or -Pbeam) has installed them.
-if [ -d "plugins/engines/beam/lib-beam" ]; then
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
-fi
-if [ -d "plugins/engines/beam" ]; then
- # Hop Beam plugin jars at the plugin root (exclude subdirs via non-recursive
glob is fine for /*.jar intent;
- # shell * includes jars and may pick non-jars; match historical lib/* style)
- CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
-fi
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.www.HopServer "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop.bat
b/assemblies/static/src/main/resources/hop.bat
index 0b624c3364..d612752ea6 100755
--- a/assemblies/static/src/main/resources/hop.bat
+++ b/assemblies/static/src/main/resources/hop.bat
@@ -33,13 +33,6 @@ if defined HOP_SPARK_CLIENT_VERSION if exist
"lib\spark-clients\%HOP_SPARK_CLIEN
set CLASSPATH=%CLASSPATH%;lib\spark-clients\%HOP_SPARK_CLIENT_VERSION%\*
)
-REM Beam SDKs under optional plugin (plugins\engines\beam\lib-beam)
-if exist "plugins\engines\beam\lib-beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\lib-beam\*
-)
-if exist "plugins\engines\beam\" (
- set CLASSPATH=%CLASSPATH%;plugins\engines\beam\*
-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
index 11a0b36b14..3ca6d7f496 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
@@ -51,6 +51,15 @@
https://github.com/pemistahl/lingua?tab=readme-ov-file#3-which-languages-are-sup
This name must be unique throughout the pipeline.
|===
-NOTE: The dependencies for this transform are not included in the application
to make this work you need to include following files in the
/plugins/transforms/detectlanguage/lib folder +
--
https://repo1.maven.org/maven2/com/github/pemistahl/lingua/1.2.2/lingua-1.2.2.jar[Lingua]
+
--
https://repo1.maven.org/maven2/it/unimi/dsi/fastutil/8.5.15/fastutil-8.5.15.jar[Fastutil]
+
+== Installation
+
+This transform is not part of the default Hop client.
+Install it from the Marketplace perspective in Hop Gui (Detect language,
category Transforms), or with the
xref:hop-tools/hop-marketplace.adoc[marketplace command line]:
+
+[source,bash]
+----
+./hop marketplace install hop-transform-detectlanguage
+----
+
+The https://github.com/pemistahl/lingua[Lingua] language models and
https://fastutil.di.unimi.it/[fastutil] are bundled with the plugin, so no jars
have to be downloaded by hand.
+Because the language models are large, the plugin is around 100MB in size.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/stanfordnlp.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/stanfordnlp.adoc
index 2ebe90a68a..2a4cc7a650 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/stanfordnlp.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/stanfordnlp.adoc
@@ -39,15 +39,29 @@ Stanford CoreNLP provides a set of natural language
analysis tools which can tak
!===
|===
-NOTE: The dependencies for this transform are not included in the application
to make this work you need to include following files in the
/plugins/transforms/stanfordnlp/lib folder +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7.jar[stanford-corenlp
4.5.7] +
-Model for your language +
+== Installation
+
+This transform is not part of the default Hop client.
+Install it from the Marketplace perspective in Hop Gui (Stanford NLP, category
Transforms), or with the xref:hop-tools/hop-marketplace.adoc[marketplace
command line]:
+
+[source,bash]
+----
+./hop marketplace install hop-transform-stanfordnlp
+----
+
+The plugin ships the supporting libraries, but Stanford CoreNLP itself is
licensed under the GPL v3 and can not be distributed by the Apache Software
Foundation.
+Download the following two jars and drop them in the
`plugins/transforms/stanfordnlp/lib` folder: +
+ +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10.jar[stanford-corenlp
4.5.10] +
+- https://repo1.maven.org/maven2/xom/xom/1.3.9/xom-1.3.9.jar[xom 1.3.9]
+
+NOTE: The language models are not distributed either: pick the model for your
language and drop it in the same `plugins/transforms/stanfordnlp/lib` folder. +
+
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-arabic.jar[Arabic
Model] +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-chinese.jar[Chinese
Model] +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-english.jar[English
Model] +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-french.jar[French
Model] +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-german.jar[German
Model] +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-hungarian.jar[Hungarian
Model] +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-italian.jar[Italian
Model] +
--
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.7/stanford-corenlp-4.5.7-models-spanish.jar[Spanish
Model]
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-arabic.jar[Arabic
Model] +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-chinese.jar[Chinese
Model] +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-english.jar[English
Model] +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-french.jar[French
Model] +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-german.jar[German
Model] +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-hungarian.jar[Hungarian
Model] +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-italian.jar[Italian
Model] +
+-
https://repo1.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/4.5.10/stanford-corenlp-4.5.10-models-spanish.jar[Spanish
Model]
diff --git a/plugins/engines/beam/pom.xml b/plugins/engines/beam/pom.xml
index e13686c527..de34cb7cf1 100644
--- a/plugins/engines/beam/pom.xml
+++ b/plugins/engines/beam/pom.xml
@@ -34,6 +34,13 @@
<dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-libs</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-sdk-bom</artifactId>
@@ -62,13 +69,6 @@
<type>pom</type>
<scope>import</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hop</groupId>
- <artifactId>hop-libs</artifactId>
- <version>${project.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
@@ -1660,31 +1660,11 @@
</dependency>
</dependencies>
- <!--
- Two zips:
- - hop-engines-beam-${version}.zip : marketplace plugin
(plugins/engines/beam + lib-beam SDKs)
- - hop-engines-beam-${version}-libs.zip : slim client libs
(lib/spark-client + shared lib/core extras)
- Opt-in full Beam in hop-client: mvn … -Pbeam (assemblies/client)
- -->
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
- <execution>
- <id>assembly-beam-libs</id>
- <goals>
- <goal>single</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <appendAssemblyId>true</appendAssemblyId>
- <descriptors>
-
<descriptor>${project.basedir}/src/assembly/assembly-libs.xml</descriptor>
- </descriptors>
- <descriptorSourceDirectory
combine.self="override"></descriptorSourceDirectory>
- </configuration>
- </execution>
<execution>
<id>assembly_package</id>
<goals>
diff --git a/plugins/engines/beam/src/assembly/assembly-beam-runtime.xml
b/plugins/engines/beam/src/assembly/assembly-beam-runtime.xml
deleted file mode 100644
index 369c7ca832..0000000000
--- a/plugins/engines/beam/src/assembly/assembly-beam-runtime.xml
+++ /dev/null
@@ -1,154 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<component xmlns="http://maven.apache.org/ASSEMBLY/2.2.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0
http://maven.apache.org/xsd/assembly-2.2.0.xsd";>
- <!-- Beam SDK / runners: shipped with marketplace plugin under
plugins/engines/beam/lib-beam -->
- <dependencySets>
- <dependencySet>
- <scope>runtime</scope>
- <excludes>
- <exclude>org.apache.hop:hop-engines-beam:jar</exclude>
- <exclude>org.apache.hadoop:*:jar</exclude>
- <exclude>org.apache.hadoop.thirdparty:*:jar</exclude>
- <exclude>org.yaml:snakeyaml:jar</exclude>
- <exclude>com.google.protobuf:protobuf-java:jar</exclude>
- <exclude>com.google.protobuf:protobuf-java-util:jar</exclude>
- <exclude>org.jetbrains:annotations:jar</exclude>
-
<exclude>com.google.auto.value:auto-value-annotations:jar</exclude>
-
<exclude>org.apache.beam:beam-vendor-guava-32_1_2-jre:jar</exclude>
- <exclude>com.github.luben:zstd-jni:jar</exclude>
-
<exclude>com.google.auth:google-auth-library-oauth2-http:jar</exclude>
-
<exclude>com.google.oauth-client:google-oauth-client:jar</exclude>
- <exclude>com.google.api-client:google-api-client:jar</exclude>
-
<exclude>com.google.http-client:google-http-client:jar</exclude>
-
<exclude>com.google.http-client:google-http-client-apache-v2:jar</exclude>
-
<exclude>com.google.http-client:google-http-client-appengine:jar</exclude>
- <exclude>google-http-client-gson:jar</exclude>
-
<exclude>com.google.http-client:google-http-client-jackson2:jar</exclude>
- <exclude>com.google.android:annotations:jar</exclude>
-
<exclude>org.codehaus.mojo:animal-sniffer-annotations:jar</exclude>
- <exclude>com.google.api:api-common:jar</exclude>
-
<exclude>com.google.auto.value:auto-value-annotations:jar</exclude>
- <exclude>com.google.guava::jar</exclude>
- <exclude>org.jetbrains.kotlin::jar</exclude>
- <exclude>io.netty::jar</exclude>
-
<exclude>org.codehaus.mojo:animal-sniffer-annotations:jar</exclude>
- <exclude>io.projectreactor.netty::jar</exclude>
- <exclude>io.grpc:grpc-netty-shaded:jar</exclude>
- <exclude>io.grpc:grpc-xds:jar</exclude>
- <exclude>org.apache.kafka::jar</exclude>
- <exclude>org.xerial.snappy:snappy-java:jar</exclude>
- <exclude>org.eclipse.jetty::jar</exclude>
- <exclude>org.eclipse.jetty.compression::jar</exclude>
- <exclude>com.google.cloud:google-cloud-storage:jar</exclude>
- <exclude>io.grpc::jar</exclude>
- <exclude>com.google.cloud:google-cloud-core-grpc:jar</exclude>
-
<exclude>com.google.api.grpc:grpc-google-cloud-storage-v2:jar</exclude>
- <exclude>com.google.api:gax-grpc:jar</exclude>
- <exclude>com.google.api.grpc::jar</exclude>
- <exclude>org.checkerframework:checker-qual:jar</exclude>
- <exclude>io.confluent:common-utils:jar</exclude>
- <exclude>commons-cli:commons-cli:jar</exclude>
- <exclude>commons-codec:commons-codec:jar</exclude>
- <exclude>commons-logging:commons-logging:jar</exclude>
- <exclude>org.apache.commons:commons-text:jar</exclude>
-
<exclude>com.google.cloud.opentelemetry:detector-resources-support:jar</exclude>
-
<exclude>com.google.errorprone:error_prone_annotations:jar</exclude>
-
<exclude>com.google.cloud.opentelemetry:exporter-metrics:jar</exclude>
- <exclude>com.google.api:gax:jar</exclude>
- <exclude>com.google.api:gax-httpjson:jar</exclude>
-
<exclude>com.google.apis:google-api-services-storage:jar</exclude>
-
<exclude>com.google.auth:google-auth-library-credentials:jar</exclude>
-
<exclude>com.google.auth:google-auth-library-oauth2-http:jar</exclude>
- <exclude>com.google.cloud:google-cloud-core:jar</exclude>
- <exclude>com.google.cloud:google-cloud-core-http:jar</exclude>
- <exclude>com.google.cloud:google-cloud-monitoring:jar</exclude>
-
<exclude>com.google.http-client:google-http-client-gson:jar</exclude>
- <exclude>org.apache.httpcomponents:httpclient:jar</exclude>
- <exclude>org.apache.httpcomponents:httpcore:jar</exclude>
- <exclude>com.google.j2objc:j2objc-annotations:jar</exclude>
- <exclude>com.fasterxml.jackson.core::jar</exclude>
- <exclude>com.fasterxml.jackson.dataformat::jar</exclude>
- <exclude>com.fasterxml.jackson.datatype::jar</exclude>
- <exclude>com.fasterxml.jackson::jar</exclude>
-
<exclude>com.github.stephenc.jcip:jcip-annotations:jar</exclude>
- <exclude>net.java.dev.jna::jar</exclude>
- <exclude>com.google.code.findbugs:jsr305:jar</exclude>
- <exclude>com.squareup.okhttp3:okhttp:jar</exclude>
- <exclude>com.squareup.okio::jar</exclude>
- <exclude>io.opentelemetry::jar</exclude>
-
<exclude>com.google.cloud.opentelemetry:shared-resourcemapping:jar</exclude>
- <exclude>org.codehaus.woodstox:stax2-api:jar</exclude>
- <exclude>org.threeten::jar</exclude>
- <exclude>org.lz4:lz4-java:jar</exclude>
- <exclude>io.opentelemetry.contrib::jar</exclude>
- <exclude>io.opentelemetry.semconv::jar</exclude>
- <exclude>io.perfmark:perfmark-api:jar</exclude>
- <exclude>com.google.re2j:re2j:jar</exclude>
- <exclude>org.reactivestreams:reactive-streams:jar</exclude>
- <exclude>io.projectreactor:reactor-core:jar</exclude>
- <exclude>org.bouncycastle::jar</exclude>
- <exclude>com.microsoft.azure:msal4j:jar</exclude>
-
<exclude>com.microsoft.azure:msal4j-persistence-extension:jar</exclude>
- <exclude>com.amazonaws:aws-java-sdk-core:jar</exclude>
- <exclude>com.amazonaws:aws-java-sdk-kms:jar</exclude>
- <exclude>com.amazonaws:aws-java-sdk-s3:jar</exclude>
- <exclude>com.nimbusds:oauth2-oidc-sdk:jar</exclude>
- <exclude>com.azure:azure-core:jar</exclude>
- <exclude>com.azure:azure-identity:jar</exclude>
- <exclude>com.azure:azure-core-http-netty:jar</exclude>
- <exclude>com.azure:azure-json:jar</exclude>
- <exclude>com.microsoft.azure:azure-keyvault-core:jar</exclude>
-
<exclude>com.azure:azure-security-keyvault-secrets:jar</exclude>
- <exclude>com.azure:azure-storage-blob:jar</exclude>
- <exclude>com.azure:azure-storage-common:jar</exclude>
- <exclude>com.azure:azure-storage-internal-avro:jar</exclude>
- <exclude>com.azure:azure-xml:jar</exclude>
- <exclude>com.nimbusds:nimbus-jose-jwt:jar</exclude>
- <exclude>io.confluent::jar</exclude>
- <exclude>com.google.code.gson:gson:jar</exclude>
- <exclude>org.apache.commons:commons-collections4:jar</exclude>
- <exclude>org.jspecify:jspecify:jar</exclude>
- <exclude>org.apache.commons:commons-lang3:jar</exclude>
- <exclude>commons-collections:commons-collections:jar</exclude>
- <exclude>com.fasterxml.woodstox:woodstox-core:jar</exclude>
- <exclude>com.ibm.icu:icu4j:jar</exclude>
-
<exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude>
- <exclude>software.amazon.awssdk:*:jar</exclude>
- <exclude>software.amazon.eventstream:eventstream:jar</exclude>
- <exclude>javax.annotation:javax.annotation-api:jar</exclude>
- <exclude>javax.inject:javax.inject:jar</exclude>
- <exclude>org.jctools:jctools-core:jar</exclude>
- <exclude>com.nimbusds:lang-tag:jar</exclude>
- <exclude>io.dropwizard.metrics:*:jar</exclude>
-
<exclude>com.eclipsesource.minimal-json:minimal-json:jar</exclude>
- <exclude>org.slf4j:slf4j-api:jar</exclude>
- <!-- Spark client runtime lives in lib/spark-client (versioned
packs); keep Beam own Spark fragments in lib-beam -->
- <exclude>org.apache.spark:spark-core_2.12:jar</exclude>
- <exclude>org.apache.spark:spark-streaming_2.12:jar</exclude>
- <exclude>org.apache.spark:spark-launcher_2.12:jar</exclude>
- <exclude>org.apache.spark:spark-kvstore_2.12:jar</exclude>
-
<exclude>org.apache.spark:spark-network-common_2.12:jar</exclude>
-
<exclude>org.apache.spark:spark-network-shuffle_2.12:jar</exclude>
- <exclude>org.apache.spark:spark-unsafe_2.12:jar</exclude>
- <exclude>org.apache.spark:spark-tags_2.12:jar</exclude>
- </excludes>
- <outputDirectory>plugins/engines/beam/lib-beam</outputDirectory>
- </dependencySet>
- </dependencySets>
-</component>
diff --git a/plugins/engines/beam/src/assembly/assembly-libs.xml
b/plugins/engines/beam/src/assembly/assembly-libs.xml
deleted file mode 100644
index 4736424a1c..0000000000
--- a/plugins/engines/beam/src/assembly/assembly-libs.xml
+++ /dev/null
@@ -1,161 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0
http://maven.apache.org/xsd/assembly-2.2.0.xsd";>
- <!-- Slim client libs: default Spark client pack + shared lib/core extras
from Beam deps. Beam SDKs live in plugin zip (lib-beam). -->
- <id>libs</id>
- <formats>
- <format>zip</format>
- </formats>
- <baseDirectory>.</baseDirectory>
-
- <dependencySets>
- <!-- hop-engines-beam.jar is packaged in the plugin zip -->
-
- <!-- Default Spark client pack (matches plugins/engines/beam
spark.version). Optional
- additional packs can be materialised under
lib/spark-clients/<version>/ via
- tools/spark-client-pack. Selected at runtime with
HOP_SPARK_CLIENT_VERSION. -->
- <dependencySet>
- <scope>runtime</scope>
- <includes>
- <include>org.apache.spark:spark-core_2.12:jar</include>
- <include>org.apache.spark:spark-streaming_2.12:jar</include>
- <include>org.apache.spark:spark-launcher_2.12:jar</include>
- <include>org.apache.spark:spark-kvstore_2.12:jar</include>
-
<include>org.apache.spark:spark-network-common_2.12:jar</include>
-
<include>org.apache.spark:spark-network-shuffle_2.12:jar</include>
- <include>org.apache.spark:spark-unsafe_2.12:jar</include>
- <include>org.apache.spark:spark-tags_2.12:jar</include>
- </includes>
- <outputDirectory>lib/spark-client</outputDirectory>
- </dependencySet>
- <dependencySet>
- <scope>runtime</scope>
- <includes>
- <include>org.apache.hadoop:*:jar</include>
- <include>org.apache.hadoop.thirdparty:*:jar</include>
- <include>org.yaml:snakeyaml:jar</include>
- <include>org.jetbrains:annotations:jar</include>
-
<include>com.google.auto.value:auto-value-annotations:jar</include>
-
<include>org.apache.beam:beam-vendor-guava-32_1_2-jre:jar</include>
- <include>com.github.luben:zstd-jni:jar</include>
-
<include>com.google.auth:google-auth-library-oauth2-http:jar</include>
-
<include>com.google.oauth-client:google-oauth-client:jar</include>
- <include>com.google.api-client:google-api-client:jar</include>
-
<include>com.google.http-client:google-http-client:jar</include>
-
<include>com.google.http-client:google-http-client-apache-v2:jar</include>
-
<include>com.google.http-client:google-http-client-appengine:jar</include>
- <include>google-http-client-gson:jar</include>
-
<include>com.google.http-client:google-http-client-jackson2:jar</include>
- <include>com.google.android:annotations:jar</include>
-
<include>org.codehaus.mojo:animal-sniffer-annotations:jar</include>
- <include>com.google.api:api-common:jar</include>
-
<include>com.google.auto.value:auto-value-annotations:jar</include>
- <include>org.jetbrains.kotlin::jar</include>
- <include>io.netty::jar</include>
- <include>io.projectreactor.netty::jar</include>
- <include>io.projectreactor:reactor-core:jar</include>
- <include>io.grpc:grpc-netty-shaded:jar</include>
- <include>org.apache.kafka::jar</include>
- <include>com.google.cloud:google-cloud-storage:jar</include>
- <include>io.grpc::jar</include>
- <include>com.google.cloud:google-cloud-core-grpc:jar</include>
-
<include>com.google.api.grpc:grpc-google-cloud-storage-v2:jar</include>
- <include>com.google.api:gax-grpc:jar</include>
- <include>com.google.api:gax:jar</include>
- <include>com.google.api.grpc::jar</include>
- <include>org.checkerframework:checker-qual:jar</include>
- <include>io.confluent:common-utils:jar</include>
- <include>commons-cli:commons-cli:jar</include>
- <include>commons-codec:commons-codec:jar</include>
- <include>commons-logging:commons-logging:jar</include>
- <include>org.apache.commons:commons-text:jar</include>
-
<include>com.google.cloud.opentelemetry:detector-resources-support:jar</include>
-
<include>com.google.errorprone:error_prone_annotations:jar</include>
-
<include>com.google.cloud.opentelemetry:exporter-metrics:jar</include>
- <include>com.google.api:gax-httpjson:jar</include>
-
<include>com.google.apis:google-api-services-storage:jar</include>
-
<include>com.google.auth:google-auth-library-credentials:jar</include>
-
<include>com.google.auth:google-auth-library-oauth2-http:jar</include>
- <include>com.google.cloud:google-cloud-core:jar</include>
- <include>com.google.cloud:google-cloud-core-http:jar</include>
- <include>com.google.cloud:google-cloud-monitoring:jar</include>
-
<include>com.google.http-client:google-http-client-gson:jar</include>
- <include>org.apache.httpcomponents:httpclient:jar</include>
- <include>org.apache.httpcomponents:httpcore:jar</include>
- <include>com.google.j2objc:j2objc-annotations:jar</include>
- <include>com.fasterxml.jackson.core::jar</include>
- <include>com.fasterxml.jackson.dataformat::jar</include>
- <include>com.fasterxml.jackson.datatype::jar</include>
- <include>com.fasterxml.jackson::jar</include>
-
<include>com.github.stephenc.jcip:jcip-annotations:jar</include>
- <include>net.java.dev.jna::jar</include>
- <include>com.google.code.findbugs:jsr305:jar</include>
- <include>com.squareup.okhttp3:okhttp:jar</include>
- <include>com.squareup.okio::jar</include>
- <include>io.opentelemetry::jar</include>
- <include>io.opentelemetry.contrib::jar</include>
- <include>io.opentelemetry.semconv::jar</include>
-
<include>com.google.cloud.opentelemetry:shared-resourcemapping:jar</include>
- <include>org.codehaus.woodstox:stax2-api:jar</include>
- <include>org.threeten::jar</include>
- <include>org.lz4:lz4-java:jar</include>
- <include>io.perfmark:perfmark-api:jar</include>
- <include>com.google.re2j:re2j:jar</include>
- <include>org.reactivestreams:reactive-streams:jar</include>
- <include>org.bouncycastle::jar</include>
- <include>com.microsoft.azure:msal4j:jar</include>
-
<include>com.microsoft.azure:msal4j-persistence-extension:jar</include>
- <include>com.amazonaws:aws-java-sdk-core:jar</include>
- <include>com.amazonaws:aws-java-sdk-kms:jar</include>
- <include>com.amazonaws:aws-java-sdk-s3:jar</include>
- <include>com.nimbusds:oauth2-oidc-sdk:jar</include>
- <include>com.azure:azure-core:jar</include>
- <include>com.azure:azure-identity:jar</include>
- <include>com.azure:azure-core-http-netty:jar</include>
- <include>com.azure:azure-json:jar</include>
- <include>com.microsoft.azure:azure-keyvault-core:jar</include>
-
<include>com.azure:azure-security-keyvault-secrets:jar</include>
- <include>com.azure:azure-storage-blob:jar</include>
- <include>com.azure:azure-storage-common:jar</include>
- <include>com.azure:azure-storage-internal-avro:jar</include>
- <include>com.azure:azure-xml:jar</include>
- <include>com.nimbusds:nimbus-jose-jwt:jar</include>
- <include>io.confluent::jar</include>
- <include>com.google.code.gson:gson:jar</include>
- <include>org.apache.commons:commons-collections4:jar</include>
- <include>org.apache.commons:commons-lang3:jar</include>
- <include>commons-collections:commons-collections:jar</include>
- <include>com.fasterxml.woodstox:woodstox-core:jar</include>
- <include>com.ibm.icu:icu4j:jar</include>
-
<include>com.fasterxml.jackson.core:jackson-annotations:jar</include>
- <include>software.amazon.awssdk:*:jar</include>
- <include>software.amazon.eventstream:eventstream:jar</include>
- <include>javax.annotation:javax.annotation-api:jar</include>
- <include>javax.inject:javax.inject:jar</include>
- <include>org.jctools:jctools-core:jar</include>
- <include>com.nimbusds:lang-tag:jar</include>
- <include>io.dropwizard.metrics:*:jar</include>
-
<include>com.eclipsesource.minimal-json:minimal-json:jar</include>
- <include>org.slf4j:slf4j-api:jar</include>
- </includes>
- <outputDirectory>lib/core</outputDirectory>
- </dependencySet>
- </dependencySets>
-</assembly>
\ No newline at end of file
diff --git a/plugins/engines/beam/src/assembly/assembly.xml
b/plugins/engines/beam/src/assembly/assembly.xml
index fe175665fb..4aa190324f 100644
--- a/plugins/engines/beam/src/assembly/assembly.xml
+++ b/plugins/engines/beam/src/assembly/assembly.xml
@@ -18,7 +18,6 @@
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0
http://maven.apache.org/xsd/assembly-2.2.0.xsd";>
- <!-- Marketplace plugin: Hop Beam jar + Beam SDKs under lib-beam (runners
etc.). -->
<id>plugin</id>
<formats>
<format>zip</format>
@@ -51,9 +50,41 @@
</includes>
<outputDirectory>plugins/engines/beam</outputDirectory>
</dependencySet>
- </dependencySets>
- <componentDescriptors>
-
<componentDescriptor>${project.basedir}/src/assembly/assembly-beam-runtime.xml</componentDescriptor>
- </componentDescriptors>
+ <!-- Default Spark client pack. Kept in its own folder so
HOP_SPARK_CLIENT_VERSION can
+ swap it for a versioned pack under lib/spark-clients/<version>/
(materialised with
+ tools/spark-client-pack) without touching the rest of the
installation. -->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <includes>
+ <include>org.apache.spark:spark-core_2.12:jar</include>
+ <include>org.apache.spark:spark-streaming_2.12:jar</include>
+ <include>org.apache.spark:spark-launcher_2.12:jar</include>
+ <include>org.apache.spark:spark-kvstore_2.12:jar</include>
+
<include>org.apache.spark:spark-network-common_2.12:jar</include>
+
<include>org.apache.spark:spark-network-shuffle_2.12:jar</include>
+ <include>org.apache.spark:spark-unsafe_2.12:jar</include>
+ <include>org.apache.spark:spark-tags_2.12:jar</include>
+ </includes>
+ <outputDirectory>lib/spark-client</outputDirectory>
+ </dependencySet>
+
+ <!-- Beam SDKs, runners and everything they need - the former lib/beam
-->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <excludes>
+ <exclude>org.apache.spark:spark-core_2.12</exclude>
+ <exclude>org.apache.spark:spark-streaming_2.12</exclude>
+ <exclude>org.apache.spark:spark-launcher_2.12</exclude>
+ <exclude>org.apache.spark:spark-kvstore_2.12</exclude>
+ <exclude>org.apache.spark:spark-network-common_2.12</exclude>
+ <exclude>org.apache.spark:spark-network-shuffle_2.12</exclude>
+ <exclude>org.apache.spark:spark-unsafe_2.12</exclude>
+ <exclude>org.apache.spark:spark-tags_2.12</exclude>
+ </excludes>
+ <outputDirectory>lib/core</outputDirectory>
+ </dependencySet>
+ </dependencySets>
</assembly>
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
index ac8efbf8ab..2f5d2dd7b1 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java
@@ -461,7 +461,7 @@ public class HopBeamGuiPlugin {
/**
* @param versionedPack when true, also skip {@code spark-*.jar} under Beam
SDK folders ({@code
- * lib/beam} legacy or {@code plugins/engines/beam/lib-beam}) so Beam's
fixed Spark fragments
+ * lib/beam} legacy or {@code plugins/engines/beam/lib}) so Beam's fixed
Spark fragments
* cannot mix with an alternate client pack
*/
private static void collectJarsExcludingSparkClientPacks(
@@ -485,7 +485,7 @@ public class HopBeamGuiPlugin {
if (child.isDirectory()) {
collectJarsExcludingSparkClientPacks(child, jarFiles, versionedPack);
} else if (child.isFile() && child.getName().endsWith(".jar")) {
- // Versioned packs own all spark-* jars; drop Beam's spark fragments
from lib-beam /
+ // Versioned packs own all spark-* jars; drop Beam's spark fragments
from the plugin /
// lib/beam
if (versionedPack
&& child.getName().startsWith("spark-")
@@ -498,7 +498,12 @@ public class HopBeamGuiPlugin {
}
}
- /** True for legacy {@code lib/beam} or marketplace {@code …/beam/lib-beam}.
*/
+ /**
+ * True for the Beam SDK folder of older layouts: the top-level {@code
lib/beam} before Beam
+ * became a marketplace plugin, and {@code plugins/engines/beam/lib-beam} in
between. Current
+ * installs put the Beam SDKs in {@code lib/core}, where the generic spark-*
guard below already
+ * covers them.
+ */
static boolean isBeamSdkJarFolder(File folder) {
if (folder == null) {
return false;
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/OptionalPluginCatalog.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/OptionalPluginCatalog.java
index e62767e0d4..35d233a548 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/OptionalPluginCatalog.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/OptionalPluginCatalog.java
@@ -149,7 +149,7 @@ public final class OptionalPluginCatalog {
"hop-engines-beam",
"Apache Beam engine",
"Engines",
- "Beam pipeline engine and transforms (includes Beam SDKs under
lib-beam).",
+ "Beam pipeline engine and transforms (includes the Beam SDKs).",
"plugins/engines/beam"));
wave1.add(
new OptionalPluginInfo(
diff --git
a/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
b/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
index f6a49b35d1..d38681230d 100644
---
a/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
+++
b/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
@@ -44,6 +44,12 @@ plugins:
description: Scripting transform with embedded language runtimes.
installPath: plugins/transforms/script
modulePath: plugins/transforms/script
+ - artifactId: hop-transform-detectlanguage
+ name: Detect language
+ category: Transforms
+ description: Detect the language of a text field; includes the Lingua
language models.
+ installPath: plugins/transforms/detectlanguage
+ modulePath: plugins/transforms/detectlanguage
- artifactId: hop-transform-tika
name: Apache Tika
category: Transforms
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
index 44ac307892..a63e343918 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
@@ -55,7 +55,7 @@ class PluginInstallerTest {
@Test
void sharedCorePaths() {
assertFalse(PluginInstaller.isSharedCorePath("lib/beam/foo.jar"));
-
assertFalse(PluginInstaller.isSharedCorePath("plugins/engines/beam/lib-beam/x.jar"));
+
assertFalse(PluginInstaller.isSharedCorePath("plugins/engines/beam/hop-engines-beam.jar"));
assertTrue(PluginInstaller.isSharedCorePath("lib/core/bar.jar"));
assertTrue(PluginInstaller.isSharedCorePath("lib/core"));
assertFalse(PluginInstaller.isSharedCorePath("plugins/engines/beam/hop.jar"));
diff --git a/plugins/transforms/detectlanguage/pom.xml
b/plugins/transforms/detectlanguage/pom.xml
index b3108ec42c..6125b6932a 100644
--- a/plugins/transforms/detectlanguage/pom.xml
+++ b/plugins/transforms/detectlanguage/pom.xml
@@ -29,11 +29,6 @@
<packaging>jar</packaging>
<name>Hop Plugins Transforms Detect Language</name>
- <properties>
- <hop.pluginlib.exclude1>lingua</hop.pluginlib.exclude1>
- <hop.pluginlib.exclude2>fastutil</hop.pluginlib.exclude2>
- </properties>
-
<dependencyManagement>
<dependencies>
<dependency>
diff --git a/plugins/transforms/stanfordnlp/pom.xml
b/plugins/transforms/stanfordnlp/pom.xml
index 2882aacd32..4813edc593 100644
--- a/plugins/transforms/stanfordnlp/pom.xml
+++ b/plugins/transforms/stanfordnlp/pom.xml
@@ -30,6 +30,13 @@
<name>Hop Plugins Transforms Stanford Simple NLP</name>
<properties>
+ <!-- These are compiled against but must never be shipped:
stanford-corenlp is GPLv3, xom is
+ LGPL 2.1 and AppleJavaExtensions carries Apple's own license, all
ASF category X.
+ Users download stanford-corenlp (and the language model of their
choice) themselves,
+ see the "Stanford NLP" page in the user manual. -->
+ <hop.pluginlib.exclude1>stanford-corenlp</hop.pluginlib.exclude1>
+ <hop.pluginlib.exclude2>xom</hop.pluginlib.exclude2>
+ <hop.pluginlib.exclude3>AppleJavaExtensions</hop.pluginlib.exclude3>
<stanford-corenlp.version>4.5.10</stanford-corenlp.version>
</properties>
