This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hop-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 0ea095b3a8 Website updated to b04100c
0ea095b3a8 is described below
commit 0ea095b3a8b3f74a9a3e62dee5ac08511edee089
Author: jenkins <[email protected]>
AuthorDate: Tue Nov 3 14:59:56 2026 +0000
Website updated to b04100c
---
dev-manual/next/ai-advisor-plugins.html | 4 +
dev-manual/next/annotation-derived-widgets.html | 4 +
.../next/apache-release/checking-a-release.html | 4 +
.../next/apache-release/creating-a-release.html | 4 +
dev-manual/next/apache-release/index.html | 4 +
.../next/apache-release/promoting-a-release.html | 4 +
.../dependencies-and-classloading.html | 4 +
dev-manual/next/architecture/execution-model.html | 4 +
.../next/architecture/logging-and-debugging.html | 4 +
dev-manual/next/architecture/performance.html | 4 +
dev-manual/next/architecture/security.html | 4 +
.../next/architecture/variables-and-metadata.html | 4 +
dev-manual/next/build-and-test.html | 4 +
dev-manual/next/contributing.html | 4 +
dev-manual/next/database/column-types.html | 4 +
dev-manual/next/database/creating-a-dialect.html | 4 +
.../next/database/extending-another-dialect.html | 4 +
dev-manual/next/database/index.html | 4 +
.../next/database/migrating-from-variants.html | 4 +
dev-manual/next/database/testing.html | 4 +
dev-manual/next/database/value-bindings.html | 4 +
dev-manual/next/docker-build.html | 4 +
dev-manual/next/engine-compatibility.html | 4 +
dev-manual/next/getting-started.html | 4 +
dev-manual/next/gui-plugins-toolbars.html | 4 +
dev-manual/next/hopweb/developer-guide.html | 4 +
dev-manual/next/hopweb/hopweb-antipatterns.html | 4 +
dev-manual/next/hopweb/index.html | 4 +
dev-manual/next/index.html | 4 +
dev-manual/next/integration-testing.html | 4 +
dev-manual/next/internationalisation.html | 4 +
dev-manual/next/lineage.html | 4 +
dev-manual/next/marketplace-publish-plugin.html | 4 +
dev-manual/next/metadata-injection.html | 4 +
dev-manual/next/metadata-plugins.html | 4 +
dev-manual/next/metadata-serialization.html | 4 +
dev-manual/next/plugin-development.html | 4 +
dev-manual/next/plugin-samples.html | 4 +
dev-manual/next/plugin-types/canvas-overlays.html | 4 +
.../compression-formats.html} | 186 ++++++++++-----------
.../next/plugin-types/data-connectivity.html | 4 +
.../next/plugin-types/diagram-exporters.html | 6 +-
.../next/plugin-types/execution-observability.html | 4 +
dev-manual/next/plugin-types/gui.html | 6 +-
dev-manual/next/plugin-types/index.html | 4 +
.../next/plugin-types/metadata-configuration.html | 4 +
.../next/plugin-types/pipeline-workflow.html | 4 +
dev-manual/next/porting-kettle-plugins.html | 4 +
dev-manual/next/sdk/hop-sdk.html | 4 +
dev-manual/next/sdk/index.html | 4 +
dev-manual/next/sdk/logging.html | 4 +
dev-manual/next/sdk/metadata.html | 4 +
dev-manual/next/sdk/pipelines.html | 4 +
dev-manual/next/sdk/plugins/actions.html | 4 +
dev-manual/next/sdk/plugins/commands.html | 4 +
dev-manual/next/sdk/plugins/configuration.html | 4 +
dev-manual/next/sdk/plugins/databases.html | 4 +
dev-manual/next/sdk/plugins/execution-info.html | 4 +
dev-manual/next/sdk/plugins/extension-points.html | 4 +
dev-manual/next/sdk/plugins/file-types.html | 4 +
dev-manual/next/sdk/plugins/gui.html | 4 +
dev-manual/next/sdk/plugins/index.html | 4 +
dev-manual/next/sdk/plugins/metadata.html | 4 +
dev-manual/next/sdk/plugins/perspectives.html | 4 +
dev-manual/next/sdk/plugins/servlets.html | 4 +
dev-manual/next/sdk/plugins/transforms.html | 4 +
dev-manual/next/sdk/plugins/value-types.html | 4 +
.../next/sdk/plugins/variable-resolvers.html | 4 +
dev-manual/next/sdk/plugins/vfs.html | 4 +
dev-manual/next/sdk/vfs.html | 4 +
dev-manual/next/sdk/workflows.html | 4 +
dev-manual/next/setup-dev-environment.html | 4 +
.../next/snippets/hop-license-header-file.html | 4 +
dev-manual/next/start-your-own-plugin.html | 4 +
dev-manual/next/svg-files.html | 4 +
dev-manual/next/testing/ui-testing.html | 4 +
dev-manual/next/testing/unit-testing.html | 4 +
dev-manual/next/value-types.html | 4 +
dev-manual/next/writing-documentation.html | 4 +
.../_images/hop-gui/pipeline/debug-partial-run.jpg | Bin 0 -> 500055 bytes
.../hop-gui/pipeline/debug-pause-condition.jpg | Bin 0 -> 116225 bytes
.../hop-gui/pipeline/debug-session-started.jpg | Bin 0 -> 285778 bytes
.../_images/hop-gui/pipeline/debug-single-step.jpg | Bin 0 -> 225962 bytes
manual/next/docker-container.html | 5 +
manual/next/hop-gui/perspective-configuration.html | 27 +++
manual/next/hop-run/index.html | 107 ++++++++++++
.../beam-flink-pipeline-engine.html | 29 ++--
.../native-local-pipeline-engine.html | 13 ++
.../native-spark-pipeline-engine.html | 2 +-
.../next/pipeline/run-preview-debug-pipeline.html | 128 ++++++++++++++
.../spark/getting-started-with-native-spark.html | 2 +-
manual/next/pipeline/spark/lakehouse.html | 2 +-
.../pipeline/spark/local-standalone-cluster.html | 4 +-
.../pipeline/transforms/parquet-file-input.html | 8 +
.../pipeline/transforms/parquet-file-output.html | 8 +
.../next/pipeline/transforms/spark-file-input.html | 4 +-
.../pipeline/transforms/spark-file-output.html | 4 +-
manual/next/variables.html | 5 -
manual/next/workflow/actions/tableexists.html | 14 +-
manual/next/workflow/actions/truncatetables.html | 4 +-
sitemap-dev-manual.xml | 3 +
101 files changed, 722 insertions(+), 149 deletions(-)
diff --git a/dev-manual/next/ai-advisor-plugins.html
b/dev-manual/next/ai-advisor-plugins.html
index 764a89b2fb..5b03af418e 100644
--- a/dev-manual/next/ai-advisor-plugins.html
+++ b/dev-manual/next/ai-advisor-plugins.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/annotation-derived-widgets.html
b/dev-manual/next/annotation-derived-widgets.html
index a17ebb42a1..2fe0e18617 100644
--- a/dev-manual/next/annotation-derived-widgets.html
+++ b/dev-manual/next/annotation-derived-widgets.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/apache-release/checking-a-release.html
b/dev-manual/next/apache-release/checking-a-release.html
index 94046009fb..572b49203f 100644
--- a/dev-manual/next/apache-release/checking-a-release.html
+++ b/dev-manual/next/apache-release/checking-a-release.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/apache-release/creating-a-release.html
b/dev-manual/next/apache-release/creating-a-release.html
index 8b887c22af..e8b1f30ecf 100644
--- a/dev-manual/next/apache-release/creating-a-release.html
+++ b/dev-manual/next/apache-release/creating-a-release.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/apache-release/index.html
b/dev-manual/next/apache-release/index.html
index eb63f9ab84..7d7514e2ee 100644
--- a/dev-manual/next/apache-release/index.html
+++ b/dev-manual/next/apache-release/index.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/apache-release/promoting-a-release.html
b/dev-manual/next/apache-release/promoting-a-release.html
index 82a4f4dc8c..fac263d3e4 100644
--- a/dev-manual/next/apache-release/promoting-a-release.html
+++ b/dev-manual/next/apache-release/promoting-a-release.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/architecture/dependencies-and-classloading.html
b/dev-manual/next/architecture/dependencies-and-classloading.html
index 8dfa1b1cdc..b3872270b2 100644
--- a/dev-manual/next/architecture/dependencies-and-classloading.html
+++ b/dev-manual/next/architecture/dependencies-and-classloading.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/architecture/execution-model.html
b/dev-manual/next/architecture/execution-model.html
index 243cd89735..5577321730 100644
--- a/dev-manual/next/architecture/execution-model.html
+++ b/dev-manual/next/architecture/execution-model.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/architecture/logging-and-debugging.html
b/dev-manual/next/architecture/logging-and-debugging.html
index ff9d2be169..a3707d840f 100644
--- a/dev-manual/next/architecture/logging-and-debugging.html
+++ b/dev-manual/next/architecture/logging-and-debugging.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/architecture/performance.html
b/dev-manual/next/architecture/performance.html
index 237c8ffe42..3f570f9ce1 100644
--- a/dev-manual/next/architecture/performance.html
+++ b/dev-manual/next/architecture/performance.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/architecture/security.html
b/dev-manual/next/architecture/security.html
index aecc8eccf1..ed575b1af2 100644
--- a/dev-manual/next/architecture/security.html
+++ b/dev-manual/next/architecture/security.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/architecture/variables-and-metadata.html
b/dev-manual/next/architecture/variables-and-metadata.html
index 5a9819bb1f..3a64981b11 100644
--- a/dev-manual/next/architecture/variables-and-metadata.html
+++ b/dev-manual/next/architecture/variables-and-metadata.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/build-and-test.html
b/dev-manual/next/build-and-test.html
index 22d5dd88a5..ee0f6243c2 100644
--- a/dev-manual/next/build-and-test.html
+++ b/dev-manual/next/build-and-test.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/contributing.html
b/dev-manual/next/contributing.html
index ead43fc172..68177b2acc 100644
--- a/dev-manual/next/contributing.html
+++ b/dev-manual/next/contributing.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/column-types.html
b/dev-manual/next/database/column-types.html
index 072e7c9375..d0e66a7346 100644
--- a/dev-manual/next/database/column-types.html
+++ b/dev-manual/next/database/column-types.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/creating-a-dialect.html
b/dev-manual/next/database/creating-a-dialect.html
index d508b9c141..a0cc3068c7 100644
--- a/dev-manual/next/database/creating-a-dialect.html
+++ b/dev-manual/next/database/creating-a-dialect.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/extending-another-dialect.html
b/dev-manual/next/database/extending-another-dialect.html
index ee014d7be2..7234dc3c58 100644
--- a/dev-manual/next/database/extending-another-dialect.html
+++ b/dev-manual/next/database/extending-another-dialect.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/index.html
b/dev-manual/next/database/index.html
index b9de55b027..bf47c21409 100644
--- a/dev-manual/next/database/index.html
+++ b/dev-manual/next/database/index.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/migrating-from-variants.html
b/dev-manual/next/database/migrating-from-variants.html
index 3d2913203f..0bf0bb3a79 100644
--- a/dev-manual/next/database/migrating-from-variants.html
+++ b/dev-manual/next/database/migrating-from-variants.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/testing.html
b/dev-manual/next/database/testing.html
index 26323b79e7..d35a871f6e 100644
--- a/dev-manual/next/database/testing.html
+++ b/dev-manual/next/database/testing.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/value-bindings.html
b/dev-manual/next/database/value-bindings.html
index 9adee90a42..b348167598 100644
--- a/dev-manual/next/database/value-bindings.html
+++ b/dev-manual/next/database/value-bindings.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/docker-build.html
b/dev-manual/next/docker-build.html
index 2159efa40e..1fe784e22a 100644
--- a/dev-manual/next/docker-build.html
+++ b/dev-manual/next/docker-build.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/engine-compatibility.html
b/dev-manual/next/engine-compatibility.html
index fb279614d2..8d131a48bf 100644
--- a/dev-manual/next/engine-compatibility.html
+++ b/dev-manual/next/engine-compatibility.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/getting-started.html
b/dev-manual/next/getting-started.html
index a263fa93ee..1f6adbb7a3 100644
--- a/dev-manual/next/getting-started.html
+++ b/dev-manual/next/getting-started.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/gui-plugins-toolbars.html
b/dev-manual/next/gui-plugins-toolbars.html
index d20e10422f..ae2181f609 100644
--- a/dev-manual/next/gui-plugins-toolbars.html
+++ b/dev-manual/next/gui-plugins-toolbars.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/hopweb/developer-guide.html
b/dev-manual/next/hopweb/developer-guide.html
index 550ed4bf89..11dee06341 100644
--- a/dev-manual/next/hopweb/developer-guide.html
+++ b/dev-manual/next/hopweb/developer-guide.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/hopweb/hopweb-antipatterns.html
b/dev-manual/next/hopweb/hopweb-antipatterns.html
index 9428d692d2..43150fc5d7 100644
--- a/dev-manual/next/hopweb/hopweb-antipatterns.html
+++ b/dev-manual/next/hopweb/hopweb-antipatterns.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/hopweb/index.html
b/dev-manual/next/hopweb/index.html
index 53ab8e4cd2..63b9b72d5f 100644
--- a/dev-manual/next/hopweb/index.html
+++ b/dev-manual/next/hopweb/index.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/index.html b/dev-manual/next/index.html
index fd7e4e61fd..305ed45608 100644
--- a/dev-manual/next/index.html
+++ b/dev-manual/next/index.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/integration-testing.html
b/dev-manual/next/integration-testing.html
index a5513d532f..c93cb5bdaa 100644
--- a/dev-manual/next/integration-testing.html
+++ b/dev-manual/next/integration-testing.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/internationalisation.html
b/dev-manual/next/internationalisation.html
index e5a1637f64..3193fb9c09 100644
--- a/dev-manual/next/internationalisation.html
+++ b/dev-manual/next/internationalisation.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/lineage.html b/dev-manual/next/lineage.html
index 0961a7698e..7a9608acda 100644
--- a/dev-manual/next/lineage.html
+++ b/dev-manual/next/lineage.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/marketplace-publish-plugin.html
b/dev-manual/next/marketplace-publish-plugin.html
index 5c4efed702..6320332a2f 100644
--- a/dev-manual/next/marketplace-publish-plugin.html
+++ b/dev-manual/next/marketplace-publish-plugin.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/metadata-injection.html
b/dev-manual/next/metadata-injection.html
index 5e8a1f4d15..40c0df6e07 100644
--- a/dev-manual/next/metadata-injection.html
+++ b/dev-manual/next/metadata-injection.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/metadata-plugins.html
b/dev-manual/next/metadata-plugins.html
index 1290addfa9..dd82f0474a 100644
--- a/dev-manual/next/metadata-plugins.html
+++ b/dev-manual/next/metadata-plugins.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/metadata-serialization.html
b/dev-manual/next/metadata-serialization.html
index d8cf460e42..3476dd6556 100644
--- a/dev-manual/next/metadata-serialization.html
+++ b/dev-manual/next/metadata-serialization.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/plugin-development.html
b/dev-manual/next/plugin-development.html
index e3e8e195c5..1d553e810c 100644
--- a/dev-manual/next/plugin-development.html
+++ b/dev-manual/next/plugin-development.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/plugin-samples.html
b/dev-manual/next/plugin-samples.html
index 6c67310e19..40fdfe1019 100644
--- a/dev-manual/next/plugin-samples.html
+++ b/dev-manual/next/plugin-samples.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/plugin-types/canvas-overlays.html
b/dev-manual/next/plugin-types/canvas-overlays.html
index 9608823be1..5b9cab40d2 100644
--- a/dev-manual/next/plugin-types/canvas-overlays.html
+++ b/dev-manual/next/plugin-types/canvas-overlays.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/database/value-bindings.html
b/dev-manual/next/plugin-types/compression-formats.html
similarity index 80%
copy from dev-manual/next/database/value-bindings.html
copy to dev-manual/next/plugin-types/compression-formats.html
index 9adee90a42..88b884ee2c 100644
--- a/dev-manual/next/database/value-bindings.html
+++ b/dev-manual/next/plugin-types/compression-formats.html
@@ -3,16 +3,16 @@
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
-<title>Moving values across JDBC | Apache Hop</title>
-<meta name="description" content="When a driver needs the values themselves
handled specially, rather than the types named differently.">
+<title>Compression formats | Apache Hop</title>
+<meta name="description" content="How Hop uses Apache Commons Compress as its
compression registry, the built-in format providers, and how to contribute a
new compression format.">
+
-<link rel="canonical"
href="https://hop.apache.org/dev-manual/latest/database/value-bindings.html">
<meta property="og:type" content="website">
<meta property="og:site_name" content="Apache Hop">
-<meta property="og:title" content="Moving values across JDBC">
-<meta property="og:description" content="When a driver needs the values
themselves handled specially, rather than the types named differently.">
-<meta property="og:url"
content="https://hop.apache.org/dev-manual/latest/database/value-bindings.html">
+<meta property="og:title" content="Compression formats">
+<meta property="og:description" content="How Hop uses Apache Commons Compress
as its compression registry, the built-in format providers, and how to
contribute a new compression format.">
+
<meta property="og:image" content="https://hop.apache.org/img/social-card.png">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image"
content="https://hop.apache.org/img/social-card.png">
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item is-current">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
@@ -249,7 +253,7 @@
<a href="../../../dev-manual/next/database/column-types.html">Column types
and type rules</a>
</li>
- <li class="doc-nav__item is-current">
+ <li class="doc-nav__item">
<a href="../../../dev-manual/next/database/value-bindings.html">Moving
values across JDBC</a>
</li>
@@ -516,10 +520,10 @@
<p class="explore__title"></p>
<ul>
<li class="is-current">
- <a
href="../../../dev-manual/next/database/value-bindings.html">2.21.0</a>
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">2.21.0</a>
</li>
- <li>
- <a
href="../../../dev-manual/latest/database/value-bindings.html">2.20.0</a>
+ <li class="is-missing">
+ <a href="../../../dev-manual/latest/index.html" title="This page does
not exist in 2.20.0; opens the start page">2.20.0</a>
</li>
<li class="is-missing">
<a href="../../../dev-manual/2.19.0/index.html" title="This page does
not exist in 2.19.0; opens the start page">2.19.0</a>
@@ -606,7 +610,7 @@
</div>
</details>
- <a class="nav-foot__edit"
href="https://github.com/apache/hop/edit/main/docs/hop-dev-manual/modules/ROOT/pages/database/value-bindings.adoc"
rel="noopener" target="_blank">
+ <a class="nav-foot__edit"
href="https://github.com/apache/hop/edit/main/docs/hop-dev-manual/modules/ROOT/pages/plugin-types/compression-formats.adoc"
rel="noopener" target="_blank">
<svg width="14" height="14" viewBox="0 0 16 16" fill="currentColor"
aria-hidden="true"><path d="M11.013 1.427a1.75 1.75 0 0 1 2.474 0l1.086
1.086a1.75 1.75 0 0 1 0 2.474l-8.61
8.61c-.21.21-.47.364-.756.445l-3.251.93a.75.75 0 0
1-.927-.928l.929-3.25c.081-.286.235-.547.445-.758zm1.414 1.06a.25.25 0 0 0-.354
0L10.811 3.75l1.439 1.44 1.263-1.263a.25.25 0 0 0 0-.354zM11.189 6.25 9.75
4.81l-6.286 6.287a.25.25 0 0 0-.064.108l-.558 1.953 1.953-.558a.25.25 0 0 0
.108-.064z"/></svg>
Edit this page
</a>
@@ -621,159 +625,139 @@
<ol>
<li><a href="../../../">Home</a></li>
<li><span aria-current="page">Plugin development</span></li>
- <li><a href="../../../dev-manual/next/database/index.html">Database
plugins</a></li>
- <li><a href="../../../dev-manual/next/database/value-bindings.html">Moving
values across JDBC</a></li>
+ <li><a href="../../../dev-manual/next/plugin-types/index.html">Plugin
types</a></li>
+ <li><a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a></li>
</ol>
</nav>
<article class="doc">
- <h1>Moving values across JDBC</h1>
+ <h1>Compression formats</h1>
<div id="preamble">
<div class="sectionbody">
<div class="paragraph">
-<p>Most of the time, once Hop knows a column is a date, reading it is
<code>getTimestamp</code> and writing it is <code>setTimestamp</code>.
-Some drivers are not that obliging.</p>
+<p>Compression in Hop is built on top of Apache Commons Compress.
+The commons-compress <code>CompressorStreamFactory</code> is the single
registry for all codecs: it provides the 14 built-in formats (zstd, gzip,
bzip2, xz, lzma, snappy-framed, snappy-raw, lz4-framed, lz4-block, deflate,
deflate64, z, pack200, brotli), and Hop contributes its own <code>none</code>
(passthrough) and <code>zip</code> codecs to the same registry through the
<code>CompressorStreamProvider</code> SPI (see
<code>org.apache.hop.core.compress.spi.HopCompressorStreamProvider</co [...]
</div>
<div class="paragraph">
-<p>A binding is for those cases.
-It is not for naming types differently, which is what <a
href="column-types.html" class="xref page">type rules</a> are for.
-Reach for a binding only when the driver misbehaves over the value itself.</p>
+<p>Formats are consumed everywhere compression is configured: Text File
Output, Text File Input, the Snowflake bulk loader and any component using
<code>CompressionProviderFactory</code>.</p>
</div>
-<div class="paragraph">
-<p>The three in Hop’s own plugins are a fair sample of what
qualifies:</p>
</div>
+</div>
+<div class="sect1">
+<h2 id="_the_lifecycle_of_a_format"><a class="link"
href="#_the_lifecycle_of_a_format">The lifecycle of a format</a></h2>
+<div class="sectionbody">
<div class="ulist">
<ul>
<li>
-<p>the Netezza driver returns nothing usable from <code>getDate</code> on a
<code>TIME</code> column</p>
+<p>The format metadata (name, description, default extension, input/output
support) is declared by the <code>@CompressionPlugin</code> annotation on the
provider class.</p>
+</li>
+<li>
+<p>The provider class is discovered by the plugin machinery
(<code>CompressionPluginType</code> + <code>PluginRegistry</code>, backed by
the Jandex annotation index) — no <code>plugin.xml</code> and no manual
registration list.</p>
</li>
<li>
-<p>the DuckDB driver has never implemented the <code>Calendar</code> overloads
of <code>setDate</code> and <code>setTimestamp</code></p>
+<p>The codec itself is provided by commons-compress, either by a built-in
format or by any <code>CompressorStreamProvider</code> registered with
<code>HopCompressionRegistry</code>.</p>
</li>
<li>
-<p>PostgreSQL takes JSON as a typed object, where other databases reject
<code>Types.OTHER</code></p>
+<p><code>CompressionProviderFactory</code> (the deprecated compatibility
facade) merges the two: it lists every annotated provider whose codec is
available, and resolves formats by name or registry id through
<code>getFormats()</code>/<code>getFormat()</code>.</p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect1">
-<h2 id="_the_interface"><a class="link" href="#_the_interface">The
interface</a></h2>
+<h2 id="_adding_a_new_format"><a class="link"
href="#_adding_a_new_format">Adding a new format</a></h2>
<div class="sectionbody">
-<div class="listingblock">
-<div class="content">
-<div class="code-lang" data-pagefind-ignore>java</div><pre class="highlightjs
highlight is-highlighted"><code class="language-java hljs"
data-lang="java"><span class="hljs-keyword">public</span> <span
class="hljs-keyword">interface</span> <span class="hljs-title
class_">IValueBinding</span> {
- <span class="hljs-type">Object</span> <span class="hljs-title
function_">read</span><span class="hljs-params">(IDatabase database, IValueMeta
valueMeta, ResultSet resultSet, <span class="hljs-type">int</span> index)</span>
- <span class="hljs-keyword">throws</span> SQLException;
-
- <span class="hljs-type">void</span> <span class="hljs-title
function_">write</span><span class="hljs-params">(IDatabase database,
IValueMeta valueMeta, PreparedStatement preparedStatement,
- <span class="hljs-type">int</span> index, Object value)</span> <span
class="hljs-keyword">throws</span> SQLException, HopValueException;
-}</code></pre>
-</div>
-</div>
<div class="paragraph">
-<p>The index is the JDBC one, counting from 1.</p>
+<p>The supported extension path is: a <code>@CompressionPlugin</code> metadata
class (step 1) plus, when the codec is not built into commons-compress, a
<code>CompressorStreamProvider</code> registered with
<code>HopCompressionRegistry</code> (step 2).
+This two-part path gives the format full metadata (display name, description,
extension, input/output support).</p>
</div>
<div class="paragraph">
-<p>A binding that only ever writes can throw
<code>UnsupportedOperationException</code> from <code>read</code>, and the
other way round.
-A binding is asked for in both directions, so that is how it says it does not
serve this one: Hop then falls back to the value type’s own handling,
which is what happened before the binding existed.</p>
-</div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_declaring_one"><a class="link" href="#_declaring_one">Declaring
one</a></h2>
-<div class="sectionbody">
-<div class="listingblock">
-<div class="content">
-<div class="code-lang" data-pagefind-ignore>java</div><pre class="highlightjs
highlight is-highlighted"><code class="language-java hljs"
data-lang="java">DatabaseTypes.rules()
- .bind(IValueMeta.TYPE_DATE, MY_DATE_BINDING)
- .build();</code></pre>
-</div>
+<p>A shortcut exists for codecs that are already built into commons-compress:
the metadata class alone is enough, nothing is registered.
+A fallback exists for codecs that only register a
<code>CompressorStreamProvider</code> without a metadata class: the factory
derives the metadata, but the display name comes from the registry key and
there is no extension, so prefer the metadata class. The two mechanisms are
merged and de-duplicated by registry key.</p>
</div>
+<div class="sect2">
+<h3 id="_declare_the_format_with_a_metadata_class"><a class="link"
href="#_declare_the_format_with_a_metadata_class">Declare the format with a
metadata class</a></h3>
<div class="paragraph">
-<p>Or, when it should only apply in some cases:</p>
+<p>The annotation carries all the metadata; the class extends
<code>CommonsCompressCompressionProvider</code> which implements the stream
delegation:</p>
</div>
<div class="listingblock">
<div class="content">
-<div class="code-lang" data-pagefind-ignore>java</div><pre class="highlightjs
highlight is-highlighted"><code class="language-java hljs"
data-lang="java">DatabaseTypes.rules()
- .bind(
- IValueMeta.TYPE_DATE,
- (database, valueMeta) ->
- valueMeta.getPrecision() == <span class="hljs-number">1</span> ||
!database.isSupportsTimeStampToDateConversion(),
- NETEZZA_DATE_BINDING)
- .build();</code></pre>
+<div class="code-lang" data-pagefind-ignore>java</div><pre class="highlightjs
highlight is-highlighted"><code class="language-java hljs"
data-lang="java"><span class="hljs-keyword">package</span>
org.apache.hop.core.compress.format;
+
+<span class="hljs-keyword">import</span>
org.apache.hop.core.compress.CompressionPlugin;
+<span class="hljs-keyword">import</span>
org.apache.hop.core.compress.CommonsCompressCompressionProvider;
+
+<span class="hljs-meta">@CompressionPlugin(
+ id = "foo", // must match the commons-compress registry key;
matching is case-insensitive
+ name = "Foo",
+ description = "Foo compression",
+ defaultExtension = "foo",
+ supportsInput = true,
+ supportsOutput = true)</span>
+<span class="hljs-keyword">public</span> <span
class="hljs-keyword">class</span> <span class="hljs-title
class_">FooCompressionProvider</span> <span class="hljs-keyword">extends</span>
<span class="hljs-title class_">CommonsCompressCompressionProvider</span>
{}</code></pre>
</div>
</div>
+<div class="paragraph">
+<p>The class must be <code>public</code> and have a no-argument constructor,
because the plugin registry instantiates it through reflection.
+Input-only formats (for example brotli) declare <code>supportsOutput =
false</code>; <code>createOutputStream</code> then throws a clear
<code>IOException</code>.</p>
</div>
</div>
-<div class="sect1">
-<h2 id="_choosing_happens_before_the_work"><a class="link"
href="#_choosing_happens_before_the_work">Choosing happens before the
work</a></h2>
-<div class="sectionbody">
+<div class="sect2">
+<h3 id="_provide_the_codec"><a class="link" href="#_provide_the_codec">Provide
the codec</a></h3>
<div class="paragraph">
-<p>A binding is chosen from the dialect and the value metadata, never from
column metadata.
-By the time rows are moving there is no column metadata left, only the value
meta the row carries.</p>
+<p>If commons-compress already knows the format (built-in codec), nothing else
is needed: <code>HopCompressionRegistry</code> merges the commons-compress
singleton with the registered providers, and the codec is available.</p>
</div>
<div class="paragraph">
-<p>So everything the choice depends on is answered when the binding is
<strong>chosen</strong>: a driver quirk, a connection capability, the
value’s own precision.
-The binding itself only does the work.
-That is why the Netezza example above tests
<code>isSupportsTimeStampToDateConversion</code> in the condition rather than
inside <code>read</code>.</p>
+<p>If the codec comes from an external library, implement
<code>org.apache.commons.compress.compressors.CompressorStreamProvider</code>
and register it:</p>
</div>
+<div class="listingblock">
+<div class="content">
+<div class="code-lang" data-pagefind-ignore>java</div><pre class="highlightjs
highlight is-highlighted"><code class="language-java hljs"
data-lang="java">HopCompressionRegistry.getInstance().register(<span
class="hljs-keyword">new</span> <span class="hljs-title
class_">MyCodecProvider</span>());</code></pre>
</div>
</div>
-<div class="sect1">
-<h2 id="_bindings_and_the_classloader"><a class="link"
href="#_bindings_and_the_classloader">Bindings and the classloader</a></h2>
-<div class="sectionbody">
<div class="paragraph">
-<p>A binding is the one place in the type system that is likely to touch the
JDBC driver’s own classes, and driver classes only resolve in the
classloader that loaded the driver.</p>
+<p>The registration survives regardless of class loader: commons-compress
discovers SPI files through <code>ClassLoader.getSystemClassLoader()</code>,
which is unreliable in embedded or isolated class loader contexts, so Hop
always merges the registered providers explicitly.
+If an extension also ships a
<code>META-INF/services/org.apache.commons.compress.compressors.CompressorStreamProvider</code>
file, both paths merge and the format is registered exactly once.</p>
</div>
-<div class="paragraph">
-<p>If the binding lives in the same plugin as the dialect, that is automatic.
-If it lives somewhere else, the plugin has to join the dialect’s
<code>classLoaderGroup</code>, or unwrapping a driver object fails with the
confusing <code>class com.acme.Thing cannot be cast to class
com.acme.Thing</code>.
-See <a href="extending-another-dialect.html" class="xref page">Extending a
dialect you don’t own</a>.</p>
</div>
+<div class="sect2">
+<h3 id="_use_it"><a class="link" href="#_use_it">Use it</a></h3>
<div class="paragraph">
-<p>Prefer exchanging driver neutral values — byte arrays, strings, numbers —
so that whatever consumes the value needs no knowledge of any driver.</p>
+<p>No further changes are needed.
+The format appears in the Text File Output compression dropdown
(<code>getOutputFormats()</code>, which filters to writable formats) and the
Text File Input dropdown (<code>getInputFormats()</code>).
+It is also resolvable by name or id through the legacy API, e.g.
<code>getFormat("Foo")</code>, <code>createCompressionOutputStream("foo",
out)</code>, so existing pipelines can reference it after a metadata update.</p>
+</div>
</div>
</div>
</div>
<div class="sect1">
-<h2 id="_where_hop_asks"><a class="link" href="#_where_hop_asks">Where Hop
asks</a></h2>
+<h2 id="_notes"><a class="link" href="#_notes">Notes</a></h2>
<div class="sectionbody">
-<div class="paragraph">
-<p>Hop asks for a binding at two places, and only two:</p>
-</div>
<div class="ulist">
<ul>
<li>
-<p>reading, in the dialect’s own <code>getValueFromResultSet</code></p>
+<p>The commons-compress registry keys are normalized to upper case internally;
Hop matches them case-insensitively.</p>
+</li>
+<li>
+<p><code>detectCompressionFormat(InputStream)</code> returns the
commons-compress name (for example <code>gz</code> or <code>zstd</code>)
detected from magic bytes, or <code>null</code> for unknown input.</p>
</li>
<li>
-<p>writing, in <code>Database.setValue</code></p>
+<p>The legacy formats <code>Snappy</code> (xerial raw framing) and
<code>Hadoop-snappy</code> (reflection based Hadoop shim) are kept for backward
compatibility because their wire format differs from the commons-compress
snappy codecs.
+Their codec is loaded from the classpath, so they remain available in the
format lists and dialogs even though commons-compress does not ship an
equivalent codec.</p>
+</li>
+<li>
+<p>A codec registered with <code>HopCompressionRegistry</code> without an
annotation class is still usable through the factory: <code>getFormat</code>
derives its metadata and stream creation delegates to the registry.</p>
+</li>
+<li>
+<p>Formats that are not commons-compress backed (legacy codecs and external
providers) are considered always available; only the commons-compress backed
formats are filtered by codec presence.</p>
</li>
</ul>
</div>
-<div class="paragraph">
-<p>Both are the points the engine goes through, which is deliberate.
-A value type that handles its own reading or writing still gets its
database’s binding, because the question is asked before the value type
is reached.</p>
-</div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_cost"><a class="link" href="#_cost">Cost</a></h2>
-<div class="sectionbody">
-<div class="paragraph">
-<p>Binding lookup runs for every value of every row.
-The rules that can supply one are collected once per dialect and cached, and
nearly every dialect has none, so the lookup is a map read and an empty list
check.</p>
-</div>
-<div class="paragraph">
-<p>A rule that can supply a binding must say so by returning true from
<code>suppliesBindings()</code>; the builder does this for you.
-A rule that never binds is skipped entirely.</p>
-</div>
</div>
</div>
</article>
<nav class="pagination" data-pagefind-ignore aria-label="Page">
- <a class="pagination__prev"
href="../../../dev-manual/next/database/column-types.html"><span>Previous</span><strong>Column
types and type rules</strong></a>
- <a class="pagination__next"
href="../../../dev-manual/next/database/extending-another-dialect.html"><span>Next</span><strong>Extending
a dialect you don’t own</strong></a>
+ <a class="pagination__prev"
href="../../../dev-manual/next/plugin-types/gui.html"><span>Previous</span><strong>GUI</strong></a>
+ <a class="pagination__next"
href="../../../dev-manual/next/plugin-types/diagram-exporters.html"><span>Next</span><strong>Diagram
exporters</strong></a>
</nav>
</main>
diff --git a/dev-manual/next/plugin-types/data-connectivity.html
b/dev-manual/next/plugin-types/data-connectivity.html
index 637756fdcb..fa1eb2e1ae 100644
--- a/dev-manual/next/plugin-types/data-connectivity.html
+++ b/dev-manual/next/plugin-types/data-connectivity.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/plugin-types/diagram-exporters.html
b/dev-manual/next/plugin-types/diagram-exporters.html
index ab8281ec4d..c234f90b04 100644
--- a/dev-manual/next/plugin-types/diagram-exporters.html
+++ b/dev-manual/next/plugin-types/diagram-exporters.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item is-current">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
@@ -1116,7 +1120,7 @@ Keep painters that need SWT behind an engine helper the
way <code>PipelineSvgPai
</div>
</article>
<nav class="pagination" data-pagefind-ignore aria-label="Page">
- <a class="pagination__prev"
href="../../../dev-manual/next/plugin-types/gui.html"><span>Previous</span><strong>GUI</strong></a>
+ <a class="pagination__prev"
href="../../../dev-manual/next/plugin-types/compression-formats.html"><span>Previous</span><strong>Compression
formats</strong></a>
<a class="pagination__next"
href="../../../dev-manual/next/ai-advisor-plugins.html"><span>Next</span><strong>AI
advisor plugins</strong></a>
</nav>
</main>
diff --git a/dev-manual/next/plugin-types/execution-observability.html
b/dev-manual/next/plugin-types/execution-observability.html
index c97a1d88e3..f93192616f 100644
--- a/dev-manual/next/plugin-types/execution-observability.html
+++ b/dev-manual/next/plugin-types/execution-observability.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/plugin-types/gui.html
b/dev-manual/next/plugin-types/gui.html
index b5e558fe89..0001dd4790 100644
--- a/dev-manual/next/plugin-types/gui.html
+++ b/dev-manual/next/plugin-types/gui.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item is-current">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
@@ -977,7 +981,7 @@ The GUI discovers every exporter that supports the active
editor’s <code>g
</article>
<nav class="pagination" data-pagefind-ignore aria-label="Page">
<a class="pagination__prev"
href="../../../dev-manual/next/plugin-types/execution-observability.html"><span>Previous</span><strong>Execution
and observability</strong></a>
- <a class="pagination__next"
href="../../../dev-manual/next/plugin-types/diagram-exporters.html"><span>Next</span><strong>Diagram
exporters</strong></a>
+ <a class="pagination__next"
href="../../../dev-manual/next/plugin-types/compression-formats.html"><span>Next</span><strong>Compression
formats</strong></a>
</nav>
</main>
diff --git a/dev-manual/next/plugin-types/index.html
b/dev-manual/next/plugin-types/index.html
index d18aba3679..fd41d3e56c 100644
--- a/dev-manual/next/plugin-types/index.html
+++ b/dev-manual/next/plugin-types/index.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/plugin-types/metadata-configuration.html
b/dev-manual/next/plugin-types/metadata-configuration.html
index f4f47e0827..83fcbc013a 100644
--- a/dev-manual/next/plugin-types/metadata-configuration.html
+++ b/dev-manual/next/plugin-types/metadata-configuration.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/plugin-types/pipeline-workflow.html
b/dev-manual/next/plugin-types/pipeline-workflow.html
index ac378659e4..c3816f32bc 100644
--- a/dev-manual/next/plugin-types/pipeline-workflow.html
+++ b/dev-manual/next/plugin-types/pipeline-workflow.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/porting-kettle-plugins.html
b/dev-manual/next/porting-kettle-plugins.html
index 138c49efdc..7d87cec68e 100644
--- a/dev-manual/next/porting-kettle-plugins.html
+++ b/dev-manual/next/porting-kettle-plugins.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/hop-sdk.html b/dev-manual/next/sdk/hop-sdk.html
index aa6c86bd0a..b32a7d39fb 100644
--- a/dev-manual/next/sdk/hop-sdk.html
+++ b/dev-manual/next/sdk/hop-sdk.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/index.html b/dev-manual/next/sdk/index.html
index 47df0e21de..1df4a78708 100644
--- a/dev-manual/next/sdk/index.html
+++ b/dev-manual/next/sdk/index.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/logging.html b/dev-manual/next/sdk/logging.html
index 75f7457213..affe78bb85 100644
--- a/dev-manual/next/sdk/logging.html
+++ b/dev-manual/next/sdk/logging.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/metadata.html
b/dev-manual/next/sdk/metadata.html
index 0fe902c7aa..a64abfecc7 100644
--- a/dev-manual/next/sdk/metadata.html
+++ b/dev-manual/next/sdk/metadata.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/pipelines.html
b/dev-manual/next/sdk/pipelines.html
index 951054d787..4fe35be25c 100644
--- a/dev-manual/next/sdk/pipelines.html
+++ b/dev-manual/next/sdk/pipelines.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/actions.html
b/dev-manual/next/sdk/plugins/actions.html
index 5807a5b075..db2cdf1987 100644
--- a/dev-manual/next/sdk/plugins/actions.html
+++ b/dev-manual/next/sdk/plugins/actions.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/commands.html
b/dev-manual/next/sdk/plugins/commands.html
index 3f3184dcd7..68a93cb9e3 100644
--- a/dev-manual/next/sdk/plugins/commands.html
+++ b/dev-manual/next/sdk/plugins/commands.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/configuration.html
b/dev-manual/next/sdk/plugins/configuration.html
index 827d6348e4..1326b7042a 100644
--- a/dev-manual/next/sdk/plugins/configuration.html
+++ b/dev-manual/next/sdk/plugins/configuration.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/databases.html
b/dev-manual/next/sdk/plugins/databases.html
index 9f1c3d34a4..e340a9b7e3 100644
--- a/dev-manual/next/sdk/plugins/databases.html
+++ b/dev-manual/next/sdk/plugins/databases.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/execution-info.html
b/dev-manual/next/sdk/plugins/execution-info.html
index ad2b2ad32b..1561d6152a 100644
--- a/dev-manual/next/sdk/plugins/execution-info.html
+++ b/dev-manual/next/sdk/plugins/execution-info.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/extension-points.html
b/dev-manual/next/sdk/plugins/extension-points.html
index 8b3299d556..365da9caed 100644
--- a/dev-manual/next/sdk/plugins/extension-points.html
+++ b/dev-manual/next/sdk/plugins/extension-points.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/file-types.html
b/dev-manual/next/sdk/plugins/file-types.html
index 24a8b2b97e..8983254841 100644
--- a/dev-manual/next/sdk/plugins/file-types.html
+++ b/dev-manual/next/sdk/plugins/file-types.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/gui.html
b/dev-manual/next/sdk/plugins/gui.html
index 38746a5a8b..dcf084a9b0 100644
--- a/dev-manual/next/sdk/plugins/gui.html
+++ b/dev-manual/next/sdk/plugins/gui.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/index.html
b/dev-manual/next/sdk/plugins/index.html
index 70109e9ea8..348768750d 100644
--- a/dev-manual/next/sdk/plugins/index.html
+++ b/dev-manual/next/sdk/plugins/index.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/metadata.html
b/dev-manual/next/sdk/plugins/metadata.html
index 3a650eb612..a58cbb5091 100644
--- a/dev-manual/next/sdk/plugins/metadata.html
+++ b/dev-manual/next/sdk/plugins/metadata.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/perspectives.html
b/dev-manual/next/sdk/plugins/perspectives.html
index beb4dee8cd..75e1b03eb5 100644
--- a/dev-manual/next/sdk/plugins/perspectives.html
+++ b/dev-manual/next/sdk/plugins/perspectives.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/servlets.html
b/dev-manual/next/sdk/plugins/servlets.html
index 396e46ae1a..0399cf4a1c 100644
--- a/dev-manual/next/sdk/plugins/servlets.html
+++ b/dev-manual/next/sdk/plugins/servlets.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/transforms.html
b/dev-manual/next/sdk/plugins/transforms.html
index e0df6e5831..44c423e378 100644
--- a/dev-manual/next/sdk/plugins/transforms.html
+++ b/dev-manual/next/sdk/plugins/transforms.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/value-types.html
b/dev-manual/next/sdk/plugins/value-types.html
index e38f61b41e..b3eb8d5722 100644
--- a/dev-manual/next/sdk/plugins/value-types.html
+++ b/dev-manual/next/sdk/plugins/value-types.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/variable-resolvers.html
b/dev-manual/next/sdk/plugins/variable-resolvers.html
index aa5afd6354..4df40bfb10 100644
--- a/dev-manual/next/sdk/plugins/variable-resolvers.html
+++ b/dev-manual/next/sdk/plugins/variable-resolvers.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/plugins/vfs.html
b/dev-manual/next/sdk/plugins/vfs.html
index a11e26a35b..2785208ed1 100644
--- a/dev-manual/next/sdk/plugins/vfs.html
+++ b/dev-manual/next/sdk/plugins/vfs.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/vfs.html b/dev-manual/next/sdk/vfs.html
index b6980f507b..9022195aa3 100644
--- a/dev-manual/next/sdk/vfs.html
+++ b/dev-manual/next/sdk/vfs.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/sdk/workflows.html
b/dev-manual/next/sdk/workflows.html
index a94f3190e5..c34377afb9 100644
--- a/dev-manual/next/sdk/workflows.html
+++ b/dev-manual/next/sdk/workflows.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/setup-dev-environment.html
b/dev-manual/next/setup-dev-environment.html
index 08a6e1cd81..4e76a68871 100644
--- a/dev-manual/next/setup-dev-environment.html
+++ b/dev-manual/next/setup-dev-environment.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/snippets/hop-license-header-file.html
b/dev-manual/next/snippets/hop-license-header-file.html
index b34f06bbb6..0f072823e2 100644
--- a/dev-manual/next/snippets/hop-license-header-file.html
+++ b/dev-manual/next/snippets/hop-license-header-file.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/start-your-own-plugin.html
b/dev-manual/next/start-your-own-plugin.html
index 88beafd49e..260d7e33a7 100644
--- a/dev-manual/next/start-your-own-plugin.html
+++ b/dev-manual/next/start-your-own-plugin.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/svg-files.html b/dev-manual/next/svg-files.html
index 3ab8c361e0..ee114f4139 100644
--- a/dev-manual/next/svg-files.html
+++ b/dev-manual/next/svg-files.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/testing/ui-testing.html
b/dev-manual/next/testing/ui-testing.html
index 68c19a99f9..52670329c6 100644
--- a/dev-manual/next/testing/ui-testing.html
+++ b/dev-manual/next/testing/ui-testing.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/testing/unit-testing.html
b/dev-manual/next/testing/unit-testing.html
index b367966c1d..f75be5b1cd 100644
--- a/dev-manual/next/testing/unit-testing.html
+++ b/dev-manual/next/testing/unit-testing.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/value-types.html b/dev-manual/next/value-types.html
index a9af95db5e..7cb06eda2e 100644
--- a/dev-manual/next/value-types.html
+++ b/dev-manual/next/value-types.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/dev-manual/next/writing-documentation.html
b/dev-manual/next/writing-documentation.html
index 72274e3023..97639ecdd2 100644
--- a/dev-manual/next/writing-documentation.html
+++ b/dev-manual/next/writing-documentation.html
@@ -207,6 +207,10 @@
<li class="doc-nav__item">
<a href="../../dev-manual/next/plugin-types/gui.html">GUI</a>
+ </li>
+ <li class="doc-nav__item">
+ <a
href="../../dev-manual/next/plugin-types/compression-formats.html">Compression
formats</a>
+
</li>
<li class="doc-nav__item">
<a
href="../../dev-manual/next/plugin-types/diagram-exporters.html">Diagram
exporters</a>
diff --git a/manual/next/_images/hop-gui/pipeline/debug-partial-run.jpg
b/manual/next/_images/hop-gui/pipeline/debug-partial-run.jpg
new file mode 100644
index 0000000000..da3c3fd102
Binary files /dev/null and
b/manual/next/_images/hop-gui/pipeline/debug-partial-run.jpg differ
diff --git a/manual/next/_images/hop-gui/pipeline/debug-pause-condition.jpg
b/manual/next/_images/hop-gui/pipeline/debug-pause-condition.jpg
new file mode 100644
index 0000000000..ea7fd34dd1
Binary files /dev/null and
b/manual/next/_images/hop-gui/pipeline/debug-pause-condition.jpg differ
diff --git a/manual/next/_images/hop-gui/pipeline/debug-session-started.jpg
b/manual/next/_images/hop-gui/pipeline/debug-session-started.jpg
new file mode 100644
index 0000000000..2dfd71f46f
Binary files /dev/null and
b/manual/next/_images/hop-gui/pipeline/debug-session-started.jpg differ
diff --git a/manual/next/_images/hop-gui/pipeline/debug-single-step.jpg
b/manual/next/_images/hop-gui/pipeline/debug-single-step.jpg
new file mode 100644
index 0000000000..8b9e6a088c
Binary files /dev/null and
b/manual/next/_images/hop-gui/pipeline/debug-single-step.jpg differ
diff --git a/manual/next/docker-container.html
b/manual/next/docker-container.html
index 38efd4be6e..b31ba18921 100644
--- a/manual/next/docker-container.html
+++ b/manual/next/docker-container.html
@@ -3244,6 +3244,11 @@ You can specify them as a comma separated list, e.g.
<code><code>PARAM_1=aaa,PAR
<td class="tableblock halign-left valign-top"><p class="tableblock">No</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">You can
specify a metadata export file here which will contain all required metadata
elements. See also the metadata export option in the Hop Conf tool.</p></td>
</tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p
class="tableblock"><code><code>HOP_RUN_SHUTDOWN_TIMEOUT</code></code></p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">No</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">The
maximum number of seconds to wait for the pipeline or workflow to stop when the
container is stopped (<code>docker stop</code>, a Kubernetes pod eviction,
…​). The default is 5 seconds. Keep it below the grace period of
the platform, see <a href="hop-run/index.html#_stopping_hop_run" class="xref
page">Stopping Hop Run</a>.</p></td>
+</tr>
</tbody>
</table>
<div class="paragraph">
diff --git a/manual/next/hop-gui/perspective-configuration.html
b/manual/next/hop-gui/perspective-configuration.html
index 70e4c2f23e..1183a87af0 100644
--- a/manual/next/hop-gui/perspective-configuration.html
+++ b/manual/next/hop-gui/perspective-configuration.html
@@ -4053,6 +4053,33 @@ Because this setting applies to the whole running
process, in Hop Web it belongs
</table>
</div>
<div class="sect3">
+<h4 id="_debug"><a class="link" href="#_debug">Debug</a></h4>
+<div class="paragraph">
+<p>These options are stored under <code>debug-sessions</code> in
<code>hop-config.json</code>.</p>
+</div>
+<table class="tableblock frame-all grid-all stretch">
+<colgroup>
+<col style="width: 25%;">
+<col style="width: 62.5%;">
+<col style="width: 12.5%;">
+</colgroup>
+<thead>
+<tr>
+<th class="tableblock halign-left valign-top">Option</th>
+<th class="tableblock halign-left valign-top">Description</th>
+<th class="tableblock halign-left valign-top">Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Debug
sessions folder</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Where
saved pipeline debug sessions are written. Leave this empty to use
.local/share/hop/debug-sessions under your home directory. Each session folder
contains row data. Accepts variables.</p></td>
+<td class="tableblock halign-left valign-top"></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="sect3">
<h4 id="_dropbox"><a class="link" href="#_dropbox">Dropbox</a></h4>
<div class="paragraph">
<p>These options are stored under <code>dropbox</code> in
<code>hop-config.json</code>.</p>
diff --git a/manual/next/hop-run/index.html b/manual/next/hop-run/index.html
index 2a32108dc3..06ef5f3da0 100644
--- a/manual/next/hop-run/index.html
+++ b/manual/next/hop-run/index.html
@@ -3024,6 +3024,9 @@
The name of the Run Configuration to use
-s,
--system-properties=<systemProperties>[,<systemProperties>...]
A comma separated list of KEY=VALUE pairs
+ -swt, --shutdown-timeout=<shutdownTimeout>
+ The maximum number of seconds to <span
class="hljs-built_in">wait</span> <span class="hljs-keyword">for</span> the
pipeline or workflow to stop
+ when hop-run receives a termination signal (default
5)
-v, --version Print version information and <span
class="hljs-built_in">exit</span></code></pre>
</div>
</div>
@@ -3130,6 +3133,11 @@ Check the <a
href="../pipeline/pipeline-run-configurations/pipeline-run-configur
<td class="tableblock halign-left valign-top"><p class="tableblock">A comma
separated list of KEY=VALUE pairs</p></td>
</tr>
<tr>
+<td class="tableblock halign-left valign-top"><p
class="tableblock"><code><code>-swt</code></code></p></td>
+<td class="tableblock halign-left valign-top"><p
class="tableblock"><code><code>--shutdown-timeout</code></code></p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">The
maximum number of seconds to wait for the pipeline or workflow to stop when Hop
Run receives a termination signal. The default is 5 seconds. Can also be set
with the <code>HOP_RUN_SHUTDOWN_TIMEOUT</code> environment variable. See <a
href="#_stopping_hop_run">Stopping Hop Run</a>.</p></td>
+</tr>
+<tr>
<td class="tableblock halign-left valign-top"><p
class="tableblock"><code><code>-ime</code></code></p></td>
<td class="tableblock halign-left valign-top"><p
class="tableblock"><code><code>--in-memory</code></code></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Keep Hop
configuration in memory. Changes are not written to
<code>hop-config.json</code>. Implied by <code>--project-locations</code> and
<code>--environments</code>.</p></td>
@@ -3201,11 +3209,41 @@ Check the <a
href="../pipeline/pipeline-run-configurations/pipeline-run-configur
<td class="tableblock halign-left valign-top"><p
class="tableblock">None</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">There was
an error parsing the provided parameter information. Hop Run shows the syntax
summary and exists.</p></td>
</tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p
class="tableblock">143</p></td>
+<td class="tableblock halign-left valign-top"><p
class="tableblock">None</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Hop Run
received SIGTERM and stopped the run, see <a href="#_stopping_hop_run">Stopping
Hop Run</a>. A process that receives SIGINT (Ctrl-C) or SIGHUP ends with 130 or
129.</p></td>
+</tr>
</tbody>
</table>
</div>
</div>
<div class="sect1">
+<h2 id="_stopping_hop_run"><a class="link" href="#_stopping_hop_run">Stopping
Hop Run</a></h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>When Hop Run receives a termination signal (<code>SIGTERM</code>,
<code>SIGINT</code> or <code>SIGHUP</code>) it stops the running pipeline or
workflow the same way the Stop button in Hop GUI does, and waits for the run to
end before the process exits.
+This happens when you press Ctrl-C, run <code>docker stop</code>, or when
Kubernetes or OpenShift evicts a pod, for example while it drains a node.
+Transforms and actions stop and release their resources (database connections,
open files, …​), and the log shows that the run was stopped.</p>
+</div>
+<div class="paragraph">
+<p>Hop Run waits at most the number of seconds set with
<code>--shutdown-timeout</code> (or the <code>HOP_RUN_SHUTDOWN_TIMEOUT</code>
environment variable), 5 seconds by default, and then exits anyway.
+When <a href="../technology/openlineage/index.html" class="xref
page">lineage</a> is enabled, Hop Run then delivers the lineage events of the
run, including its end, before it exits.</p>
+</div>
+<div class="paragraph">
+<p>Keep the timeout below the time the platform gives a process before it
kills it: 30 seconds by default in Kubernetes
(<code>terminationGracePeriodSeconds</code>) and 10 seconds for <code>docker
stop</code> (<code>docker stop --time</code>). Leave room for the lineage
delivery as well.
+To give a run more time to stop, raise both: for example
<code>HOP_RUN_SHUTDOWN_TIMEOUT=60</code> with <code>docker run --stop-timeout
90</code> (<code>stop_grace_period</code> in Docker Compose) or
<code>terminationGracePeriodSeconds: 90</code>.</p>
+</div>
+<div class="paragraph">
+<p><code>SIGKILL</code> (<code>kill -9</code>, or the platform giving up)
cannot be handled: the process ends at once, in the middle of whatever it was
doing.</p>
+</div>
+<div class="paragraph">
+<p>Stopping a run does not undo the work it did so far.
+Design pipelines and workflows that may be stopped so that they can be
restarted.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
<h2 id="_examples"><a class="link" href="#_examples">Examples</a></h2>
<div class="sectionbody">
<div class="paragraph">
@@ -3302,6 +3340,75 @@ Check the <a
href="../pipeline/pipeline-run-configurations/pipeline-run-configur
<p>See <a
href="../projects/advanced.html#_use_a_project_without_hop_config_json"
class="xref page">Use a project without hop-config.json</a> for the location
and environment formats.</p>
</div>
<div class="sect2">
+<h3 id="_running_a_pipeline_with_a_parameter"><a class="link"
href="#_running_a_pipeline_with_a_parameter">Running a pipeline with a
parameter</a></h3>
+<div class="paragraph">
+<p>The <code>pipelines/pipeline-with-parameter.hpl</code> pipeline in the
samples project has a parameter <code>PRM_EXAMPLE</code> without a default
value.
+It writes the value it received to the log, followed by the value of the
variable <code>ENV_VARIABLE</code>.</p>
+</div>
+<div class="paragraph">
+<p>Give the parameter a value with <code>-p</code> (<code>--parameters</code>).
+The single quotes keep the shell from replacing <code>${PROJECT_HOME}</code>,
so Hop gets it as it is.
+On Windows, use <code>hop-run.bat</code> instead of <code>./hop-run.sh</code>
and leave out the single quotes: CMD doesn’t remove them.</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre>./hop-run.sh -j samples -r local -f
'${PROJECT_HOME}/pipelines/pipeline-with-parameter.hpl' -p
PRM_EXAMPLE=Hello</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>The log includes these lines, with the value of the parameter:</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre>2026/11/02 12:57:45 - HopRun - Enabling project 'samples'
+2026/11/02 12:57:45 - HopRun - Starting pipeline:
config/projects/samples/pipelines/pipeline-with-parameter.hpl
+2026/11/02 12:57:45 - pipeline-with-parameter - Executing this pipeline using
the Local Pipeline Engine with run configuration 'local'
+2026/11/02 12:57:45 - pipeline-with-parameter - Execution started for pipeline
[pipeline-with-parameter]
+2026/11/02 12:57:45 - get ${PRM_EXAMPLE}.0 - Finished processing (I=0, O=0,
R=1, W=1, U=0, E=0)
+2026/11/02 12:57:45 - write parameter to log.0 -
+2026/11/02 12:57:45 - write parameter to log.0 - ------------> Linenr
1------------------------------
+2026/11/02 12:57:45 - write parameter to log.0 - we received 'Hello' as the
value for this pipeline's parameter.
+2026/11/02 12:57:45 - write parameter to log.0 -
+2026/11/02 12:57:45 - write parameter to log.0 - example = Hello
+2026/11/02 12:57:45 - write parameter to log.0 -
+2026/11/02 12:57:45 - write parameter to log.0 - ====================
+2026/11/02 12:57:45 - write env_variable to log.0 -
+2026/11/02 12:57:45 - write env_variable to log.0 - ------------> Linenr
1------------------------------
+2026/11/02 12:57:45 - write env_variable to log.0 - the value for variable
ENV_VARIABLE is ${ENV_VARIABLE}
+2026/11/02 12:57:45 - write env_variable to log.0 -
+2026/11/02 12:57:45 - write env_variable to log.0 - env_variable =
${ENV_VARIABLE}
+2026/11/02 12:57:45 - write env_variable to log.0 -
+2026/11/02 12:57:45 - write env_variable to log.0 - ====================
+2026/11/02 12:57:45 - write parameter to log.0 - Finished processing (I=0,
O=0, R=1, W=1, U=0, E=0)
+2026/11/02 12:57:45 - get ${ENV_VARIABLE}.0 - Finished processing (I=0, O=0,
R=1, W=1, U=0, E=0)
+2026/11/02 12:57:45 - write env_variable to log.0 - Finished processing (I=0,
O=0, R=1, W=1, U=0, E=0)
+2026/11/02 12:57:45 - pipeline-with-parameter - Pipeline duration : 0.039
seconds [ 0.039" ]</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Without <code>-p</code>, the parameter gets its default value, which is
empty here: the log shows <code>we received '' as the value for this
pipeline’s parameter.</code></p>
+</div>
+<div class="paragraph">
+<p><code>ENV_VARIABLE</code> isn’t set in the samples project, so the
log shows <code>${ENV_VARIABLE}</code> as it is.
+Set it, for example, as a system property with <code>-s</code>, or in an <a
href="../projects/projects-environments.html" class="xref page">environment</a>.
+A value with spaces goes between quotes (double quotes on Windows):</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre>./hop-run.sh -j samples -r local -f
'${PROJECT_HOME}/pipelines/pipeline-with-parameter.hpl' -p 'PRM_EXAMPLE=Hello
Hop' -s ENV_VARIABLE=dev</pre>
+</div>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre>... - write parameter to log.0 - we received 'Hello Hop' as the value for
this pipeline's parameter.
+... - write env_variable to log.0 - the value for variable ENV_VARIABLE is
dev</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>See <a href="../variables/parameters.html" class="xref page">Working with
parameters</a> for more on parameters.</p>
+</div>
+</div>
+<div class="sect2">
<h3 id="_parameter_examples"><a class="link"
href="#_parameter_examples">Parameter Examples</a></h3>
<div class="paragraph">
<p>This is a list of examples on how the parameters on this command are
parsed</p>
diff --git
a/manual/next/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.html
b/manual/next/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.html
index 02dfebbaea..0aae40f6c7 100644
---
a/manual/next/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.html
+++
b/manual/next/pipeline/pipeline-run-configurations/beam-flink-pipeline-engine.html
@@ -2971,7 +2971,7 @@
<h2 id="_beam_flink"><a class="link" href="#_beam_flink">Beam Flink</a></h2>
<div class="sectionbody">
<div class="paragraph">
-<p>This runner allows you to run Hop pipelines on <a
href="https://flink.apache.org" rel="noopener" target="_blank">Apache Flink</a>
version 1.13.</p>
+<p>This runner allows you to run Hop pipelines on <a
href="https://flink.apache.org" rel="noopener" target="_blank">Apache Flink</a>
version 1.19. See <a href="../beam/getting-started-with-beam.html" class="xref
page">Getting started with Apache Beam</a> for the Beam and Flink versions of
each Hop release.</p>
</div>
<div class="paragraph">
<p>The Flink runner supports two modes: Local Direct Flink Runner and Flink
Runner.</p>
@@ -3069,7 +3069,7 @@ A value of -1 indicates that the system default value (as
defined in the configu
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Execution
retry delay (ms)</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Sets the
delay in milliseconds between executions.
-A value of {@code -1} indicates that the default value should be used.</p></td>
+A value of -1 indicates that the default value should be used.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td>
</tr>
<tr>
@@ -3121,7 +3121,7 @@ Interval value ⇐ 0 disables the feature.</p></td>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Batch
execution mode</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Flink mode
for data exchange of batch pipelines.
-Reference {@link org.apache.flink.api.common.ExecutionMode}.
+The values are those of Flink’s
<code>org.apache.flink.api.common.ExecutionMode</code>.
Set this to BATCH_FORCED if pipelines get blocked, see <a
href="https://issues.apache.org/jira/browse/FLINK-10672" rel="noopener"
target="_blank">FLINK-10672</a></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">P</p></td>
</tr>
@@ -3171,10 +3171,11 @@ Set this to BATCH_FORCED if pipelines get blocked, see
<a href="https://issues.a
</div>
</div>
<div class="sect1">
-<h2 id="_job_id"><a class="link" href="#_job_id">Job id</a></h2>
+<h2 id="_flink_job_id"><a class="link" href="#_flink_job_id">Flink job
ID</a></h2>
<div class="sectionbody">
<div class="paragraph">
-<p>Beam does not return the Flink job id on the pipeline result. Hop chooses
the id before submitting the job. After <code>prepareExecution()</code> it is
available from the engine:</p>
+<p>Beam doesn’t return the Flink job ID on the pipeline result, so Hop
chooses the ID itself before it submits the job, when the Flink master is
<code>[local]</code> or <code>host:port</code>.
+After <code>prepareExecution()</code> it is available from the engine:</p>
</div>
<div class="listingblock">
<div class="content">
@@ -3184,7 +3185,12 @@ Set this to BATCH_FORCED if pipelines get blocked, see
<a href="https://issues.a
</div>
</div>
<div class="paragraph">
-<p>The same value is written to the pipeline log. When the run configuration
has an execution information location, it is stored as
<code>flink.job.id</code> and shown on the execution viewer info tab. A master
of <code>[collection]</code> does not submit a cluster job, so no id is
assigned.</p>
+<p>The same value is written to the pipeline log. When the run configuration
has an execution information location, it is stored as
<code>flink.job.id</code> and shown on the execution viewer info tab.</p>
+</div>
+<div class="paragraph">
+<p>With a master of <code>[auto]</code>, which is also used when the master is
empty, Flink takes its environment from <code>flink run</code> or the
Kubernetes operator and ignores the ID Hop would set, and a master of
<code>[collection]</code> doesn’t submit a cluster job: in those cases
Hop has no Flink job ID.
+Use the Beam job name instead: Hop derives it from the pipeline name, and you
can find the job by that name in the Flink UI or with the Flink REST API.
+The Dataflow runner is different: Hop reports that job ID through the
execution detail <code>dataflow.job.id</code>.</p>
</div>
</div>
</div>
@@ -3284,17 +3290,6 @@ PROJECT_HOME is as good as any variable to use.</p>
It’s primarily used for testing locally.</p>
</div>
</div>
-</div>
-<div class="sect1">
-<h2 id="_flink_job_id"><a class="link" href="#_flink_job_id">Flink job
ID</a></h2>
-<div class="sectionbody">
-<div class="paragraph">
-<p>Hop does not show a Flink job ID after a Beam run. Beam 2.74 keeps the
attached result as accumulators and a runtime, and the detached result keeps
its job client in a private field with no accessor. There is no supported way
to read the job ID from <code>PipelineResult</code>.</p>
-</div>
-<div class="paragraph">
-<p>The Beam job name is still available. Hop derives it from the pipeline
name. On a Flink cluster, use that name in the Flink UI or the Flink REST API
to find the job. The Dataflow runner is different: Hop reports that job ID
through the execution detail <code>dataflow.job.id</code>.</p>
-</div>
-</div>
</div>
</article>
<nav class="pagination" data-pagefind-ignore aria-label="Page">
diff --git
a/manual/next/pipeline/pipeline-run-configurations/native-local-pipeline-engine.html
b/manual/next/pipeline/pipeline-run-configurations/native-local-pipeline-engine.html
index 1f504d5ec3..21b630557e 100644
---
a/manual/next/pipeline/pipeline-run-configurations/native-local-pipeline-engine.html
+++
b/manual/next/pipeline/pipeline-run-configurations/native-local-pipeline-engine.html
@@ -3034,6 +3034,19 @@ Also be aware that while you <strong>can</strong> commit
and rollback across mul
This is not a two-phase-commit system.</p></td>
<td class="tableblock halign-left valign-top"><p
class="tableblock">false</p></td>
</tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Lock-free
row sets (experimental)</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">An
experimental performance option.
+Rows are passed between transforms through lock-free buffers instead of
lock-based ones.
+The transform that reads a buffer is woken once a batch of rows is ready,
instead of for every row.
+A transform with several inputs waits on all of them at once and is woken by
whichever input has rows.
+A transform that has had no rows for a while waits longer between checks, up
to 20 ms, and is then woken by the next row, so idle transforms in long-running
pipelines use little CPU.
+This lowers the cost per row, which matters most in pipelines of fast
transforms.
+A buffer holds at most the configured <strong>Row set size</strong>, as the
lock-based buffers do, and the row counts of the transforms are exact.
+With this option the <strong>Wait time buffer check (ms)</strong> setting is
not used: transforms no longer poll their inputs.
+Hops that spill to disk to avoid a buffer deadlock keep their spilling
buffer.</p></td>
+<td class="tableblock halign-left valign-top"><p
class="tableblock">false</p></td>
+</tr>
</tbody>
</table>
</div>
diff --git
a/manual/next/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.html
b/manual/next/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.html
index f7411ea533..1898e88f38 100644
---
a/manual/next/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.html
+++
b/manual/next/pipeline/pipeline-run-configurations/native-spark-pipeline-engine.html
@@ -3057,7 +3057,7 @@ It builds a Spark SQL Dataset graph from the pipeline and
runs transforms via na
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Spark
master</p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">Spark
master URL: <code>local[<strong>]</code>, <code>local[4]</code>,
<code>spark://host:7077</code> or <code>yarn</code>. Leave empty under
<code>spark-submit</code> so the CLI master is used; outside
<code>spark-submit</code> an empty master runs on
<code>local[</strong>]</code>.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Spark
master URL: <code>local[*]</code>, <code>local[4]</code>,
<code>spark://host:7077</code> or <code>yarn</code>. Leave empty under
<code>spark-submit</code> so the CLI master is used; outside
<code>spark-submit</code> an empty master runs on
<code>local[*]</code>.</p></td>
<td class="tableblock halign-left valign-top"><p
class="tableblock"><code>local[*]</code></p></td>
</tr>
<tr>
diff --git a/manual/next/pipeline/run-preview-debug-pipeline.html
b/manual/next/pipeline/run-preview-debug-pipeline.html
index ce6cd45d8c..b5973d8391 100644
--- a/manual/next/pipeline/run-preview-debug-pipeline.html
+++ b/manual/next/pipeline/run-preview-debug-pipeline.html
@@ -3241,6 +3241,134 @@ To change the pipeline run configuration, click
<strong>Configure</strong>.</p>
</div>
</div>
<div class="sect1">
+<h2 id="_debug_a_pipeline"><a class="link" href="#_debug_a_pipeline">Debug a
pipeline</a></h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p><strong>Debug</strong> starts a local session and parks every transform
before the first row.
+A <strong>Debug mode</strong> badge stays in the top left of the canvas.
+Each hop shows <code>cursor at stored</code>: how far the next transform has
read, and how many rows are kept on that hop.
+A green border means that transform is running.
+A blue border means it is parked.
+Several transforms can run at the same time.
+Click <strong>Debug</strong> again, or <strong>Exit debug</strong>, to leave
the session and return to the editor.</p>
+</div>
+<div class="imageblock text-left">
+<div class="content">
+<img src="../_images/hop-gui/pipeline/debug-session-started.jpg" alt="A
started debugging session" width="75%">
+</div>
+</div>
+<div class="paragraph">
+<p>The <strong>Debugger</strong> tab in the execution results has the
transforms, the hops, and the rows on the selected hop.</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><strong>Process 1</strong> runs the selected transform copies for one row.
<strong>Process N</strong> uses the number in the <strong>N</strong> field. A
copy only takes a row when one is waiting. A request for more rows than the hop
holds does not take rows that arrive later.</p>
+</li>
+<li>
+<p><strong>Try</strong> runs the selected copies and then puts the input
cursors back. It does not undo the transform’s own state or side effects.
Sort rows, Group by, Unique rows, Add sequence, Table output, and similar
transforms will see the row again if you process it for real afterwards. The
button asks before it runs.</p>
+</li>
+<li>
+<p><strong>Run selected</strong> lets those copies run until they finish.
<strong>Run to here</strong> runs upstream until a row reaches the
selection.</p>
+</li>
+<li>
+<p><strong>Continue</strong> and <strong>Pause</strong> run or park the whole
pipeline.</p>
+</li>
+<li>
+<p><strong>Save</strong>, <strong>Open</strong>, and <strong>Delete</strong>
keep a session in the debug sessions folder from the configuration perspective.
<strong>Open</strong> loads the rows into a new local session.
<strong>Import</strong> on the hop toolbar appends a tab-separated file to the
selected hop.</p>
+</li>
+<li>
+<p><strong>Follow output</strong>, when it is on, selects the first output hop
of the transform you click.</p>
+</li>
+<li>
+<p>The hop toolbar rewinds a hop to the first stored row, or clears the rows
on the selected hops. Clear keeps the producer’s finished flag, so a
reader whose producer has already finished does not wait forever. Rewind is
refused once that reader has finished.</p>
+</li>
+<li>
+<p><strong>Rows</strong> shows one page of the stored rows. The
<strong><mark></strong> column is the row’s place in the hop, starting at
1. The arrows move a page, and <strong>Rows</strong> sets how many rows a page
holds, up to 5,000. New rows are added to the current page instead of
rebuilding it. <strong></mark></strong> with the target button jumps to that
row. <strong>Find</strong> and <strong>Find next</strong> search the stored
rows without loading the whole hop.</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>Keyboard shortcuts, when the pipeline graph has focus:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p>Process 1 is F10, or F6 on macOS.</p>
+</li>
+<li>
+<p>Continue is F11, or F9 on macOS.</p>
+</li>
+<li>
+<p>Pause is Shift+F11, or Shift+F9 on macOS.</p>
+</li>
+<li>
+<p>Exit debug is Ctrl+F2, or Command+F2 on macOS.</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p><strong>Debug</strong> does not stop a pipeline that is already running.
Stop that run first. Leaving the session with <strong>Exit debug</strong>,
closing the graph, or starting a normal run removes the hop buffers and the
spill files. A normal run does not keep the yellow step marker.</p>
+</div>
+<div class="sect2">
+<h3 id="_run_part_of_the_pipeline"><a class="link"
href="#_run_part_of_the_pipeline">Run part of the pipeline</a></h3>
+<div class="paragraph">
+<p><strong>Run selected</strong> or <strong>Continue</strong> leaves the
chosen transforms running and the others parked.
+The hop labels keep up with the rows that have been written and read.</p>
+</div>
+<div class="imageblock text-left">
+<div class="content">
+<img src="../_images/hop-gui/pipeline/debug-partial-run.jpg" alt="A partially
running pipeline in debug mode" width="75%">
+</div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_step_one_row"><a class="link" href="#_step_one_row">Step one
row</a></h3>
+<div class="paragraph">
+<p>With a transform selected, <strong>Process 1</strong> lets that copy handle
a single row and then parks it again.
+The rows on its output hop show what that step produced.</p>
+</div>
+<div class="imageblock text-left">
+<div class="content">
+<img src="../_images/hop-gui/pipeline/debug-single-step.jpg" alt="Single
stepping a transform" width="75%">
+</div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_pause_when_a_row_matches"><a class="link"
href="#_pause_when_a_row_matches">Pause when a row matches</a></h3>
+<div class="paragraph">
+<p>The filter button on the hop toolbar opens <strong>Pause when a row enters
this hop</strong>.
+The condition uses the fields on that hop, which are the output of the
transform that writes it.
+<strong>Pause</strong> chooses what stops when a matching row arrives:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><strong>This transform</strong> parks the transform that wrote the row.</p>
+</li>
+<li>
+<p><strong>The whole pipeline</strong> parks every transform.</p>
+</li>
+<li>
+<p><strong>Downstream transforms</strong> parks the transform that reads the
hop, and everything after it.</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>The matching row is already stored on the hop.
+The next row is what waits.
+A hop with a condition shows <code>pause</code> on its label.
+The clear button on the hop toolbar removes the condition.</p>
+</div>
+<div class="imageblock text-left">
+<div class="content">
+<img src="../_images/hop-gui/pipeline/debug-pause-condition.jpg"
alt="Conditional pause dialog" width="60%">
+</div>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
<h2 id="table-views"><a class="link" href="#table-views">Table views</a></h2>
<div class="sectionbody">
<div class="paragraph">
diff --git a/manual/next/pipeline/spark/getting-started-with-native-spark.html
b/manual/next/pipeline/spark/getting-started-with-native-spark.html
index e2f21eb5a1..3a39106165 100644
--- a/manual/next/pipeline/spark/getting-started-with-native-spark.html
+++ b/manual/next/pipeline/spark/getting-started-with-native-spark.html
@@ -3225,7 +3225,7 @@
<p><strong>Java:</strong> Hop requires <strong>Java 21</strong> for the GUI,
<code>hop-run</code>, and the fat jar executed on the Spark driver/executors.
Your Spark cluster must also run a JVM that can load that bytecode (Java
21).</p>
</div>
<div class="paragraph">
-<p><strong>Spark:</strong> The native engine is built and tested against
<strong>Spark 4.1.x</strong> (currently <strong>4.1.3</strong> in the Hop
source tree, Scala <strong>2.13</strong>). For <code>local[<strong>]</code>
runs, Spark libraries are packaged under
<code>plugins/engines/spark/lib</code>, so you do *not</strong> need a separate
Spark install on your laptop.</p>
+<p><strong>Spark:</strong> The native engine is built and tested against
<strong>Spark 4.1.x</strong> (currently <strong>4.1.3</strong> in the Hop
source tree, Scala <strong>2.13</strong>). For <code>local[*]</code> runs,
Spark libraries are packaged under <code>plugins/engines/spark/lib</code>, so
you do <strong>not</strong> need a separate Spark install on your laptop.</p>
</div>
</td>
</tr>
diff --git a/manual/next/pipeline/spark/lakehouse.html
b/manual/next/pipeline/spark/lakehouse.html
index 928d627715..cadb452eb5 100644
--- a/manual/next/pipeline/spark/lakehouse.html
+++ b/manual/next/pipeline/spark/lakehouse.html
@@ -3000,7 +3000,7 @@
<p>This guide covers <strong>Delta Lake</strong> and <strong>Apache
Iceberg</strong> support on Hop’s <a
href="getting-started-with-native-spark.html" class="xref page">native Spark
pipeline engine</a> (Spark <strong>4.1.x</strong>, no Beam).</p>
</div>
<div class="paragraph">
-<p>It assumes you already have a working <strong>Native Spark</strong>
pipeline run configuration (<code>local[<strong>]</code> or cluster
<code>spark-submit</code>). Classic file formats (CSV, Parquet, JSON, ORC,
text) stay on *Spark file input</strong> / <strong>Spark file output</strong>.
Lakehouse work uses dedicated <strong>Lake Table</strong> transforms.</p>
+<p>It assumes you already have a working <strong>Native Spark</strong>
pipeline run configuration (<code>local[*]</code> or cluster
<code>spark-submit</code>). Classic file formats (CSV, Parquet, JSON, ORC,
text) stay on <strong>Spark file input</strong> / <strong>Spark file
output</strong>. Lakehouse work uses dedicated <strong>Lake Table</strong>
transforms.</p>
</div>
</div>
</div>
diff --git a/manual/next/pipeline/spark/local-standalone-cluster.html
b/manual/next/pipeline/spark/local-standalone-cluster.html
index 9467f57fe3..eb44a329a6 100644
--- a/manual/next/pipeline/spark/local-standalone-cluster.html
+++ b/manual/next/pipeline/spark/local-standalone-cluster.html
@@ -2993,7 +2993,7 @@
<div id="preamble">
<div class="sectionbody">
<div class="paragraph">
-<p><code>local[<strong>]</code> runs Spark inside the Hop process. A
*standalone cluster</strong> runs a Spark master and one or more workers as
separate processes, the way a real cluster does: your pipeline is split into
tasks, shipped to executors, and the Hop code those tasks need travels in a fat
jar.</p>
+<p><code>local[*]</code> runs Spark inside the Hop process. A
<strong>standalone cluster</strong> runs a Spark master and one or more workers
as separate processes, the way a real cluster does: your pipeline is split into
tasks, shipped to executors, and the Hop code those tasks need travels in a fat
jar.</p>
</div>
<div class="paragraph">
<p>This page sets up such a cluster on one machine and runs a pipeline on it
from Hop GUI or <code>hop-run</code>. Hop is the Spark <strong>driver</strong>:
you design and start the pipeline in Hop as usual, and the work runs on the
workers.</p>
@@ -3540,7 +3540,7 @@ Spark pipeline finished, result row count=0</code></pre>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">The log
shows <code>WARNING: run configuration '…​' has no Spark master and
this pipeline does not run under spark-submit</code></p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">An empty
<strong>Spark master</strong> means <code>local[<strong>]</code> inside the Hop
process (Hop GUI, <code>hop-run</code> or Hop Server). Set the master, for
example <code>spark://localhost:7077</code>. The *spark-submit
(cluster)</strong> template leaves it empty on purpose, for
<code>spark-submit</code>.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">An empty
<strong>Spark master</strong> means <code>local[*]</code> inside the Hop
process (Hop GUI, <code>hop-run</code> or Hop Server). Set the master, for
example <code>spark://localhost:7077</code>. The <strong>spark-submit
(cluster)</strong> template leaves it empty on purpose, for
<code>spark-submit</code>.</p></td>
</tr>
</tbody>
</table>
diff --git a/manual/next/pipeline/transforms/parquet-file-input.html
b/manual/next/pipeline/transforms/parquet-file-input.html
index fd6df0323e..9288784d9c 100644
--- a/manual/next/pipeline/transforms/parquet-file-input.html
+++ b/manual/next/pipeline/transforms/parquet-file-input.html
@@ -3212,6 +3212,14 @@ You can pick another Hop type from the last column
instead.</p>
</tbody>
</table>
</div>
+</div>
+<div class="sect1">
+<h2 id="_beam"><a class="link" href="#_beam">Beam</a></h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>On a Beam runner each file is read by one worker, and the configured
columns are appended to the row that named the file. The filename is opened
through Beam’s file systems, so a local path or a <code>file://</code>,
<code>gs://</code>, <code>s3://</code> or <code>hdfs://</code> URI works. A
null or empty filename is skipped. The field list or a metadata filename is
required. When the incoming collection is empty and "send a null row when
empty" is set, one null row is emitted. [...]
+</div>
+</div>
</div>
</article>
<nav class="pagination" data-pagefind-ignore aria-label="Page">
diff --git a/manual/next/pipeline/transforms/parquet-file-output.html
b/manual/next/pipeline/transforms/parquet-file-output.html
index eda0c53519..eb8c32ebcb 100644
--- a/manual/next/pipeline/transforms/parquet-file-output.html
+++ b/manual/next/pipeline/transforms/parquet-file-output.html
@@ -3468,6 +3468,14 @@ The nanoseconds beyond the microsecond are
dropped.</p></td>
<p>Other Hop types can’t be written to Parquet.</p>
</div>
</div>
+</div>
+<div class="sect1">
+<h2 id="_beam"><a class="link" href="#_beam">Beam</a></h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>On a Beam runner the transform writes one Parquet file with
<code>ParquetIO</code>. The filename, including the date and time, is resolved
when the pipeline is submitted. The copy number and the split number are not
extra files. Partitioned output, and a field that sets its own Parquet type,
are refused. A JSON field and a UUID field are refused as well: on the local
engine they become a JSON column and a UUID column, and on Beam
<code>ParquetIO</code> would write both as plain string [...]
+</div>
+</div>
</div>
</article>
<nav class="pagination" data-pagefind-ignore aria-label="Page">
diff --git a/manual/next/pipeline/transforms/spark-file-input.html
b/manual/next/pipeline/transforms/spark-file-input.html
index e32d760577..a173ca94d9 100644
--- a/manual/next/pipeline/transforms/spark-file-input.html
+++ b/manual/next/pipeline/transforms/spark-file-input.html
@@ -3040,8 +3040,8 @@ It is the bulk input of the native Spark engine: Spark
reads the files in parall
<p>This transform is <strong>not</strong> available on the local Hop engine or
on the Beam engines.</p>
</div>
<div class="paragraph">
-<p>The path is a Spark/Hadoop <code>FileSystem</code> URI that must be visible
to every executor, <strong>not</strong> a Hop VFS path: a local path on
<code>local[<strong>]</code>, <code>hdfs://…</code> or <code>s3a://…</code> on
a cluster.
-Hop-only schemes such as <code>s3://</code>, <code>azure://</code> or a named
MinIO connection do not work here; see <a
href="../spark/paths-and-filesystems.html" class="xref page">Paths and file
systems</a> for the mapping and for the *Path scheme map</strong> on the run
configuration.</p>
+<p>The path is a Spark/Hadoop <code>FileSystem</code> URI that must be visible
to every executor, <strong>not</strong> a Hop VFS path: a local path on
<code>local[*]</code>, <code>hdfs://…</code> or <code>s3a://…</code> on a
cluster.
+Hop-only schemes such as <code>s3://</code>, <code>azure://</code> or a named
MinIO connection do not work here; see <a
href="../spark/paths-and-filesystems.html" class="xref page">Paths and file
systems</a> for the mapping and for the <strong>Path scheme map</strong> on the
run configuration.</p>
</div>
</div>
</div>
diff --git a/manual/next/pipeline/transforms/spark-file-output.html
b/manual/next/pipeline/transforms/spark-file-output.html
index a09a2de3be..5db1d3a40c 100644
--- a/manual/next/pipeline/transforms/spark-file-output.html
+++ b/manual/next/pipeline/transforms/spark-file-output.html
@@ -3041,8 +3041,8 @@ It is the bulk output of the native Spark engine: Spark
writes one part file per
It is a leaf: the write is executed as a Spark action when the engine
materialises the pipeline, and no rows are passed on.</p>
</div>
<div class="paragraph">
-<p>The path is a Spark/Hadoop <code>FileSystem</code> URI that must be
writable from every executor, <strong>not</strong> a Hop VFS path: a local
directory on <code>local[<strong>]</code>, <code>hdfs://…</code> or
<code>s3a://…</code> on a cluster.
-On Windows, a <code>local[</strong>]</code> path does not require
<code>HADOOP_HOME</code> or <code>winutils.exe</code>. See <a
href="../spark/paths-and-filesystems.html#windows-local-paths" class="xref
page">Windows and local mode</a>.
+<p>The path is a Spark/Hadoop <code>FileSystem</code> URI that must be
writable from every executor, <strong>not</strong> a Hop VFS path: a local
directory on <code>local[*]</code>, <code>hdfs://…</code> or
<code>s3a://…</code> on a cluster.
+On Windows, a <code>local[*]</code> path does not require
<code>HADOOP_HOME</code> or <code>winutils.exe</code>. See <a
href="../spark/paths-and-filesystems.html#windows-local-paths" class="xref
page">Windows and local mode</a>.
Spark always writes a <strong>directory</strong> of part files at that
location, never a single file with that name; see <a
href="#single-file">Getting a single file</a> below.
See <a href="../spark/paths-and-filesystems.html" class="xref page">Paths and
file systems</a> for the scheme mapping and for the <strong>Path scheme
map</strong> on the run configuration.</p>
</div>
diff --git a/manual/next/variables.html b/manual/next/variables.html
index 15b735b65b..cf7061133e 100644
--- a/manual/next/variables.html
+++ b/manual/next/variables.html
@@ -3421,11 +3421,6 @@ See also the variable
HOP_AGGREGATION_ALL_NULLS_ARE_ZERO.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Set this
variable to Y to start a pipeline that has main hops into transforms that do
not consume input (for example existing files whose upstream produces no rows).
Default N fails init and stops on leftover input. Verify still reports an
error. See <a href="pipeline/pipeline-sources.html" class="xref page">Pipeline
sources</a>.</p></td>
</tr>
<tr>
-<td class="tableblock halign-left valign-top"><p
class="tableblock">HOP_BATCHING_ROWSET</p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">N</p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">Set this
variable to 'Y' if you want to test a more efficient batching row set.</p></td>
-</tr>
-<tr>
<td class="tableblock halign-left valign-top"><p
class="tableblock">HOP_DATABASE_CONNECTION_TIMEOUT</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">30</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Default
login/connection timeout in <strong>seconds</strong> applied when opening a
JDBC database connection (maps to <code>DriverManager.setLoginTimeout</code>).
diff --git a/manual/next/workflow/actions/tableexists.html
b/manual/next/workflow/actions/tableexists.html
index e5fa10c9bc..ef036f8d97 100644
--- a/manual/next/workflow/actions/tableexists.html
+++ b/manual/next/workflow/actions/tableexists.html
@@ -3037,7 +3037,7 @@
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p
class="tableblock">False</p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">The table
doesn’t exist: the number of errors is 0. When the connection is empty or
can’t be opened, the number of errors is 1.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">The table
doesn’t exist: the number of errors is 0. When the connection is empty,
unknown, or can’t be opened, the number of errors is 1.</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Result
files</p></td>
@@ -3049,18 +3049,6 @@
</tr>
</tbody>
</table>
-<div class="admonitionblock note">
-<table>
-<tr>
-<td class="icon">
-<i class="fa icon-note" title="Note"></i>
-</td>
-<td class="content">
-A <strong>Connection</strong> that doesn’t exist in the project also
gives false with 0 errors, and nothing is logged: the same result as a table
that doesn’t exist.
-</td>
-</tr>
-</table>
-</div>
</div>
</div>
</article>
diff --git a/manual/next/workflow/actions/truncatetables.html
b/manual/next/workflow/actions/truncatetables.html
index e5f677d00f..8472ac3e98 100644
--- a/manual/next/workflow/actions/truncatetables.html
+++ b/manual/next/workflow/actions/truncatetables.html
@@ -3029,7 +3029,7 @@ If the table is in a different schema than the default
one, please specify the s
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p
class="tableblock">False</p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">A table
doesn’t exist, or the connection is unknown or can’t be opened. The
number of errors is 1: the action stops at the first table that fails, and the
tables after it are not truncated.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">At least
one error occurred. A table that doesn’t exist, or a connection that is
unknown or can’t be opened, counts as an error and stops the action: the
tables after it are not truncated. An empty table name (in the selected tables
grid or in a previous result row) also counts as an error, but later tables are
still truncated.</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Result
files</p></td>
@@ -3052,7 +3052,7 @@ If the table is in a different schema than the default
one, please specify the s
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
-With <strong>Previous results to args</strong>, the first field of each result
row is the table name and the second field the schema name. A row with an empty
table name is skipped: it is logged as an error, but doesn’t count as one.
+With <strong>Previous results to args</strong>, the first field of each result
row is the table name and the second field the schema name. A row with an empty
table name is logged as an error and counts as one: later rows are still
processed.
</td>
</tr>
</table>
diff --git a/sitemap-dev-manual.xml b/sitemap-dev-manual.xml
index b50b7d0d41..8bafdd6f90 100644
--- a/sitemap-dev-manual.xml
+++ b/sitemap-dev-manual.xml
@@ -118,6 +118,9 @@
<loc>https://hop.apache.org/dev-manual/next/plugin-types/canvas-overlays.html</loc>
</url>
<url>
+<loc>https://hop.apache.org/dev-manual/next/plugin-types/compression-formats.html</loc>
+</url>
+<url>
<loc>https://hop.apache.org/dev-manual/next/plugin-types/data-connectivity.html</loc>
</url>
<url>