This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new cb0a0b07d6 Fixes #9168 : Stop hop-run gracefully on SIGTERM/SIGINT
(#9179)
cb0a0b07d6 is described below
commit cb0a0b07d6eedeb5d1875c324f42a8bfdacc4d7a
Author: Bart Maertens <[email protected]>
AuthorDate: Tue Nov 3 09:24:19 2026 +0100
Fixes #9168 : Stop hop-run gracefully on SIGTERM/SIGINT (#9179)
* Fixes #9168 : Stop hop-run gracefully on SIGTERM/SIGINT
hop-run now stops the running workflow or pipeline when it receives a
termination signal, waits up to --shutdown-timeout / HOP_RUN_SHUTDOWN_TIMEOUT
(25s) for it to end, and delivers lineage before the JVM exits.
The launcher scripts exec the JVM and the Docker entrypoint forwards the
signal, so docker stop and Kubernetes pod termination reach hop-run. Hop
Server's existing shutdown hook now runs too. A container stopped this way
exits 143 instead of 7.
LineageHub.shutdown() is serialized so that a second call cannot shut the
sinks down while the first is still delivering.
* Issue #9168 : Deliver lineage emitted during shutdown, lower the default
stop timeout to 5s
LineageHub.emit() now waits for a running shutdown() instead of starting a
worker that the shutdown loses track of. The default HOP_RUN_SHUTDOWN_TIMEOUT
is 5 seconds, so it fits the 10 seconds docker stop allows.
* harden the code based on the remarks
---------
Co-authored-by: Hans Van Akelyen <[email protected]>
---
assemblies/static/src/main/resources/hop | 7 +-
assemblies/static/src/main/resources/hop-run.sh | 7 +-
assemblies/static/src/main/resources/hop-server.sh | 7 +-
.../integration-tests-hop_run.yaml | 25 ++
docker/resources/load-and-execute.sh | 58 +++-
docker/resources/run.sh | 4 +-
.../modules/ROOT/pages/docker-container.adoc | 4 +
.../modules/ROOT/pages/hop-run/index.adoc | 25 ++
.../java/org/apache/hop/core/HopEnvironment.java | 4 +-
.../org/apache/hop/lineage/hub/LineageHub.java | 111 +++++--
.../main/java/org/apache/hop/run/HopRunBase.java | 46 +++
.../org/apache/hop/run/HopRunShutdownHook.java | 196 +++++++++++
.../hub/LineageHubConcurrentShutdownTest.java | 235 +++++++++++++
.../hop/run/HopRunShutdownHookEngineTest.java | 111 +++++++
.../org/apache/hop/run/HopRunShutdownHookTest.java | 368 +++++++++++++++++++++
.../hop/run/HopRunShutdownTimeoutOptionTest.java | 66 ++++
.../java/org/apache/hop/run/HopRunSignalTest.java | 120 +++++++
.../org/apache/hop/run/run-until-stopped.hwf | 70 ++++
.../hop_run/0001-run-until-stopped.hwf | 70 ++++
.../hop_run/0002-run-until-stopped.hpl | 87 +++++
integration-tests/hop_run/dev-env-config.json | 3 +
integration-tests/hop_run/hop-config.json | 290 ++++++++++++++++
.../hop_run/main-0001-sigterm-stops-workflow.hwf | 130 ++++++++
.../hop_run/main-0002-sigterm-stops-pipeline.hwf | 130 ++++++++
...ain-0003-docker-entrypoint-forwards-sigterm.hwf | 130 ++++++++
.../metadata/pipeline-run-configuration/local.json | 17 +
.../metadata/workflow-run-configuration/local.json | 9 +
integration-tests/hop_run/project-config.json | 13 +
integration-tests/hop_run/sigterm-checks.sh | 55 +++
.../hop_run/sigterm-docker-entrypoint.sh | 79 +++++
integration-tests/hop_run/sigterm-hop-run.sh | 52 +++
.../openlineage/0012-openlineage-sigterm.hpl | 87 +++++
.../openlineage/0012-openlineage-sigterm.sh | 100 ++++++
.../openlineage/main-0004-openlineage-sigterm.hwf | 134 ++++++++
integration-tests/scripts/run-tests.sh | 29 +-
35 files changed, 2804 insertions(+), 75 deletions(-)
diff --git a/assemblies/static/src/main/resources/hop
b/assemblies/static/src/main/resources/hop
index 573fa25c6c..19c7cd889d 100755
--- a/assemblies/static/src/main/resources/hop
+++ b/assemblies/static/src/main/resources/hop
@@ -160,8 +160,5 @@ fi
# The JVM runs from the Hop installation, so user.dir is never where the user
typed the command.
# Pass the directory they were in, for subcommands which take a path from them.
-"${_HOP_JAVA}" ${HOP_OPTIONS} -Dhop.origin.dir="${ORIGINDIR}"
-Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}"
org.apache.hop.hop.Hop "$@"
-EXITCODE=$?
-
-cd "${ORIGINDIR}" || exit 1
-exit $EXITCODE
+# Use exec so that the JVM receives the signals sent to this script (docker
stop, Ctrl-C, ...)
+exec "${_HOP_JAVA}" ${HOP_OPTIONS} -Dhop.origin.dir="${ORIGINDIR}"
-Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}"
org.apache.hop.hop.Hop "$@"
diff --git a/assemblies/static/src/main/resources/hop-run.sh
b/assemblies/static/src/main/resources/hop-run.sh
index 456372fc56..68d7d2ea53 100755
--- a/assemblies/static/src/main/resources/hop-run.sh
+++ b/assemblies/static/src/main/resources/hop-run.sh
@@ -136,8 +136,5 @@ if [ -n "${HOP_PLUGIN_BASE_FOLDERS:-}" ]; then
unset _hop_plugin_rest _hop_plugin_folder _hop_plugin_home
fi
-"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.run.HopRun "$@"
-EXITCODE=$?
-
-cd "${ORIGINDIR}" || exit 1
-exit $EXITCODE
+# Use exec so that the JVM receives the signals sent to this script (docker
stop, Ctrl-C, ...)
+exec "${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.run.HopRun "$@"
diff --git a/assemblies/static/src/main/resources/hop-server.sh
b/assemblies/static/src/main/resources/hop-server.sh
index 16fc1a62e1..38182af530 100755
--- a/assemblies/static/src/main/resources/hop-server.sh
+++ b/assemblies/static/src/main/resources/hop-server.sh
@@ -141,8 +141,5 @@ if [ -n "${HOP_PLUGIN_BASE_FOLDERS:-}" ]; then
unset _hop_plugin_rest _hop_plugin_folder _hop_plugin_home
fi
-"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.www.HopServer "$@"
-EXITCODE=$?
-
-cd "${ORIGINDIR}" || exit 1
-exit $EXITCODE
+# Use exec so that the JVM receives the signals sent to this script (docker
stop, Ctrl-C, ...)
+exec "${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.www.HopServer "$@"
diff --git a/docker/integration-tests/integration-tests-hop_run.yaml
b/docker/integration-tests/integration-tests-hop_run.yaml
new file mode 100644
index 0000000000..34e7df330d
--- /dev/null
+++ b/docker/integration-tests/integration-tests-hop_run.yaml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+services:
+ integration_test_hop_run:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ volumes:
+ # The entry point scripts of the Hop image, to test that they pass
SIGTERM on to hop-run.
+ - ../../docker/resources:/docker-resources:ro
diff --git a/docker/resources/load-and-execute.sh
b/docker/resources/load-and-execute.sh
index 60931c0ee7..3a94e75f57 100755
--- a/docker/resources/load-and-execute.sh
+++ b/docker/resources/load-and-execute.sh
@@ -29,6 +29,45 @@ exitWithCode() {
exit "${1}"
}
+# Run a Hop command with its output copied to the log file, then exit with its
exit code.
+# The termination signals this script gets (docker stop, a Kubernetes pod
eviction, ...) are
+# passed on to the command, so that the JVM can stop the running work before
it exits. The Hop
+# scripts exec their JVM, so the signal reaches the JVM itself.
+runAndForwardSignals() {
+ local fifo
+ fifo="$(mktemp -u /tmp/hop-output.XXXXXX)"
+ mkfifo "${fifo}"
+ tee "${HOP_LOG_PATH}" <"${fifo}" &
+ local tee_pid=$!
+
+ # The trap is in place before the command starts and stays until this script
exits, so that no
+ # signal is lost. A signal that comes in before the command's pid is known
is sent once it is.
+ HOP_COMMAND_PID=""
+ HOP_SIGNALLED=false
+ trap 'HOP_SIGNALLED=true; [ -n "${HOP_COMMAND_PID}" ] && kill -TERM
"${HOP_COMMAND_PID}" 2>/dev/null' SIGTERM SIGINT SIGHUP
+ "$@" >"${fifo}" 2>&1 &
+ HOP_COMMAND_PID=$!
+ if [ "${HOP_SIGNALLED}" = "true" ]; then
+ kill -TERM "${HOP_COMMAND_PID}" 2>/dev/null
+ fi
+
+ local code=0
+ wait "${HOP_COMMAND_PID}" || code=$?
+ # A trapped signal interrupts wait while the command is still stopping: wait
for it to end.
+ while kill -0 "${HOP_COMMAND_PID}" 2>/dev/null; do
+ code=0
+ wait "${HOP_COMMAND_PID}" || code=$?
+ done
+
+ # The same goes for tee, which may still be writing the end of the output to
the log.
+ wait "${tee_pid}" || true
+ while kill -0 "${tee_pid}" 2>/dev/null; do
+ wait "${tee_pid}" || true
+ done
+ rm -f "${fifo}"
+ exitWithCode "${code}"
+}
+
# Download JDBC drivers on container start, before Hop is launched, so they
are picked up by the
# lib/jdbc scan. Driven by environment variables:
# HOP_DRIVERS_DOWNLOAD comma-separated driver ids, each optionally
with a version,
@@ -404,12 +443,9 @@ fi
if [ -z "${HOP_FILE_PATH}" ] && [ -z "${HOP_COMMAND}" ]; then
write_server_config
log "Starting a hop-server on port "${HOP_SERVER_PORT}
- "${DEPLOYMENT_PATH}"/hop-server.sh \
+ runAndForwardSignals "${DEPLOYMENT_PATH}"/hop-server.sh \
"${HOP_EXEC_OPTIONS[@]}" \
- /tmp/hop-server.xml \
- 2>&1 | tee ${HOP_LOG_PATH}
-
- exitWithCode "${PIPESTATUS[0]}"
+ /tmp/hop-server.xml
else
if [ -z "${HOP_RUN_CONFIG}" ] && [ -z "${HOP_COMMAND}" ]; then
@@ -436,20 +472,16 @@ else
if [ -n "${HOP_COMMAND}" ]; then
log "Executing command: ${DEPLOYMENT_PATH}/hop ${HOP_COMMAND}
${HOP_EXEC_OPTIONS[*]} ${HOP_COMMAND_PARAMETERS}"
- "${DEPLOYMENT_PATH}"/hop \
+ runAndForwardSignals "${DEPLOYMENT_PATH}"/hop \
${HOP_COMMAND} \
"${HOP_EXEC_OPTIONS[@]}" \
- ${HOP_COMMAND_PARAMETERS} \
- 2>&1 | tee "${HOP_LOG_PATH}"
- exitWithCode "${PIPESTATUS[0]}"
+ ${HOP_COMMAND_PARAMETERS}
else
log "Running a single hop workflow / pipeline (${HOP_FILE_PATH})"
- "${DEPLOYMENT_PATH}"/hop-run.sh \
+ runAndForwardSignals "${DEPLOYMENT_PATH}"/hop-run.sh \
--file="${HOP_FILE_PATH}" \
--runconfig="${HOP_RUN_CONFIG}" \
--parameters="${HOP_RUN_PARAMETERS}" \
- "${HOP_EXEC_OPTIONS[@]}" \
- 2>&1 | tee "${HOP_LOG_PATH}"
- exitWithCode "${PIPESTATUS[0]}"
+ "${HOP_EXEC_OPTIONS[@]}"
fi
fi
diff --git a/docker/resources/run.sh b/docker/resources/run.sh
index 814b48d062..9d4249a132 100755
--- a/docker/resources/run.sh
+++ b/docker/resources/run.sh
@@ -36,14 +36,14 @@ trapper() {
"$@" &
pid="$!"
log "Running the entrypoint script with PID ${pid}"
- trap "log 'Stopping entrypoint script with $pid'; kill -SIGTERM $pid" SIGINT
SIGTERM
+ trap "log 'Stopping entrypoint script with $pid'; kill -SIGTERM $pid" SIGINT
SIGTERM SIGHUP
while kill -0 $pid >/dev/null 2>&1; do
wait
done
}
-trapper /opt/hop/load-and-execute.sh "$@"
+trapper "$(dirname "${BASH_SOURCE[0]}")"/load-and-execute.sh "$@"
if [ -f /tmp/exitcode.txt ]; then
EXIT_CODE=$(cat /tmp/exitcode.txt)
diff --git a/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
b/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
index 2b38be89b2..0d2530e111 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
@@ -207,6 +207,10 @@ You can specify them as a comma separated list, e.g.
```PARAM_1=aaa,PARAM_2=bbb`
| No
| You can specify a metadata export file here which will contain all required
metadata elements. See also the metadata export option in the Hop Conf tool.
+|```HOP_RUN_SHUTDOWN_TIMEOUT```
+| No
+| The maximum number of seconds to wait for the pipeline or workflow to stop
when the container is stopped (`docker stop`, a Kubernetes pod eviction, ...).
The default is 5 seconds. Keep it below the grace period of the platform, see
xref:hop-run/index.adoc#_stopping_hop_run[Stopping Hop Run].
+
|===
Below are the variables you can use for a **long-lived** container, running
Hop Server:
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-run/index.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-run/index.adoc
index 626cac882b..202f15b05b 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/hop-run/index.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-run/index.adoc
@@ -79,6 +79,9 @@ Usage: <main class> [-ho] [-e=<environmentOption>]
[-f=<filename>]
The name of the Run Configuration to use
-s, --system-properties=<systemProperties>[,<systemProperties>...]
A comma separated list of KEY=VALUE pairs
+ -swt, --shutdown-timeout=<shutdownTimeout>
+ The maximum number of seconds to wait for the
pipeline or workflow to stop
+ when hop-run receives a termination signal (default
5)
-v, --version Print version information and exit
----
@@ -155,6 +158,10 @@ Check the
xref:pipeline/pipeline-run-configurations/pipeline-run-configurations.
|```--system-properties```
|A comma separated list of KEY=VALUE pairs
+|```-swt```
+|```--shutdown-timeout```
+|The maximum number of seconds to wait for the pipeline or workflow to stop
when Hop Run receives a termination signal. The default is 5 seconds. Can also
be set with the `HOP_RUN_SHUTDOWN_TIMEOUT` environment variable. See
<<_stopping_hop_run>>.
+
|```-ime```
|```--in-memory```
|Keep Hop configuration in memory. Changes are not written to
`hop-config.json`. Implied by `--project-locations` and `--environments`.
@@ -189,8 +196,26 @@ In all cases except `0`, check the error logs for more
detailed information for
|1|Error found during execution!|There was an error executing the workflow or
pipeline.
|2|General error found, something went horribly wrong!|There was a general
error that caused Hop Run to fail completely.
|9|None|There was an error parsing the provided parameter information. Hop Run
shows the syntax summary and exists.
+|143|None|Hop Run received SIGTERM and stopped the run, see
<<_stopping_hop_run>>. A process that receives SIGINT (Ctrl-C) or SIGHUP ends
with 130 or 129.
|===
+== Stopping Hop Run
+
+When Hop Run receives a termination signal (`SIGTERM`, `SIGINT` or `SIGHUP`)
it stops the running pipeline or workflow the same way the Stop button in Hop
GUI does, and waits for the run to end before the process exits.
+This happens when you press Ctrl-C, run `docker stop`, or when Kubernetes or
OpenShift evicts a pod, for example while it drains a node.
+Transforms and actions stop and release their resources (database connections,
open files, ...), and the log shows that the run was stopped.
+
+Hop Run waits at most the number of seconds set with `--shutdown-timeout` (or
the `HOP_RUN_SHUTDOWN_TIMEOUT` environment variable), 5 seconds by default, and
then exits anyway.
+When xref:technology/openlineage/index.adoc[lineage] is enabled, Hop Run then
delivers the lineage events of the run, including its end, before it exits.
+
+Keep the timeout below the time the platform gives a process before it kills
it: 30 seconds by default in Kubernetes (`terminationGracePeriodSeconds`) and
10 seconds for `docker stop` (`docker stop --time`). Leave room for the lineage
delivery as well.
+To give a run more time to stop, raise both: for example
`HOP_RUN_SHUTDOWN_TIMEOUT=60` with `docker run --stop-timeout 90`
(`stop_grace_period` in Docker Compose) or `terminationGracePeriodSeconds: 90`.
+
+`SIGKILL` (`kill -9`, or the platform giving up) cannot be handled: the
process ends at once, in the middle of whatever it was doing.
+
+Stopping a run does not undo the work it did so far.
+Design pipelines and workflows that may be stopped so that they can be
restarted.
+
== Examples
diff --git a/engine/src/main/java/org/apache/hop/core/HopEnvironment.java
b/engine/src/main/java/org/apache/hop/core/HopEnvironment.java
index 7898d31eaf..02a8e6e58c 100644
--- a/engine/src/main/java/org/apache/hop/core/HopEnvironment.java
+++ b/engine/src/main/java/org/apache/hop/core/HopEnvironment.java
@@ -250,7 +250,9 @@ public class HopEnvironment {
new Thread() {
@Override
public void run() {
- shutdown();
+ // waits for a hop-run that is still stopping, so that the end
of the run is
+ // delivered too
+ LineageHub.getInstance().shutdownOnExit();
}
});
}
diff --git a/engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java
b/engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java
index 879036ac06..07133b3b85 100644
--- a/engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java
+++ b/engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java
@@ -20,12 +20,16 @@ package org.apache.hop.lineage.hub;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
+import java.util.Set;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.logging.ILogChannel;
@@ -71,6 +75,20 @@ public final class LineageHub {
private volatile List<ILineageSink> sinks = List.of();
private final AtomicBoolean sinksInitialized = new AtomicBoolean(false);
+ /**
+ * {@link #shutdown()} takes the write lock, {@link #emit(LineageEvent)} the
read lock. Shutdowns
+ * are serialized, and an event emitted during a shutdown waits for it to
end instead of starting
+ * a worker that the shutdown then loses track of. Not {@code this}: the
worker takes that lock
+ * while it initializes the sinks, and shutdown() waits for the worker.
+ */
+ private final ReadWriteLock shutdownLock = new ReentrantReadWriteLock();
+
+ /**
+ * Runs that may still emit events when the JVM exits, for example a hop-run
that is stopping on a
+ * termination signal. {@link #shutdownOnExit()} waits until each of these
is counted down.
+ */
+ private final Set<CountDownLatch> exitHolds = ConcurrentHashMap.newKeySet();
+
private LineageHub(LineageConfiguration fixedConfiguration,
List<ILineageSink> explicitSinks) {
this.fixedConfiguration = fixedConfiguration;
this.explicitSinks = explicitSinks;
@@ -117,14 +135,19 @@ public final class LineageHub {
if (!cfg.isEnabled() || sinkInitFailed.get()) {
return;
}
- ensureWorkerStarted(cfg);
- if (!queue.offer(event)) {
- droppedEvents.incrementAndGet();
- log.logError(
- "Lineage event dropped because the queue is full (capacity "
- + cfg.getQueueCapacity()
- + "). Total drops: "
- + droppedEvents.get());
+ shutdownLock.readLock().lock();
+ try {
+ ensureWorkerStarted(cfg);
+ if (!queue.offer(event)) {
+ droppedEvents.incrementAndGet();
+ log.logError(
+ "Lineage event dropped because the queue is full (capacity "
+ + cfg.getQueueCapacity()
+ + "). Total drops: "
+ + droppedEvents.get());
+ }
+ } finally {
+ shutdownLock.readLock().unlock();
}
}
@@ -162,25 +185,65 @@ public final class LineageHub {
}
}
- /** Stop the worker thread and shut down sinks. */
+ /**
+ * Stop the worker thread and shut down sinks. Calls are serialized: a
second call waits until the
+ * first one has delivered the remaining events, instead of shutting the
sinks down under it. An
+ * event emitted in the meantime is queued once this call is done, for the
next worker or the next
+ * shutdown to deliver.
+ */
public void shutdown() {
- running.set(false);
- Thread t = worker;
- if (t != null) {
- t.interrupt();
- try {
- t.join(5000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ shutdownLock.writeLock().lock();
+ try {
+ running.set(false);
+ Thread t = worker;
+ if (t != null) {
+ t.interrupt();
+ try {
+ t.join(5000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ worker = null;
+ drainRemaining();
+ shutdownSinks();
+ sinksInitialized.set(false);
+ sinkInitFailed.set(false);
+ sinks = List.of();
+ LineageConfiguration.invalidate();
+ } finally {
+ shutdownLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Keep {@link #shutdownOnExit()} from shutting the hub down until {@code
until} is counted down:
+ * the JVM runs every shutdown hook at once, and a run that is still
stopping emits its end only
+ * after that. Call {@link #releaseShutdownOnExit(CountDownLatch)} once the
run is over.
+ */
+ public void holdShutdownOnExit(CountDownLatch until) {
+ exitHolds.add(until);
+ }
+
+ /** Undo {@link #holdShutdownOnExit(CountDownLatch)}. */
+ public void releaseShutdownOnExit(CountDownLatch until) {
+ exitHolds.remove(until);
+ }
+
+ /**
+ * {@link #shutdown()} for a shutdown hook: first waits for the runs that
hold it (see {@link
+ * #holdShutdownOnExit(CountDownLatch)}), so that their last events are
queued and delivered once,
+ * instead of shutting the sinks down under a run that is still stopping.
+ */
+ public void shutdownOnExit() {
+ try {
+ for (CountDownLatch hold : exitHolds) {
+ hold.await();
}
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
}
- worker = null;
- drainRemaining();
- shutdownSinks();
- sinksInitialized.set(false);
- sinkInitFailed.set(false);
- sinks = List.of();
- LineageConfiguration.invalidate();
+ shutdown();
}
/**
diff --git a/engine/src/main/java/org/apache/hop/run/HopRunBase.java
b/engine/src/main/java/org/apache/hop/run/HopRunBase.java
index 141f7eca44..bf27b7229b 100644
--- a/engine/src/main/java/org/apache/hop/run/HopRunBase.java
+++ b/engine/src/main/java/org/apache/hop/run/HopRunBase.java
@@ -20,6 +20,7 @@ package org.apache.hop.run;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -48,6 +49,7 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.lineage.hub.LineageHub;
import org.apache.hop.metadata.api.IHasHopMetadataProvider;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.metadata.util.HopMetadataInstance;
@@ -156,6 +158,16 @@ public abstract class HopRunBase implements Runnable,
IHasHopMetadataProvider {
+ "this flag is the CLI equivalent of the GUI \"Run anyway\"
override.")
protected boolean allowUnsupported = false;
+ @CommandLine.Option(
+ names = {"-swt", "--shutdown-timeout"},
+ description =
+ "The maximum number of seconds to wait for the pipeline or workflow
to stop when hop-run "
+ + "receives a termination signal (SIGTERM, SIGINT, SIGHUP), for
example from docker "
+ + "stop or Kubernetes. Hop Run stops the run like the Stop
button in Hop GUI does. "
+ + "Can also be set with the HOP_RUN_SHUTDOWN_TIMEOUT environment
variable.",
+ defaultValue = "${env:HOP_RUN_SHUTDOWN_TIMEOUT:-5}")
+ protected long shutdownTimeout = 5;
+
protected IVariables variables;
protected String realRunConfigurationName;
protected String realFilename;
@@ -163,6 +175,7 @@ public abstract class HopRunBase implements Runnable,
IHasHopMetadataProvider {
protected ILogChannel log;
protected MultiMetadataProvider metadataProvider;
protected boolean finishedWithoutError;
+ protected HopRunShutdownHook shutdownHook;
protected HopRunBase() {
variables = new Variables();
@@ -172,12 +185,28 @@ public abstract class HopRunBase implements Runnable,
IHasHopMetadataProvider {
public void run() {
validateOptions();
FileLoggingEventListener fileLoggingEventListener = null;
+ Thread shutdownHookThread = null;
try {
log = new LogChannel("HopRun");
log.setLogLevel(determineLogLevel());
log.logDetailed("Start of Hop Run");
+ // Stop the pipeline or workflow like Hop GUI does when the process gets
a termination
+ // signal, for example when a container platform moves the pod to
another node. Then
+ // deliver the lineage of the run: the JVM exits as soon as the hook
returns.
+ //
+ shutdownHook =
+ new HopRunShutdownHook(
+ log,
+ Duration.ofSeconds(Math.max(0L, shutdownTimeout)),
+ () -> LineageHub.getInstance().shutdown());
+ shutdownHookThread = new Thread(shutdownHook,
"hop-run-stop-on-shutdown");
+ // The lineage shutdown hook of HopEnvironment waits for this run, so
that its end is
+ // delivered as well.
+ LineageHub.getInstance().holdShutdownOnExit(shutdownHook.getRunOver());
+ Runtime.getRuntime().addShutdownHook(shutdownHookThread);
+
// Allow plugins to modify the elements loaded so far, before a pipeline
or workflow is even
// loaded
//
@@ -239,6 +268,17 @@ public abstract class HopRunBase implements Runnable,
IHasHopMetadataProvider {
if (fileLoggingEventListener != null) {
HopLogStore.getAppender().removeLoggingEventListener(fileLoggingEventListener);
}
+ if (shutdownHook != null) {
+ shutdownHook.runFinished();
+
LineageHub.getInstance().releaseShutdownOnExit(shutdownHook.getRunOver());
+ }
+ if (shutdownHookThread != null) {
+ try {
+ Runtime.getRuntime().removeShutdownHook(shutdownHookThread);
+ } catch (IllegalStateException e) {
+ // The JVM is already shutting down: the hook is running and waits
for this run.
+ }
+ }
}
}
@@ -350,6 +390,9 @@ public abstract class HopRunBase implements Runnable,
IHasHopMetadataProvider {
//
pipeline.prepareExecution();
pipeline.startThreads();
+ if (shutdownHook != null) {
+ shutdownHook.track(pipeline);
+ }
pipeline.waitUntilFinished();
setFinishedWithoutError(pipeline.getResult().getNrErrors() == 0L);
} catch (Exception e) {
@@ -470,6 +513,9 @@ public abstract class HopRunBase implements Runnable,
IHasHopMetadataProvider {
log.logMinimal("Starting workflow: " + workflowMeta.getFilename());
+ if (shutdownHook != null) {
+ shutdownHook.track(workflow);
+ }
workflow.startExecution();
setFinishedWithoutError(workflow.getResult().isResult());
} catch (Exception e) {
diff --git a/engine/src/main/java/org/apache/hop/run/HopRunShutdownHook.java
b/engine/src/main/java/org/apache/hop/run/HopRunShutdownHook.java
new file mode 100644
index 0000000000..1917a28ab1
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/run/HopRunShutdownHook.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.run;
+
+import java.time.Duration;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import lombok.Getter;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.workflow.engine.IWorkflowEngine;
+
+/**
+ * Stops the pipeline or workflow hop-run is executing when the JVM shuts down
on a termination
+ * signal (SIGTERM, SIGINT, SIGHUP), for example when Kubernetes or OpenShift
drains a node. It
+ * stops the execution the same way the Stop button in Hop GUI does, then
waits for the run to end,
+ * up to a timeout, before the JVM is allowed to exit. Then it runs the given
task, which delivers
+ * the lineage of the run. SIGKILL cannot be handled.
+ */
+public class HopRunShutdownHook implements Runnable {
+
+ /** How often the hook checks whether the run has ended, and repeats the
stop if needed. */
+ static final long CHECK_INTERVAL_MS = 250L;
+
+ /** How long the hook waits before it tries again when stopping the run
failed. */
+ static final long RETRY_INTERVAL_MS = 1000L;
+
+ /** Longer timeouts are cut to this, so that the deadline can be computed in
nanoseconds. */
+ static final Duration MAX_TIMEOUT = Duration.ofDays(365L * 100);
+
+ private final ILogChannel log;
+ private final Duration timeout;
+ private final Runnable afterStop;
+ private final CountDownLatch runFinished = new CountDownLatch(1);
+
+ /**
+ * Counted down once the run is over, or the hook stopped waiting for it.
Until then lineage may
+ * not be shut down by another shutdown hook: the run still has to emit its
end.
+ */
+ @Getter private final CountDownLatch runOver = new CountDownLatch(1);
+
+ private Execution execution;
+ private boolean stopRequested;
+
+ private record Execution(
+ String description, Runnable stop, BooleanSupplier stopped,
BooleanSupplier finished) {}
+
+ /**
+ * @param afterStop runs once the run has stopped, or the timeout has
passed, before the JVM
+ * exits. The JVM exits as soon as the hook returns, so work the main
thread still does after
+ * the run, like delivering lineage, has to happen here as well.
+ */
+ public HopRunShutdownHook(ILogChannel log, Duration timeout, Runnable
afterStop) {
+ this.log = log;
+ this.timeout = timeout.compareTo(MAX_TIMEOUT) > 0 ? MAX_TIMEOUT : timeout;
+ this.afterStop = afterStop;
+ }
+
+ /** The pipeline to stop on a termination signal. Call it once the pipeline
has started. */
+ public void track(IPipelineEngine<?> pipeline) {
+ track(
+ new Execution(
+ "pipeline '" + pipeline.getPipelineMeta().getName() + "'",
+ pipeline::stopAll,
+ pipeline::isStopped,
+ pipeline::isFinished));
+ }
+
+ /** The workflow to stop on a termination signal. Call it before the
workflow starts. */
+ public void track(IWorkflowEngine<?> workflow) {
+ track(
+ new Execution(
+ "workflow '" + workflow.getWorkflowName() + "'",
+ workflow::stopExecution,
+ workflow::isStopped,
+ workflow::isFinished));
+ }
+
+ private synchronized void track(Execution execution) {
+ this.execution = execution;
+ }
+
+ /** Tells the hook the run is over, so it no longer needs to stop or wait
for anything. */
+ public void runFinished() {
+ runFinished.countDown();
+ runOver.countDown();
+ }
+
+ @Override
+ public void run() {
+ if (runFinished.getCount() == 0) {
+ return;
+ }
+ synchronized (this) {
+ if (stopRequested) {
+ return;
+ }
+ stopRequested = true;
+ }
+ log.logMinimal(
+ "Termination signal received, stopping the run. Waiting up to "
+ + timeout.toSeconds()
+ + " seconds for it to end.");
+ try {
+ stopAndWait();
+ } finally {
+ runOver.countDown();
+ afterStop.run();
+ }
+ }
+
+ private void stopAndWait() {
+ long deadline = System.nanoTime() + timeout.toNanos();
+ Execution target = null;
+ boolean stopNeeded = false;
+ boolean stopHeld = false;
+ int failures = 0;
+ long nextAttempt = System.nanoTime();
+ try {
+ do {
+ Execution current;
+ synchronized (this) {
+ current = execution;
+ }
+ // The execution may only start after the signal came in.
+ //
+ if (current != null && current != target) {
+ target = current;
+ stopNeeded = true;
+ stopHeld = false;
+ failures = 0;
+ nextAttempt = System.nanoTime();
+ } else if (current != null && !stopNeeded) {
+ // A workflow clears its stopped flag when it starts, which undoes a
stop sent just
+ // before. Only stop again when the flag was seen set and is cleared
now: a remote engine
+ // only sets it once the server reports the run as stopped.
+ //
+ boolean isStopped = current.stopped().getAsBoolean();
+ if (stopHeld && !isStopped && !current.finished().getAsBoolean()) {
+ stopNeeded = true;
+ }
+ stopHeld = isStopped;
+ }
+ if (stopNeeded && System.nanoTime() - nextAttempt >= 0) {
+ if (failures == 0) {
+ log.logMinimal("Stopping " + current.description());
+ }
+ try {
+ current.stop().run();
+ stopNeeded = false;
+ stopHeld = current.stopped().getAsBoolean();
+ failures = 0;
+ } catch (RuntimeException e) {
+ // For example a Hop server that can't be reached, or a remote run
not submitted yet.
+ failures++;
+ String message =
+ "Stopping " + current.description() + " failed, trying again
in a second.";
+ if (failures == 1) {
+ log.logError(message, e);
+ } else {
+ log.logError(message);
+ }
+ nextAttempt = System.nanoTime() +
TimeUnit.MILLISECONDS.toNanos(RETRY_INTERVAL_MS);
+ }
+ }
+ if (runFinished.await(CHECK_INTERVAL_MS, TimeUnit.MILLISECONDS)) {
+ log.logMinimal("The run has stopped, hop-run exits.");
+ return;
+ }
+ } while (System.nanoTime() - deadline < 0);
+ log.logError(
+ "The run did not stop within "
+ + timeout.toSeconds()
+ + " seconds, hop-run exits anyway. Use --shutdown-timeout or
HOP_RUN_SHUTDOWN_TIMEOUT to wait longer.");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+}
diff --git
a/engine/src/test/java/org/apache/hop/lineage/hub/LineageHubConcurrentShutdownTest.java
b/engine/src/test/java/org/apache/hop/lineage/hub/LineageHubConcurrentShutdownTest.java
new file mode 100644
index 0000000000..9fcaf7c0a8
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/lineage/hub/LineageHubConcurrentShutdownTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.hub;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.lineage.context.LineageContext;
+import org.apache.hop.lineage.model.LineageEvent;
+import org.apache.hop.lineage.model.LineageEventKind;
+import org.apache.hop.lineage.spi.ILineageSink;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+/**
+ * When hop-run is stopped by a signal, its shutdown hook and its main thread
both shut the hub
+ * down. The second call may not cut off the delivery the first one is still
doing.
+ */
+@Timeout(value = 30, unit = TimeUnit.SECONDS)
+class LineageHubConcurrentShutdownTest {
+
+ private static final int EVENTS = 5;
+ private static final long DELIVERY_MS = 100;
+
+ @BeforeAll
+ static void initLogStore() {
+ HopLogStore.init();
+ }
+
+ @Test
+ void secondShutdownWaitsForTheFirstToDeliverEverything() throws Exception {
+ AtomicInteger delivered = new AtomicInteger();
+ AtomicBoolean deliveredAfterSinkShutdown = new AtomicBoolean();
+ AtomicBoolean sinkShutdown = new AtomicBoolean();
+ ILineageSink slowSink =
+ new ILineageSink() {
+ @Override
+ public void accept(List<LineageEvent> events) {
+ // Spin rather than sleep: the hub interrupts its worker when it
shuts down.
+ long end = System.nanoTime() +
TimeUnit.MILLISECONDS.toNanos(DELIVERY_MS);
+ while (System.nanoTime() < end) {
+ Thread.onSpinWait();
+ }
+ if (sinkShutdown.get()) {
+ deliveredAfterSinkShutdown.set(true);
+ }
+ delivered.addAndGet(events.size());
+ }
+
+ @Override
+ public void shutdown() {
+ sinkShutdown.set(true);
+ }
+ };
+ LineageConfiguration cfg = LineageConfiguration.forTesting(true, 100, 1,
5000L, Set.of());
+ LineageHub hub = LineageHub.newIsolatedForTesting(cfg, List.of(slowSink));
+
+ for (int i = 0; i < EVENTS; i++) {
+ hub.emit(LineageEvent.of(LineageEventKind.RUN_LIFECYCLE,
LineageContext.empty(), null));
+ }
+
+ Thread first = new Thread(hub::shutdown, "first-shutdown");
+ first.start();
+ // The first shutdown is delivering the queued events one by one.
+ Thread.sleep(DELIVERY_MS + DELIVERY_MS / 2);
+ hub.shutdown();
+ first.join(20_000);
+
+ assertEquals(EVENTS, delivered.get(), "Every queued event should reach the
sink");
+ assertFalse(
+ deliveredAfterSinkShutdown.get(), "The sink was shut down while events
were delivered");
+ }
+
+ /**
+ * A run that is still going when the hook shuts the hub down emits its end
while the shutdown
+ * delivers. That event may not start a worker the shutdown loses track of:
the next shutdown has
+ * to deliver it.
+ */
+ @Test
+ void eventEmittedDuringShutdownIsDeliveredByTheNextShutdown() throws
Exception {
+ AtomicInteger delivered = new AtomicInteger();
+ AtomicInteger delivering = new AtomicInteger();
+ AtomicBoolean deliveredConcurrently = new AtomicBoolean();
+ AtomicBoolean deliveredAfterSinkShutdown = new AtomicBoolean();
+ AtomicBoolean sinkShutdown = new AtomicBoolean();
+ ILineageSink slowSink =
+ new ILineageSink() {
+ @Override
+ public void init(IVariables variables, ILogChannel log) {
+ sinkShutdown.set(false);
+ }
+
+ @Override
+ public void accept(List<LineageEvent> events) {
+ if (delivering.incrementAndGet() > 1) {
+ deliveredConcurrently.set(true);
+ }
+ // Spin rather than sleep: the hub interrupts its worker when it
shuts down.
+ long end = System.nanoTime() +
TimeUnit.MILLISECONDS.toNanos(DELIVERY_MS);
+ while (System.nanoTime() < end) {
+ Thread.onSpinWait();
+ }
+ if (sinkShutdown.get()) {
+ deliveredAfterSinkShutdown.set(true);
+ }
+ delivered.addAndGet(events.size());
+ delivering.decrementAndGet();
+ }
+
+ @Override
+ public void shutdown() {
+ sinkShutdown.set(true);
+ }
+ };
+ LineageConfiguration cfg = LineageConfiguration.forTesting(true, 100, 1,
5000L, Set.of());
+ LineageHub hub = LineageHub.newIsolatedForTesting(cfg, List.of(slowSink));
+
+ for (int i = 0; i < EVENTS; i++) {
+ hub.emit(LineageEvent.of(LineageEventKind.RUN_LIFECYCLE,
LineageContext.empty(), null));
+ }
+
+ Thread first = new Thread(hub::shutdown, "first-shutdown");
+ first.start();
+ // The first shutdown is delivering the queued events one by one.
+ Thread.sleep(DELIVERY_MS + DELIVERY_MS / 2);
+ hub.emit(LineageEvent.of(LineageEventKind.RUN_LIFECYCLE,
LineageContext.empty(), null));
+ first.join(20_000);
+ hub.shutdown();
+
+ assertEquals(EVENTS + 1, delivered.get(), "Every emitted event should
reach the sink");
+ assertFalse(deliveredConcurrently.get(), "A second worker delivered during
the shutdown");
+ assertFalse(
+ deliveredAfterSinkShutdown.get(), "The sink was shut down while events
were delivered");
+ }
+
+ /**
+ * The JVM runs the shutdown hook of HopEnvironment at the same time as the
one of hop-run. While
+ * hop-run holds the hub, that hook may not shut it down: the run still has
to emit its end.
+ */
+ @Test
+ void shutdownOnExitWaitsForTheRunThatHoldsIt() throws Exception {
+ AtomicInteger delivered = new AtomicInteger();
+ AtomicInteger sinkShutdowns = new AtomicInteger();
+ LineageHub hub =
+ LineageHub.newIsolatedForTesting(configuration(),
List.of(sink(delivered, sinkShutdowns)));
+ hub.emit(event()); // the start of the run
+ CountDownLatch runOver = new CountDownLatch(1);
+ hub.holdShutdownOnExit(runOver);
+
+ Thread exitHook = new Thread(hub::shutdownOnExit, "exit-hook");
+ exitHook.start();
+ // hop-run is still stopping the run
+ Thread.sleep(300);
+ assertTrue(exitHook.isAlive(), "The hub was shut down while a run held
it");
+ assertEquals(0, sinkShutdowns.get(), "The sinks were shut down while the
run was stopping");
+
+ hub.emit(event()); // the end of the run
+ runOver.countDown();
+ exitHook.join(10_000);
+
+ assertFalse(exitHook.isAlive());
+ assertEquals(2, delivered.get(), "Every emitted event should reach the
sink");
+ assertEquals(1, sinkShutdowns.get());
+ }
+
+ /** A run that is over releases its hold, so a later exit does not wait for
it. */
+ @Test
+ void aReleasedHoldNoLongerDelaysTheShutdown() {
+ AtomicInteger sinkShutdowns = new AtomicInteger();
+ LineageHub hub =
+ LineageHub.newIsolatedForTesting(
+ configuration(), List.of(sink(new AtomicInteger(),
sinkShutdowns)));
+ hub.emit(event());
+ CountDownLatch neverCountedDown = new CountDownLatch(1);
+ hub.holdShutdownOnExit(neverCountedDown);
+ hub.releaseShutdownOnExit(neverCountedDown);
+
+ hub.shutdownOnExit(); // the class timeout fails the test if this waits
+
+ assertEquals(1, sinkShutdowns.get());
+ }
+
+ private static LineageConfiguration configuration() {
+ return LineageConfiguration.forTesting(true, 100, 1, 5000L, Set.of());
+ }
+
+ private static LineageEvent event() {
+ return LineageEvent.of(LineageEventKind.RUN_LIFECYCLE,
LineageContext.empty(), null);
+ }
+
+ private static ILineageSink sink(AtomicInteger delivered, AtomicInteger
shutdowns) {
+ return new ILineageSink() {
+ @Override
+ public void init(IVariables variables, ILogChannel log) {
+ // nothing to set up
+ }
+
+ @Override
+ public void accept(List<LineageEvent> events) {
+ delivered.addAndGet(events.size());
+ }
+
+ @Override
+ public void shutdown() {
+ shutdowns.incrementAndGet();
+ }
+ };
+ }
+}
diff --git
a/engine/src/test/java/org/apache/hop/run/HopRunShutdownHookEngineTest.java
b/engine/src/test/java/org/apache/hop/run/HopRunShutdownHookEngineTest.java
new file mode 100644
index 0000000000..6db6b8d0c2
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/run/HopRunShutdownHookEngineTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.run;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.engines.local.LocalWorkflowEngine;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/** The shutdown hook against the local pipeline and workflow engines, without
mocks. */
+@ExtendWith(RestoreHopEngineEnvironmentExtension.class)
+@Timeout(value = 60, unit = TimeUnit.SECONDS)
+class HopRunShutdownHookEngineTest {
+
+ @BeforeAll
+ static void beforeAll() throws Exception {
+ HopEnvironment.init();
+ }
+
+ /** The Injector waits for rows that never come, so the pipeline only ends
when it's stopped. */
+ @Test
+ void runningPipelineIsStopped() throws Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("wait-for-rows");
+ TransformMeta injector = new TransformMeta("Injector", new InjectorMeta());
+ TransformMeta dummy = new TransformMeta("Dummy", new DummyMeta());
+ pipelineMeta.addTransform(injector);
+ pipelineMeta.addTransform(dummy);
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(injector, dummy));
+
+ LocalPipelineEngine pipeline = new LocalPipelineEngine(pipelineMeta);
+ pipeline.prepareExecution();
+ pipeline.addRowProducer("Injector", 0);
+ pipeline.startThreads();
+
+ HopRunShutdownHook hook =
+ new HopRunShutdownHook(new LogChannel("test"), Duration.ofSeconds(30),
() -> {});
+ hook.track(pipeline);
+ assertFalse(pipeline.isFinished());
+
+ CompletableFuture<Void> signal = CompletableFuture.runAsync(hook);
+ pipeline.waitUntilFinished();
+ hook.runFinished();
+ signal.get(10, TimeUnit.SECONDS);
+
+ assertTrue(pipeline.isStopped(), "The pipeline should be stopped");
+ assertTrue(pipeline.isFinished(), "The pipeline should be finished");
+ }
+
+ /**
+ * The signal arrives before the workflow starts, and the workflow clears
its stopped flag when it
+ * starts. The hook has to repeat the stop for the workflow to end.
+ */
+ @Test
+ void workflowSignalledBeforeItStartsIsStopped() throws Exception {
+ String filename =
+ new
File(getClass().getResource("/org/apache/hop/run/run-until-stopped.hwf").toURI())
+ .getPath();
+ WorkflowMeta workflowMeta =
+ new WorkflowMeta(new Variables(), filename, new
MemoryMetadataProvider());
+ LocalWorkflowEngine workflow = new LocalWorkflowEngine(workflowMeta);
+
+ HopRunShutdownHook hook =
+ new HopRunShutdownHook(new LogChannel("test"), Duration.ofSeconds(30),
() -> {});
+ hook.track(workflow);
+
+ CompletableFuture<Void> signal = CompletableFuture.runAsync(hook);
+ Thread.sleep(500);
+ workflow.startExecution();
+ hook.runFinished();
+ signal.get(10, TimeUnit.SECONDS);
+
+ assertTrue(workflow.isStopped(), "The workflow should be stopped");
+ }
+}
diff --git
a/engine/src/test/java/org/apache/hop/run/HopRunShutdownHookTest.java
b/engine/src/test/java/org/apache/hop/run/HopRunShutdownHookTest.java
new file mode 100644
index 0000000000..92fee02353
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/run/HopRunShutdownHookTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.run;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hop.core.exception.HopRuntimeException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.engine.IWorkflowEngine;
+import org.junit.jupiter.api.Test;
+
+class HopRunShutdownHookTest {
+
+ private final ILogChannel log = mock(ILogChannel.class);
+ private final AtomicInteger afterStopCalls = new AtomicInteger();
+ private final Runnable afterStop = afterStopCalls::incrementAndGet;
+
+ @Test
+ void nothingIsStoppedWhenTheRunIsOver() {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(pipeline);
+ hook.runFinished();
+
+ hook.run();
+
+ verify(pipeline, never()).stopAll();
+ verify(log, never()).logMinimal(anyString());
+ assertEquals(0, afterStopCalls.get());
+ }
+
+ @Test
+ void hookReturnsAtOnceWhenNothingWasStarted() throws Exception {
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ Thread hookThread = start(hook);
+
+ // hop-run is still loading the pipeline: the hook waits for the run to
end.
+ Thread.sleep(300);
+ assertTrue(hookThread.isAlive());
+
+ hook.runFinished();
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ }
+
+ @Test
+ void pipelineIsStoppedAndTheHookWaitsUntilTheRunEnds() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(pipeline);
+
+ Thread hookThread = start(hook);
+ verify(pipeline, timeout(5000)).stopAll();
+
+ // The pipeline is still winding down, so the JVM must not exit yet, nor
may lineage be
+ // delivered: the end of the run is not in it yet.
+ Thread.sleep(300);
+ assertTrue(hookThread.isAlive());
+ assertEquals(0, afterStopCalls.get());
+
+ hook.runFinished();
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ assertEquals(1, afterStopCalls.get());
+ verify(pipeline, times(1)).stopAll();
+ verify(log).logMinimal("The run has stopped, hop-run exits.");
+ }
+
+ @Test
+ void workflowIsStopped() throws Exception {
+ IWorkflowEngine<WorkflowMeta> workflow = mock(IWorkflowEngine.class);
+ when(workflow.getWorkflowName()).thenReturn("wf");
+ when(workflow.isStopped()).thenReturn(true);
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(workflow);
+
+ Thread hookThread = start(hook);
+ verify(workflow, timeout(5000)).stopExecution();
+ hook.runFinished();
+ hookThread.join(5000);
+
+ assertFalse(hookThread.isAlive());
+ verify(workflow, times(1)).stopExecution();
+ verify(log).logMinimal("Stopping workflow 'wf'");
+ }
+
+ /** A workflow clears its stopped flag when it starts. A stop sent just
before that is lost. */
+ @Test
+ void stopIsRepeatedWhenTheWorkflowClearsItsStoppedFlag() throws Exception {
+ IWorkflowEngine<WorkflowMeta> workflow = mock(IWorkflowEngine.class);
+ when(workflow.getWorkflowName()).thenReturn("wf");
+ AtomicBoolean stopped = new AtomicBoolean();
+ when(workflow.isStopped()).thenAnswer(invocation -> stopped.get());
+ doAnswer(
+ invocation -> {
+ stopped.set(true);
+ return null;
+ })
+ .when(workflow)
+ .stopExecution();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(workflow);
+
+ Thread hookThread = start(hook);
+ verify(workflow, timeout(5000)).stopExecution();
+
+ // The workflow starts and clears the flag.
+ stopped.set(false);
+ verify(workflow, timeout(5000).times(2)).stopExecution();
+
+ // Once the stop holds it is not repeated.
+ Thread.sleep(3 * HopRunShutdownHook.CHECK_INTERVAL_MS);
+ verify(workflow, times(2)).stopExecution();
+
+ hook.runFinished();
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ }
+
+ /**
+ * A remote engine only reports the run as stopped once the server says so.
Until then the hook
+ * may not send a stop request on every check.
+ */
+ @Test
+ void stopIsSentOnceWhileTheStoppedFlagIsNotSetYet() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ when(pipeline.isStopped()).thenReturn(false);
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(pipeline);
+
+ Thread hookThread = start(hook);
+ verify(pipeline, timeout(5000)).stopAll();
+ Thread.sleep(4 * HopRunShutdownHook.CHECK_INTERVAL_MS);
+ verify(pipeline, times(1)).stopAll();
+
+ hook.runFinished();
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ }
+
+ /** A remote engine throws when the server can't be reached or the run isn't
submitted yet. */
+ @Test
+ void failedStopIsTriedAgainAndTheHookKeepsWaiting() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ doThrow(new HopRuntimeException("server down"))
+ .doThrow(new HopRuntimeException("server down"))
+ .doNothing()
+ .when(pipeline)
+ .stopAll();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(pipeline);
+
+ Thread hookThread = start(hook);
+ verify(pipeline, timeout(5000).times(3)).stopAll();
+ assertTrue(hookThread.isAlive(), "The hook may not give up when a stop
fails");
+ assertEquals(0, afterStopCalls.get());
+
+ String message = "Stopping pipeline 'pl' failed, trying again in a
second.";
+ verify(log).logError(eq(message), any(Throwable.class));
+ verify(log).logError(message);
+ verify(log, times(1)).logMinimal("Stopping pipeline 'pl'");
+
+ // The retries are spaced out.
+ Thread.sleep(3 * HopRunShutdownHook.CHECK_INTERVAL_MS);
+ verify(pipeline, times(3)).stopAll();
+
+ hook.runFinished();
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ assertEquals(1, afterStopCalls.get());
+ }
+
+ /** A timeout meant as "wait forever" may not overflow the deadline. */
+ @Test
+ void hugeTimeoutStillStops() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook =
+ new HopRunShutdownHook(log, Duration.ofSeconds(Long.MAX_VALUE),
afterStop);
+ hook.track(pipeline);
+
+ Thread hookThread = start(hook);
+ verify(pipeline, timeout(5000)).stopAll();
+ Thread.sleep(300);
+ assertTrue(hookThread.isAlive());
+
+ hook.runFinished();
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ assertEquals(1, afterStopCalls.get());
+ }
+
+ @Test
+ void executionStartedAfterTheSignalIsStopped() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+
+ Thread hookThread = start(hook);
+ Thread.sleep(300);
+ hook.track(pipeline);
+
+ verify(pipeline, timeout(5000)).stopAll();
+ hook.runFinished();
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ }
+
+ @Test
+ void finishedExecutionIsNotStoppedAgain() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ when(pipeline.isStopped()).thenReturn(false);
+ when(pipeline.isFinished()).thenReturn(true);
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(pipeline);
+
+ Thread hookThread = start(hook);
+ verify(pipeline, timeout(5000)).stopAll();
+ Thread.sleep(3 * HopRunShutdownHook.CHECK_INTERVAL_MS);
+ hook.runFinished();
+ hookThread.join(5000);
+
+ verify(pipeline, times(1)).stopAll();
+ }
+
+ @Test
+ void hookGivesUpAfterTheTimeout() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(1), afterStop);
+ hook.track(pipeline);
+
+ long start = System.nanoTime();
+ hook.run();
+ long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+
+ assertTrue(elapsedMs >= 1000, "The hook returned after " + elapsedMs +
"ms");
+ assertTrue(elapsedMs < 5000, "The hook returned after " + elapsedMs +
"ms");
+ assertEquals(1, afterStopCalls.get(), "Lineage is delivered after a
timeout too");
+ verify(log)
+ .logError(
+ "The run did not stop within 1 seconds, hop-run exits anyway. Use
--shutdown-timeout or HOP_RUN_SHUTDOWN_TIMEOUT to wait longer.");
+ }
+
+ @Test
+ void zeroTimeoutStillStops() {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log, Duration.ZERO,
afterStop);
+ hook.track(pipeline);
+
+ hook.run();
+
+ verify(pipeline).stopAll();
+ }
+
+ @Test
+ void secondCallDoesNothing() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(pipeline);
+
+ Thread first = start(hook);
+ verify(pipeline, timeout(5000)).stopAll();
+
+ Thread second = start(hook);
+ second.join(5000);
+ assertFalse(second.isAlive(), "A second call may not wait as well");
+
+ hook.runFinished();
+ first.join(5000);
+ verify(pipeline, times(1)).stopAll();
+ assertEquals(1, afterStopCalls.get());
+ verify(log, atLeast(1)).logMinimal(anyString());
+ }
+
+ /** Until the run is over, the lineage shutdown hook of HopEnvironment has
to wait. */
+ @Test
+ void lineageIsHeldUntilTheRunIsOver() throws Exception {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.track(pipeline);
+
+ Thread hookThread = start(hook);
+ verify(pipeline, timeout(5000)).stopAll();
+ Thread.sleep(300);
+ assertEquals(1, hook.getRunOver().getCount(), "Released while the run was
still stopping");
+
+ hook.runFinished();
+ assertTrue(hook.getRunOver().await(5, TimeUnit.SECONDS));
+ hookThread.join(5000);
+ assertFalse(hookThread.isAlive());
+ }
+
+ /** A run that does not stop in time does not keep the other shutdown hooks
waiting either. */
+ @Test
+ void lineageIsReleasedBeforeItIsDeliveredWhenTheHookGivesUp() {
+ IPipelineEngine<PipelineMeta> pipeline = pipeline();
+ AtomicLong heldDuringAfterStop = new AtomicLong(-1);
+ HopRunShutdownHook[] hook = new HopRunShutdownHook[1];
+ hook[0] =
+ new HopRunShutdownHook(
+ log, Duration.ZERO, () ->
heldDuringAfterStop.set(hook[0].getRunOver().getCount()));
+ hook[0].track(pipeline);
+
+ hook[0].run();
+
+ assertEquals(0, heldDuringAfterStop.get());
+ }
+
+ /** A run that ends without a signal releases lineage too. */
+ @Test
+ void lineageIsReleasedWhenTheRunEndsWithoutASignal() {
+ HopRunShutdownHook hook = new HopRunShutdownHook(log,
Duration.ofSeconds(10), afterStop);
+ hook.runFinished();
+ assertEquals(0, hook.getRunOver().getCount());
+ }
+
+ private IPipelineEngine<PipelineMeta> pipeline() {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("pl");
+ IPipelineEngine<PipelineMeta> pipeline = mock(IPipelineEngine.class);
+ when(pipeline.getPipelineMeta()).thenReturn(pipelineMeta);
+ when(pipeline.isStopped()).thenReturn(true);
+ return pipeline;
+ }
+
+ private static Thread start(HopRunShutdownHook hook) {
+ Thread thread = new Thread(hook, "test-shutdown-hook");
+ thread.setDaemon(true);
+ thread.start();
+ return thread;
+ }
+}
diff --git
a/engine/src/test/java/org/apache/hop/run/HopRunShutdownTimeoutOptionTest.java
b/engine/src/test/java/org/apache/hop/run/HopRunShutdownTimeoutOptionTest.java
new file mode 100644
index 0000000000..dfa767df5a
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/run/HopRunShutdownTimeoutOptionTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.run;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import org.junit.jupiter.api.Test;
+import picocli.CommandLine;
+
+class HopRunShutdownTimeoutOptionTest {
+
+ @Test
+ void defaultIs5Seconds() {
+ assumeTrue(
+ System.getenv("HOP_RUN_SHUTDOWN_TIMEOUT") == null,
+ "HOP_RUN_SHUTDOWN_TIMEOUT is set in this environment");
+
+ assertEquals(5L, parse().getShutdownTimeout());
+ }
+
+ @Test
+ void longOption() {
+ assertEquals(60L, parse("--shutdown-timeout=60").getShutdownTimeout());
+ }
+
+ @Test
+ void shortOption() {
+ assertEquals(8L, parse("-swt", "8").getShutdownTimeout());
+ }
+
+ @Test
+ void helpDocumentsTheOption() {
+ StringWriter writer = new StringWriter();
+ new CommandLine(new HopRun()).usage(new PrintWriter(writer, true));
+ String usage = writer.toString().replaceAll("\\s+", " ");
+
+ assertTrue(usage.contains("-swt, --shutdown-timeout"), usage);
+ assertTrue(usage.contains("HOP_RUN_SHUTDOWN_TIMEOUT"), usage);
+ }
+
+ private static HopRun parse(String... args) {
+ HopRun hopRun = new HopRun();
+ new CommandLine(hopRun).parseArgs(args);
+ return hopRun;
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/run/HopRunSignalTest.java
b/engine/src/test/java/org/apache/hop/run/HopRunSignalTest.java
new file mode 100644
index 0000000000..14aabdf09b
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/run/HopRunSignalTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.run;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.Const;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Runs hop-run in a JVM of its own and sends it SIGTERM, the way docker stop
or a Kubernetes pod
+ * eviction does. The workflow repeats until it is stopped, so it only ends
because of the signal.
+ */
+@Timeout(value = 3, unit = TimeUnit.MINUTES)
+class HopRunSignalTest {
+
+ private static final int SIGTERM_EXIT_CODE = 128 + 15;
+
+ @TempDir Path tempDir;
+
+ @Test
+ void sigtermStopsTheRunningWorkflowBeforeTheJvmExits() throws Exception {
+ assumeFalse(Const.isWindows(), "Windows has no SIGTERM");
+
+ Path metadata = tempDir.resolve("metadata");
+ Files.createDirectories(metadata.resolve("workflow-run-configuration"));
+ Files.writeString(
+ metadata.resolve("workflow-run-configuration/local.json"),
+ """
+ {
+ "engineRunConfiguration": { "Local": { "safe_mode": false } },
+ "name": "local"
+ }
+ """);
+ Path config = Files.createDirectories(tempDir.resolve("config"));
+ String workflow =
+ new
File(getClass().getResource("/org/apache/hop/run/run-until-stopped.hwf").toURI())
+ .getPath();
+
+ List<String> command = new ArrayList<>();
+ command.add(Path.of(System.getProperty("java.home"), "bin",
"java").toString());
+ command.add("-DHOP_CONFIG_FOLDER=" + config);
+ command.add("-DHOP_AUTO_CREATE_CONFIG=Y");
+ command.add("-DHOP_METADATA_FOLDER=" + metadata);
+ command.add("-classpath");
+ command.add(System.getProperty("java.class.path"));
+ command.add(HopRun.class.getName());
+ command.add("--file=" + workflow);
+ command.add("--runconfig=local");
+ command.add("--level=Basic");
+ command.add("--shutdown-timeout=20");
+
+ // A file, not a pipe: the JDK closes the pipe when the process exits, and
the last lines,
+ // the ones written while stopping, could get lost.
+ Path outputFile = tempDir.resolve("hop-run.log");
+ Process process =
+ new ProcessBuilder(command)
+ .directory(tempDir.toFile())
+ .redirectErrorStream(true)
+ .redirectOutput(outputFile.toFile())
+ .start();
+
+ try {
+ // The first repetition is done: the workflow is running.
+ long deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(2);
+ while (!Files.readString(outputFile).contains("Finished action
[Dummy]")) {
+ assertTrue(process.isAlive(), "hop-run ended early:\n" +
Files.readString(outputFile));
+ assertTrue(
+ System.nanoTime() < deadline,
+ "The workflow did not start in time:\n" +
Files.readString(outputFile));
+ Thread.sleep(100);
+ }
+
+ // On Unix Process.destroy() sends SIGTERM.
+ long signalled = System.nanoTime();
+ process.destroy();
+ assertTrue(
+ process.waitFor(60, TimeUnit.SECONDS),
+ "hop-run did not exit:\n" + Files.readString(outputFile));
+ long stopMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() -
signalled);
+
+ String log = Files.readString(outputFile);
+ assertEquals(SIGTERM_EXIT_CODE, process.exitValue(), log);
+ assertTrue(
+ log.contains("Termination signal received, stopping the run. Waiting
up to 20 seconds"),
+ log);
+ assertTrue(log.contains("Stopping workflow 'run-until-stopped'"), log);
+ assertTrue(log.contains("The run has stopped, hop-run exits."), log);
+ assertTrue(stopMs < 20_000, "Stopping took " + stopMs + "ms:\n" + log);
+ } finally {
+ process.destroyForcibly();
+ }
+ }
+}
diff --git a/engine/src/test/resources/org/apache/hop/run/run-until-stopped.hwf
b/engine/src/test/resources/org/apache/hop/run/run-until-stopped.hwf
new file mode 100644
index 0000000000..dd12412565
--- /dev/null
+++ b/engine/src/test/resources/org/apache/hop/run/run-until-stopped.hwf
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>run-until-stopped</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Repeats every second until it is stopped</description>
+ <extended_description/>
+ <workflow_version/>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>0</intervalMinutes>
+ <intervalSeconds>1</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>Y</repeat>
+ <schedulerType>1</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Dummy</name>
+ <description/>
+ <type>DUMMY</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Dummy</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/hop_run/0001-run-until-stopped.hwf
b/integration-tests/hop_run/0001-run-until-stopped.hwf
new file mode 100644
index 0000000000..0e69567969
--- /dev/null
+++ b/integration-tests/hop_run/0001-run-until-stopped.hwf
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0001-run-until-stopped</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Repeats every second until it is stopped</description>
+ <extended_description/>
+ <workflow_version/>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>0</intervalMinutes>
+ <intervalSeconds>1</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>Y</repeat>
+ <schedulerType>1</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Dummy</name>
+ <description/>
+ <type>DUMMY</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Dummy</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/hop_run/0002-run-until-stopped.hpl
b/integration-tests/hop_run/0002-run-until-stopped.hpl
new file mode 100644
index 0000000000..f049ed77d5
--- /dev/null
+++ b/integration-tests/hop_run/0002-run-until-stopped.hpl
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0002-run-until-stopped</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Generates a row every 100ms until it is stopped</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>Dummy</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <limit>10</limit>
+ <never_ending>Y</never_ending>
+ <interval_in_ms>100</interval_in_ms>
+ <row_time_field/>
+ <last_time_field/>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Dummy</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/hop_run/dev-env-config.json
b/integration-tests/hop_run/dev-env-config.json
new file mode 100644
index 0000000000..8b1ae25912
--- /dev/null
+++ b/integration-tests/hop_run/dev-env-config.json
@@ -0,0 +1,3 @@
+{
+ "variables": [ ]
+}
diff --git a/integration-tests/hop_run/hop-config.json
b/integration-tests/hop_run/hop-config.json
new file mode 100644
index 0000000000..d9e1e6562e
--- /dev/null
+++ b/integration-tests/hop_run/hop-config.json
@@ -0,0 +1,290 @@
+{
+ "variables": [
+ {
+ "name": "HOP_LENIENT_STRING_TO_NUMBER_CONVERSION",
+ "value": "N",
+ "description": "System wide flag to allow lenient string to number
conversion for backward compatibility. If this setting is set to \"Y\", an
string starting with digits will be converted successfully into a number.
(example: 192.168.1.1 will be converted into 192 or 192.168 or 192168 depending
on the decimal and grouping symbol). The default (N) will be to throw an error
if non-numeric symbols are found in the string."
+ },
+ {
+ "name": "HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE",
+ "value": "N",
+ "description": "System wide flag to ignore timezone while writing
date/timestamp value to the database."
+ },
+ {
+ "name": "HOP_LOG_SIZE_LIMIT",
+ "value": "0",
+ "description": "The log size limit for all pipelines and workflows that
don't have the \"log size limit\" property set in their respective properties."
+ },
+ {
+ "name": "HOP_EMPTY_STRING_DIFFERS_FROM_NULL",
+ "value": "N",
+ "description": "NULL vs Empty String. If this setting is set to Y, an
empty string and null are different. Otherwise they are not."
+ },
+ {
+ "name": "HOP_MAX_LOG_SIZE_IN_LINES",
+ "value": "0",
+ "description": "The maximum number of log lines that are kept internally
by Hop. Set to 0 to keep all rows (default)"
+ },
+ {
+ "name": "HOP_MAX_LOG_TIMEOUT_IN_MINUTES",
+ "value": "1440",
+ "description": "The maximum age (in minutes) of a log line while being
kept internally by Hop. Set to 0 to keep all rows indefinitely (default)"
+ },
+ {
+ "name": "HOP_MAX_WORKFLOW_TRACKER_SIZE",
+ "value": "5000",
+ "description": "The maximum number of workflow trackers kept in memory"
+ },
+ {
+ "name": "HOP_MAX_ACTIONS_LOGGED",
+ "value": "5000",
+ "description": "The maximum number of action results kept in memory for
logging purposes."
+ },
+ {
+ "name": "HOP_MAX_LOGGING_REGISTRY_SIZE",
+ "value": "10000",
+ "description": "The maximum number of logging registry entries kept in
memory for logging purposes."
+ },
+ {
+ "name": "HOP_LOG_TAB_REFRESH_DELAY",
+ "value": "1000",
+ "description": "The hop log tab refresh delay."
+ },
+ {
+ "name": "HOP_LOG_TAB_REFRESH_PERIOD",
+ "value": "1000",
+ "description": "The hop log tab refresh period."
+ },
+ {
+ "name": "HOP_PLUGIN_CLASSES",
+ "value": null,
+ "description": "A comma delimited list of classes to scan for plugin
annotations"
+ },
+ {
+ "name": "HOP_PLUGIN_PACKAGES",
+ "value": null,
+ "description": "A comma delimited list of packages to scan for plugin
annotations (warning: slow!!)"
+ },
+ {
+ "name": "HOP_TRANSFORM_PERFORMANCE_SNAPSHOT_LIMIT",
+ "value": "0",
+ "description": "The maximum number of transform performance snapshots to
keep in memory. Set to 0 to keep all snapshots indefinitely (default)"
+ },
+ {
+ "name": "HOP_ROWSET_GET_TIMEOUT",
+ "value": "50",
+ "description": "The name of the variable that optionally contains an
alternative rowset get timeout (in ms). This only makes a difference for
extremely short lived pipelines."
+ },
+ {
+ "name": "HOP_ROWSET_PUT_TIMEOUT",
+ "value": "50",
+ "description": "The name of the variable that optionally contains an
alternative rowset put timeout (in ms). This only makes a difference for
extremely short lived pipelines."
+ },
+ {
+ "name": "HOP_CORE_TRANSFORMS_FILE",
+ "value": null,
+ "description": "The name of the project variable that will contain the
alternative location of the hop-transforms.xml file. You can use this to
customize the list of available internal transforms outside of the codebase."
+ },
+ {
+ "name": "HOP_CORE_WORKFLOW_ACTIONS_FILE",
+ "value": null,
+ "description": "The name of the project variable that will contain the
alternative location of the hop-workflow-actions.xml file."
+ },
+ {
+ "name": "HOP_SERVER_OBJECT_TIMEOUT_MINUTES",
+ "value": "1440",
+ "description": "This project variable will set a time-out after which
waiting, completed or stopped pipelines and workflows will be automatically
cleaned up. The default value is 1440 (one day)."
+ },
+ {
+ "name": "HOP_PIPELINE_PAN_JVM_EXIT_CODE",
+ "value": null,
+ "description": "Set this variable to an integer that will be returned as
the Pan JVM exit code."
+ },
+ {
+ "name": "HOP_DISABLE_CONSOLE_LOGGING",
+ "value": "N",
+ "description": "Set this variable to Y to disable standard Hop logging
to the console. (stdout)"
+ },
+ {
+ "name": "HOP_REDIRECT_STDERR",
+ "value": "N",
+ "description": "Set this variable to Y to redirect stderr to Hop
logging."
+ },
+ {
+ "name": "HOP_REDIRECT_STDOUT",
+ "value": "N",
+ "description": "Set this variable to Y to redirect stdout to Hop
logging."
+ },
+ {
+ "name": "HOP_DEFAULT_NUMBER_FORMAT",
+ "value": null,
+ "description": "The name of the variable containing an alternative
default number format"
+ },
+ {
+ "name": "HOP_DEFAULT_BIGNUMBER_FORMAT",
+ "value": null,
+ "description": "The name of the variable containing an alternative
default bignumber format"
+ },
+ {
+ "name": "HOP_DEFAULT_INTEGER_FORMAT",
+ "value": null,
+ "description": "The name of the variable containing an alternative
default integer format"
+ },
+ {
+ "name": "HOP_DEFAULT_DATE_FORMAT",
+ "value": null,
+ "description": "The name of the variable containing an alternative
default date format"
+ },
+ {
+ "name": "HOP_DEFAULT_TIMESTAMP_FORMAT",
+ "value": null,
+ "description": "The name of the variable containing an alternative
default timestamp format"
+ },
+ {
+ "name": "HOP_DEFAULT_SERVLET_ENCODING",
+ "value": null,
+ "description": "Defines the default encoding for servlets, leave it
empty to use Java default encoding"
+ },
+ {
+ "name": "HOP_FAIL_ON_LOGGING_ERROR",
+ "value": "N",
+ "description": "Set this variable to Y when you want the
workflow/pipeline fail with an error when the related logging process (e.g. to
a database) fails."
+ },
+ {
+ "name": "HOP_AGGREGATION_MIN_NULL_IS_VALUED",
+ "value": "N",
+ "description": "Set this variable to Y to set the minimum to NULL if
NULL is within an aggregate. Otherwise by default NULL is ignored by the MIN
aggregate and MIN is set to the minimum value that is not NULL. See also the
variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO."
+ },
+ {
+ "name": "HOP_AGGREGATION_ALL_NULLS_ARE_ZERO",
+ "value": "N",
+ "description": "Set this variable to Y to return 0 when all values
within an aggregate are NULL. Otherwise by default a NULL is returned when all
values are NULL."
+ },
+ {
+ "name": "HOP_COMPATIBILITY_TEXT_FILE_OUTPUT_APPEND_NO_HEADER",
+ "value": "N",
+ "description": "Set this variable to Y for backward compatibility for
the Text File Output transform. Setting this to Ywill add no header row at all
when the append option is enabled, regardless if the file is existing or not."
+ },
+ {
+ "name": "HOP_PASSWORD_ENCODER_PLUGIN",
+ "value": "Hop",
+ "description": "Specifies the password encoder plugin to use by ID (Hop
is the default)."
+ },
+ {
+ "name": "HOP_SYSTEM_HOSTNAME",
+ "value": null,
+ "description": "You can use this variable to speed up hostname lookup.
Hostname lookup is performed by Hop so that it is capable of logging the server
on which a workflow or pipeline is executed."
+ },
+ {
+ "name": "HOP_SERVER_JETTY_ACCEPTORS",
+ "value": null,
+ "description": "A variable to configure jetty option: acceptors for
Carte"
+ },
+ {
+ "name": "HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE",
+ "value": null,
+ "description": "A variable to configure jetty option: acceptQueueSize
for Carte"
+ },
+ {
+ "name": "HOP_SERVER_JETTY_RES_MAX_IDLE_TIME",
+ "value": null,
+ "description": "A variable to configure jetty option:
lowResourcesMaxIdleTime for Carte"
+ },
+ {
+ "name":
"HOP_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL",
+ "value": "N",
+ "description": "Set this variable to Y for backward compatibility for
the Merge Rows (diff) transform. Setting this to Y will use the data from the
reference stream (instead of the comparison stream) in case the compared rows
are identical."
+ },
+ {
+ "name": "HOP_SPLIT_FIELDS_REMOVE_ENCLOSURE",
+ "value": "false",
+ "description": "Set this variable to false to preserve enclosure symbol
after splitting the string in the Split fields transform. Changing it to true
will remove first and last enclosure symbol from the resulting string chunks."
+ },
+ {
+ "name": "HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES",
+ "value": "false",
+ "description": "Set this variable to TRUE to allow your pipeline to pass
'null' fields and/or empty types."
+ },
+ {
+ "name": "HOP_GLOBAL_LOG_VARIABLES_CLEAR_ON_EXPORT",
+ "value": "false",
+ "description": "Set this variable to false to preserve global log
variables defined in pipeline / workflow Properties -> Log panel. Changing it
to true will clear it when export pipeline / workflow."
+ },
+ {
+ "name": "HOP_FILE_OUTPUT_MAX_STREAM_COUNT",
+ "value": "1024",
+ "description": "This project variable is used by the Text File Output
transform. It defines the max number of simultaneously open files within the
transform. The transform will close/reopen files as necessary to insure the max
is not exceeded"
+ },
+ {
+ "name": "HOP_FILE_OUTPUT_MAX_STREAM_LIFE",
+ "value": "0",
+ "description": "This project variable is used by the Text File Output
transform. It defines the max number of milliseconds between flushes of files
opened by the transform."
+ },
+ {
+ "name": "HOP_USE_NATIVE_FILE_DIALOG",
+ "value": "N",
+ "description": "Set this value to Y if you want to use the system file
open/save dialog when browsing files"
+ },
+ {
+ "name": "HOP_AUTO_CREATE_CONFIG",
+ "value": "Y",
+ "description": "Set this value to N if you don't want to automatically
create a hop configuration file (hop-config.json) when it's missing"
+ }
+ ],
+ "LocaleDefault": "en_BE",
+ "guiProperties": {
+ "FontFixedSize": "13",
+ "MaxUndo": "100",
+ "DarkMode": "Y",
+ "FontNoteSize": "13",
+ "ShowOSLook": "Y",
+ "FontFixedStyle": "0",
+ "FontNoteName": ".AppleSystemUIFont",
+ "FontFixedName": "Monospaced",
+ "FontGraphStyle": "0",
+ "FontDefaultSize": "13",
+ "GraphColorR": "255",
+ "FontGraphSize": "13",
+ "IconSize": "32",
+ "BackgroundColorB": "255",
+ "FontNoteStyle": "0",
+ "FontGraphName": ".AppleSystemUIFont",
+ "FontDefaultName": ".AppleSystemUIFont",
+ "GraphColorG": "255",
+ "UseGlobalFileBookmarks": "Y",
+ "FontDefaultStyle": "0",
+ "GraphColorB": "255",
+ "BackgroundColorR": "255",
+ "BackgroundColorG": "255",
+ "WorkflowDialogStyle": "RESIZE,MAX,MIN",
+ "LineWidth": "1",
+ "ContextDialogShowCategories": "Y"
+ },
+ "projectsConfig": {
+ "enabled": true,
+ "projectMandatory": true,
+ "environmentMandatory": false,
+ "defaultProject": "default",
+ "defaultEnvironment": null,
+ "standardParentProject": "default",
+ "standardProjectsFolder": null,
+ "projectConfigurations": [
+ {
+ "projectName": "default",
+ "projectHome": "${HOP_CONFIG_FOLDER}",
+ "configFilename": "project-config.json"
+ }
+ ],
+ "lifecycleEnvironments": [
+ {
+ "name": "dev",
+ "purpose": "Testing",
+ "projectName": "default",
+ "configurationFiles": [
+ "${PROJECT_HOME}/dev-env-config.json"
+ ]
+ }
+ ],
+ "projectLifecycles": []
+ }
+}
\ No newline at end of file
diff --git a/integration-tests/hop_run/main-0001-sigterm-stops-workflow.hwf
b/integration-tests/hop_run/main-0001-sigterm-stops-workflow.hwf
new file mode 100644
index 0000000000..079cef8b76
--- /dev/null
+++ b/integration-tests/hop_run/main-0001-sigterm-stops-workflow.hwf
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0001-sigterm-stops-workflow</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>SIGTERM to hop-run stops the running workflow before the JVM
exits (issue #9168).</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/10/30 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/10/30 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Send SIGTERM</name>
+ <description>Runs 0001-run-until-stopped.hwf until it is stopped, sends
SIGTERM and checks that hop-run stopped the run before it exited. The script
logs why it failed.</description>
+ <type>SHELL</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/sigterm-hop-run.sh</filename>
+ <work_directory>${PROJECT_HOME}</work_directory>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>N</insertScript>
+ <script/>
+ <arguments>
+ <argument>
+ <value>0001-run-until-stopped.hwf</value>
+ <hidden>N</hidden>
+ </argument>
+ </arguments>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>512</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <always_log_rows>N</always_log_rows>
+ <message>hop-run did not stop the run on SIGTERM, see the output of the
script above</message>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>240</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Send SIGTERM</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/hop_run/main-0002-sigterm-stops-pipeline.hwf
b/integration-tests/hop_run/main-0002-sigterm-stops-pipeline.hwf
new file mode 100644
index 0000000000..05f4b2eb0f
--- /dev/null
+++ b/integration-tests/hop_run/main-0002-sigterm-stops-pipeline.hwf
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0002-sigterm-stops-pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>SIGTERM to hop-run stops the running pipeline before the JVM
exits (issue #9168).</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/10/30 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/10/30 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Send SIGTERM</name>
+ <description>Runs 0002-run-until-stopped.hpl until it is stopped, sends
SIGTERM and checks that hop-run stopped the run before it exited. The script
logs why it failed.</description>
+ <type>SHELL</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/sigterm-hop-run.sh</filename>
+ <work_directory>${PROJECT_HOME}</work_directory>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>N</insertScript>
+ <script/>
+ <arguments>
+ <argument>
+ <value>0002-run-until-stopped.hpl</value>
+ <hidden>N</hidden>
+ </argument>
+ </arguments>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>512</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <always_log_rows>N</always_log_rows>
+ <message>hop-run did not stop the run on SIGTERM, see the output of the
script above</message>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>240</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Send SIGTERM</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/hop_run/main-0003-docker-entrypoint-forwards-sigterm.hwf
b/integration-tests/hop_run/main-0003-docker-entrypoint-forwards-sigterm.hwf
new file mode 100644
index 0000000000..f8dc04d4e9
--- /dev/null
+++ b/integration-tests/hop_run/main-0003-docker-entrypoint-forwards-sigterm.hwf
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0003-docker-entrypoint-forwards-sigterm</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>SIGTERM to the entry point of the Hop Docker image reaches
hop-run, which stops the running pipeline before the JVM exits (issue
#9168).</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/10/30 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/10/30 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Send SIGTERM</name>
+ <description>Runs 0002-run-until-stopped.hpl until it is stopped, sends
SIGTERM and checks that hop-run stopped the run before it exited. The script
logs why it failed.</description>
+ <type>SHELL</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/sigterm-docker-entrypoint.sh</filename>
+ <work_directory>${PROJECT_HOME}</work_directory>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>N</insertScript>
+ <script/>
+ <arguments>
+ <argument>
+ <value>0002-run-until-stopped.hpl</value>
+ <hidden>N</hidden>
+ </argument>
+ </arguments>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>512</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <always_log_rows>N</always_log_rows>
+ <message>hop-run did not stop the run on SIGTERM, see the output of the
script above</message>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>240</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Send SIGTERM</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/hop_run/metadata/pipeline-run-configuration/local.json
b/integration-tests/hop_run/metadata/pipeline-run-configuration/local.json
new file mode 100644
index 0000000000..63794efcaf
--- /dev/null
+++ b/integration-tests/hop_run/metadata/pipeline-run-configuration/local.json
@@ -0,0 +1,17 @@
+{
+ "engineRunConfiguration": {
+ "Local": {
+ "feedback_size": "50000",
+ "sample_size": "100",
+ "sample_type_in_gui": "Last",
+ "rowset_size": "10000",
+ "safe_mode": false,
+ "show_feedback": false,
+ "topo_sort": false,
+ "gather_metrics": false
+ }
+ },
+ "configurationVariables": [],
+ "name": "local",
+ "description": "Runs your pipelines locally with the standard local Hop
pipeline engine"
+}
\ No newline at end of file
diff --git
a/integration-tests/hop_run/metadata/workflow-run-configuration/local.json
b/integration-tests/hop_run/metadata/workflow-run-configuration/local.json
new file mode 100644
index 0000000000..e37a93039a
--- /dev/null
+++ b/integration-tests/hop_run/metadata/workflow-run-configuration/local.json
@@ -0,0 +1,9 @@
+{
+ "engineRunConfiguration": {
+ "Local": {
+ "safe_mode": false
+ }
+ },
+ "name": "local",
+ "description": "Runs your workflows locally with the standard local Hop
workflow engine"
+}
\ No newline at end of file
diff --git a/integration-tests/hop_run/project-config.json
b/integration-tests/hop_run/project-config.json
new file mode 100644
index 0000000000..6a91171e1c
--- /dev/null
+++ b/integration-tests/hop_run/project-config.json
@@ -0,0 +1,13 @@
+{
+ "metadataBaseFolder" : "${PROJECT_HOME}/metadata",
+ "unitTestsBasePath" : "${PROJECT_HOME}",
+ "dataSetsCsvFolder" : "${PROJECT_HOME}/datasets",
+ "enforcingExecutionInHome" : true,
+ "config" : {
+ "variables" : [ {
+ "name" : "HOP_LICENSE_HEADER_FILE",
+ "value" : "${PROJECT_HOME}/../asf-header.txt",
+ "description" : "This will automatically serialize the ASF license
header into pipelines and workflows in the integration test projects"
+ } ]
+ }
+}
\ No newline at end of file
diff --git a/integration-tests/hop_run/sigterm-checks.sh
b/integration-tests/hop_run/sigterm-checks.sh
new file mode 100755
index 0000000000..035fd4edb4
--- /dev/null
+++ b/integration-tests/hop_run/sigterm-checks.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Checks shared by the SIGTERM tests. Expects LOG, PID and FILE to be set.
+
+fail() {
+ echo "FAILED: ${1}"
+ echo "----- output -----"
+ cat "${LOG}"
+ kill -KILL "${PID}" 2>/dev/null
+ pkill -KILL -f "org.apache.hop.run.HopRun.*${FILE}" 2>/dev/null
+ exit 1
+}
+
+# waitForLog <log file> <extended regular expression> <seconds>
+waitForLog() {
+ local waited=0
+ until grep -q -E "${2}" "${1}"; do
+ kill -0 "${PID}" 2>/dev/null || fail "the process ended before '${2}' was
logged"
+ [ "${waited}" -lt "${3}" ] || fail "'${2}' was not logged within ${3}s"
+ sleep 1
+ waited=$((waited + 1))
+ done
+}
+
+# checkStopped <log file> <exit code> <seconds it took to stop>
+checkStopped() {
+ # 128 + 15: the JVM exits with the status of the signal that ended it.
+ [ "${2}" -eq 143 ] || fail "the exit code is ${2}, expected 143"
+ grep -q "Termination signal received, stopping the run" "${1}" ||
+ fail "hop-run did not log that it got the termination signal"
+ grep -q "The run has stopped, hop-run exits." "${1}" ||
+ fail "hop-run did not log that the run stopped"
+ [ "${3}" -lt 20 ] || fail "stopping took ${3}s"
+ if pgrep -f "org.apache.hop.run.HopRun.*${FILE}" >/dev/null; then
+ fail "the hop-run JVM is still running"
+ fi
+}
diff --git a/integration-tests/hop_run/sigterm-docker-entrypoint.sh
b/integration-tests/hop_run/sigterm-docker-entrypoint.sh
new file mode 100755
index 0000000000..df8867c047
--- /dev/null
+++ b/integration-tests/hop_run/sigterm-docker-entrypoint.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Runs a pipeline or workflow ($1) that only ends when it is stopped through
the entry point of
+# the Hop Docker image (docker/resources/run.sh, mounted on
/docker-resources), sends the entry
+# point SIGTERM like docker stop or a Kubernetes pod eviction does, and checks
that the signal
+# reached the JVM and that hop-run stopped the run before it exited (issue
#9168).
+
+set -u
+
+FILE="${1}"
+DOCKER_RESOURCES="${DOCKER_RESOURCES:-/docker-resources}"
+WORK="$(mktemp -d)"
+LOG="${WORK}/entrypoint.log"
+
+# The environment of the Hop image (docker/Dockerfile), with a project for
this test.
+DEPLOYMENT_PATH="$(dirname "$(command -v hop-run.sh)")"
+export DEPLOYMENT_PATH
+mkdir -p "${WORK}/config"
+cp "${DEPLOYMENT_PATH}/config/hop-config.json" "${WORK}/config/"
+export HOP_CONFIG_FOLDER="${WORK}/config"
+export HOP_LOG_PATH="${WORK}/hop.err.log"
+export HOP_LOG_LEVEL=Basic
+export HOP_FILE_PATH="${PROJECT_HOME}/${FILE}"
+export HOP_PROJECT_NAME=hop-run-sigterm
+export HOP_PROJECT_FOLDER="${PROJECT_HOME}"
+export HOP_PROJECT_CONFIG_FILE_NAME=project-config.json
+export HOP_PARENT_PROJECT_CONFIG_FILE_NAME=project-config.json
+export HOP_ENVIRONMENT_NAME=environment1
+export HOP_RUN_CONFIG=local
+for variable in HOP_SHARED_JDBC_FOLDERS HOP_PROJECT_DIRECTORY
HOP_PARENT_PROJECT_NAME \
+ HOP_PARENT_PROJECT_FOLDER HOP_ENVIRONMENT_CONFIG_FILE_NAME_PATHS HOP_COMMAND
\
+ HOP_COMMAND_PARAMETERS HOP_RUN_PARAMETERS HOP_START_ACTION
HOP_RUN_METADATA_EXPORT \
+ HOP_SYSTEM_PROPERTIES HOP_CUSTOM_ENTRYPOINT_EXTENSION_SHELL_FILE_PATH
HOP_CONFIG_OPTIONS \
+ HOP_SERVER_METADATA_FOLDER HOP_SERVER_KEYSTORE HOP_SERVER_KEYSTORE_PASSWORD \
+ HOP_SERVER_KEY_PASSWORD HOP_SERVER_MAX_LOG_LINES HOP_SERVER_MAX_LOG_TIMEOUT \
+ HOP_SERVER_MAX_OBJECT_TIMEOUT; do
+ export "${variable}="
+done
+
+bash "${DOCKER_RESOURCES}/run.sh" >"${LOG}" 2>&1 &
+PID=$!
+
+# shellcheck source=/dev/null
+. "${PROJECT_HOME}/sigterm-checks.sh"
+
+waitForLog "${LOG}" "Starting (pipeline|workflow): " 180
+
+# Let the run get going.
+sleep 3
+
+# docker stop signals the entry point, PID 1 in the container.
+START=$(date +%s)
+kill -TERM "${PID}"
+wait "${PID}"
+CODE=$?
+ELAPSED=$(($(date +%s) - START))
+
+checkStopped "${LOG}" "${CODE}" "${ELAPSED}"
+grep -q "The run has stopped, hop-run exits." "${HOP_LOG_PATH}" ||
+ fail "the log file ${HOP_LOG_PATH} misses the end of the run"
+echo "The Docker entry point stopped ${FILE} in ${ELAPSED}s after SIGTERM"
diff --git a/integration-tests/hop_run/sigterm-hop-run.sh
b/integration-tests/hop_run/sigterm-hop-run.sh
new file mode 100755
index 0000000000..ee032d5492
--- /dev/null
+++ b/integration-tests/hop_run/sigterm-hop-run.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Starts hop-run.sh on a pipeline or workflow ($1) that only ends when it is
stopped, sends it
+# SIGTERM and checks that hop-run stopped the run before it exited (issue
#9168).
+
+set -u
+
+FILE="${1}"
+WORK="$(mktemp -d)"
+LOG="${WORK}/hop-run.log"
+
+hop-run.sh -e dev -r local -l Basic -f "${PROJECT_HOME}/${FILE}" >"${LOG}"
2>&1 &
+PID=$!
+
+# shellcheck source=/dev/null
+. "${PROJECT_HOME}/sigterm-checks.sh"
+
+waitForLog "${LOG}" "Starting (pipeline|workflow): " 120
+
+# hop-run.sh execs the JVM, so the signal goes to the JVM itself.
+COMMAND="$(ps -o comm= -p "${PID}")"
+[ "${COMMAND}" = "java" ] || fail "hop-run.sh did not exec java, the process
is '${COMMAND}'"
+
+# Let the run get going.
+sleep 3
+
+START=$(date +%s)
+kill -TERM "${PID}"
+wait "${PID}"
+CODE=$?
+ELAPSED=$(($(date +%s) - START))
+
+checkStopped "${LOG}" "${CODE}" "${ELAPSED}"
+echo "hop-run stopped ${FILE} in ${ELAPSED}s after SIGTERM"
diff --git a/integration-tests/openlineage/0012-openlineage-sigterm.hpl
b/integration-tests/openlineage/0012-openlineage-sigterm.hpl
new file mode 100644
index 0000000000..7658176d1d
--- /dev/null
+++ b/integration-tests/openlineage/0012-openlineage-sigterm.hpl
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0012-openlineage-sigterm</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Generates a row every 100ms until it is stopped</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>Dummy</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <limit>10</limit>
+ <never_ending>Y</never_ending>
+ <interval_in_ms>100</interval_in_ms>
+ <row_time_field/>
+ <last_time_field/>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Dummy</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/openlineage/0012-openlineage-sigterm.sh
b/integration-tests/openlineage/0012-openlineage-sigterm.sh
new file mode 100755
index 0000000000..9853e4af78
--- /dev/null
+++ b/integration-tests/openlineage/0012-openlineage-sigterm.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Runs a pipeline that only ends when it is stopped, sends hop-run SIGTERM and
checks that the
+# end of the run still reaches Marquez: the run may not be left RUNNING there
(issue #9168).
+#
+# Arguments: <Marquez base URL> <OpenLineage namespace>
+
+set -u
+
+MARQUEZ_API="${1}"
+NAMESPACE="${2}"
+JOB="0012-openlineage-sigterm"
+RUNS_URL="${MARQUEZ_API}/api/v1/namespaces/${NAMESPACE}/jobs/${JOB}/runs?limit=1"
+WORK="$(mktemp -d)"
+LOG="${WORK}/hop-run.log"
+
+fail() {
+ echo "FAILED: ${1}"
+ echo "----- lineage environment -----"
+ env | grep '^HOP_LINEAGE' | sort
+ echo "----- hop-run output -----"
+ cat "${LOG}"
+ kill -KILL "${PID}" 2>/dev/null
+ exit 1
+}
+
+# The state and id of the most recent run of the job in Marquez, empty when
there is none.
+latestRun() {
+ curl -s "${RUNS_URL}" |
+ python3 -c 'import json, sys
+try:
+ runs = json.load(sys.stdin).get("runs", [])
+except ValueError:
+ runs = []
+print(runs[0]["state"] + " " + runs[0]["id"] if runs else "")'
+}
+
+# The Shell action hands this script the Hop variables of the workflow, which
replace the lineage
+# settings of the container
(docker/integration-tests/integration-tests-openlineage.yaml) with
+# their defaults. Set the two that switch the OpenLineage sink on again.
+export HOP_LINEAGE_ENABLED=Y
+export HOP_LINEAGE_SINK_IDS=openlineage
+
+BEFORE="$(latestRun)"
+
+hop-run.sh -e dev -r local -l Basic -f "${PROJECT_HOME}/${JOB}.hpl" >"${LOG}"
2>&1 &
+PID=$!
+
+# Wait until Marquez has the START of this run.
+RUN=""
+for _ in $(seq 1 120); do
+ kill -0 "${PID}" 2>/dev/null || fail "hop-run ended before the run reached
Marquez"
+ LATEST="$(latestRun)"
+ if [ -n "${LATEST}" ] && [ "${LATEST}" != "${BEFORE}" ] && [ "${LATEST%% *}"
= "RUNNING" ]; then
+ RUN="${LATEST#* }"
+ break
+ fi
+ sleep 1
+done
+[ -n "${RUN}" ] || fail "Marquez did not show the run as RUNNING within 120s"
+
+kill -TERM "${PID}"
+wait "${PID}"
+CODE=$?
+[ "${CODE}" -eq 143 ] || fail "the exit code is ${CODE}, expected 143"
+
+# hop-run delivered its lineage before it exited; give Marquez a moment to
store it.
+STATE=""
+for _ in $(seq 1 10); do
+ LATEST="$(latestRun)"
+ if [ "${LATEST#* }" = "${RUN}" ]; then
+ STATE="${LATEST%% *}"
+ [ "${STATE}" != "RUNNING" ] && break
+ fi
+ sleep 1
+done
+case "${STATE}" in
+COMPLETED | ABORTED | FAILED) ;;
+*) fail "run ${RUN} is '${STATE}' in Marquez after hop-run exited, the end of
the run was lost" ;;
+esac
+
+echo "Marquez has run ${RUN} of ${JOB} as ${STATE} after SIGTERM"
diff --git a/integration-tests/openlineage/main-0004-openlineage-sigterm.hwf
b/integration-tests/openlineage/main-0004-openlineage-sigterm.hwf
new file mode 100644
index 0000000000..20c9d4ea9c
--- /dev/null
+++ b/integration-tests/openlineage/main-0004-openlineage-sigterm.hwf
@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0004-openlineage-sigterm</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>SIGTERM to hop-run stops the running pipeline and still
delivers the end of the run to Marquez (issue #9168).</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/10/30 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/10/30 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Send SIGTERM</name>
+ <description>Runs 0012-openlineage-sigterm.hpl until it is stopped,
sends SIGTERM and checks that Marquez no longer shows the run as RUNNING. The
script logs why it failed.</description>
+ <type>SHELL</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0012-openlineage-sigterm.sh</filename>
+ <work_directory>${PROJECT_HOME}</work_directory>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>N</insertScript>
+ <script/>
+ <arguments>
+ <argument>
+ <value>${MARQUEZ_API}</value>
+ <hidden>N</hidden>
+ </argument>
+ <argument>
+ <value>${OPENLINEAGE_NAMESPACE}</value>
+ <hidden>N</hidden>
+ </argument>
+ </arguments>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>512</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <always_log_rows>N</always_log_rows>
+ <message>The end of a run stopped by SIGTERM did not reach Marquez, see
the output of the script above</message>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>240</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Send SIGTERM</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Send SIGTERM</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/scripts/run-tests.sh
b/integration-tests/scripts/run-tests.sh
index 5c6ddcc8b8..67d2af11cb 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -294,27 +294,16 @@ run_hop_with_watchdog() {
echo "Sending SIGQUIT to the JVM for a thread dump, then terminating it."
echo "${SPACER}"
- # hop-run.sh does not exec java, so the JVM is a child of the shell we
started.
- local jvm_pid
- jvm_pid=$(pgrep -P "${runner_pid}" java | head -n1)
- if [ -z "${jvm_pid}" ]; then
- jvm_pid=$(pgrep -f 'org.apache.hop.run.HopRun' | head -n1)
- fi
- if [ -n "${jvm_pid}" ]; then
- # SIGQUIT makes the JVM print every thread's stack on its stdout,
which tee captures.
- # Twice: two dumps a few seconds apart show whether anything is moving
at all.
- kill -QUIT "${jvm_pid}" 2>/dev/null || true
- sleep 15
- kill -QUIT "${jvm_pid}" 2>/dev/null || true
- sleep 10
- kill -TERM "${jvm_pid}" 2>/dev/null || true
- sleep 5
- kill -KILL "${jvm_pid}" 2>/dev/null || true
- else
- echo "WARNING: could not find the hop-run JVM process, killing the
wrapper only"
- fi
-
+ # hop-run.sh execs java, so the process we started is the JVM.
+ # SIGQUIT makes the JVM print every thread's stack on its stdout, which
tee captures.
+ # Twice: two dumps a few seconds apart show whether anything is moving
at all.
+ kill -QUIT "${runner_pid}" 2>/dev/null || true
+ sleep 15
+ kill -QUIT "${runner_pid}" 2>/dev/null || true
+ sleep 10
kill -TERM "${runner_pid}" 2>/dev/null || true
+ sleep 5
+ kill -KILL "${runner_pid}" 2>/dev/null || true
wait "${runner_pid}" 2>/dev/null
# The surefire report is built from these files, so say there why the
run has no result.