This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 839ff52c67 issue #7516 : fix integration tests broken by single-JVM
runner (#7517)
839ff52c67 is described below
commit 839ff52c672dabba915f1afc9f9dde48e6503e3a
Author: Matt Casters <[email protected]>
AuthorDate: Mon Jul 13 20:04:13 2026 +0200
issue #7516 : fix integration tests broken by single-JVM runner (#7517)
Preserve project/environment variables when hop-run starts a pipeline
(do not re-initialize from null after the factory already applied them).
Prefer existing variables over empty parameter defaults on activate, and
pass parent variables into nested workflow parameters. Use hop-local for
the suite runner on Beam projects, configure AES2 encoder in the aes IT
project, and update 0083 expected WorkflowExecutor execution-result fields.
---
.../hop/core/parameters/NamedParameters.java | 15 +++++++++--
.../core/parameters/NamedParamsDefaultTest.java | 30 ++++++++++++++++++++++
.../main/java/org/apache/hop/run/HopRunBase.java | 5 +++-
integration-tests/aes/dev-env-config.json | 8 ++++++
integration-tests/aes/hop-config.json | 4 +--
integration-tests/scripts/run-tests.sh | 18 ++++++++++---
.../transforms/0083-workflow-executor-run-wf.hpl | 2 +-
.../workflow/actions/pipeline/ActionPipeline.java | 3 ++-
.../workflow/actions/workflow/ActionWorkflow.java | 8 ++++++
9 files changed, 82 insertions(+), 11 deletions(-)
diff --git
a/core/src/main/java/org/apache/hop/core/parameters/NamedParameters.java
b/core/src/main/java/org/apache/hop/core/parameters/NamedParameters.java
index 2bda141187..24c341dcc6 100644
--- a/core/src/main/java/org/apache/hop/core/parameters/NamedParameters.java
+++ b/core/src/main/java/org/apache/hop/core/parameters/NamedParameters.java
@@ -137,8 +137,19 @@ public class NamedParameters implements INamedParameters {
public void activateParameters(IVariables variables) {
for (NamedParameter param : params.values()) {
if (StringUtils.isNotEmpty(param.key)) {
- variables.setVariable(
- param.getKey(), Const.NVL(param.getValue(),
Const.NVL(param.getDefaultValue(), "")));
+ String value = param.getValue();
+ if (StringUtils.isEmpty(value)) {
+ // Prefer an already-set variable (environment / parent) over the
parameter default.
+ // Nested workflows and pipelines used to clobber env values such as
HOSTNAME or
+ // BOOTSTRAP_SERVERS when activateParameters applied empty defaults.
+ String existing = variables != null ?
variables.getVariable(param.getKey()) : null;
+ if (StringUtils.isNotEmpty(existing)) {
+ value = existing;
+ } else {
+ value = Const.NVL(param.getDefaultValue(), "");
+ }
+ }
+ variables.setVariable(param.getKey(), value);
}
}
}
diff --git
a/core/src/test/java/org/apache/hop/core/parameters/NamedParamsDefaultTest.java
b/core/src/test/java/org/apache/hop/core/parameters/NamedParamsDefaultTest.java
index fa050fc01a..252630fb1f 100644
---
a/core/src/test/java/org/apache/hop/core/parameters/NamedParamsDefaultTest.java
+++
b/core/src/test/java/org/apache/hop/core/parameters/NamedParamsDefaultTest.java
@@ -78,6 +78,36 @@ class NamedParamsDefaultTest {
assertEquals("CCC", variables.getVariable("var3"));
}
+ @Test
+ void testActivateParametersPrefersExistingVariableOverDefault() throws
Exception {
+ Variables variables = new Variables();
+ variables.setVariable("HOSTNAME", "http");
+ variables.setVariable("BOOTSTRAP_SERVERS", "kafka:9092");
+
+ namedParams.addParameterDefinition("HOSTNAME", "localhost", "HTTP host");
+ namedParams.addParameterDefinition("BOOTSTRAP_SERVERS", "", "Kafka
bootstrap");
+ namedParams.addParameterDefinition("ONLY_DEFAULT", "from-default", "No
prior variable");
+
+ namedParams.activateParameters(variables);
+
+ assertEquals("http", variables.getVariable("HOSTNAME"));
+ assertEquals("kafka:9092", variables.getVariable("BOOTSTRAP_SERVERS"));
+ assertEquals("from-default", variables.getVariable("ONLY_DEFAULT"));
+ }
+
+ @Test
+ void testActivateParametersExplicitValueWinsOverExistingVariable() throws
Exception {
+ Variables variables = new Variables();
+ variables.setVariable("HOSTNAME", "http");
+
+ namedParams.addParameterDefinition("HOSTNAME", "localhost", "HTTP host");
+ namedParams.setParameterValue("HOSTNAME", "explicit-host");
+
+ namedParams.activateParameters(variables);
+
+ assertEquals("explicit-host", variables.getVariable("HOSTNAME"));
+ }
+
@Test
void testAddParameterDefinitionWithException() throws
DuplicateParamException {
namedParams.addParameterDefinition("key", "value", "description");
diff --git a/engine/src/main/java/org/apache/hop/run/HopRunBase.java
b/engine/src/main/java/org/apache/hop/run/HopRunBase.java
index ebf6edb64c..7ce773bc8f 100644
--- a/engine/src/main/java/org/apache/hop/run/HopRunBase.java
+++ b/engine/src/main/java/org/apache/hop/run/HopRunBase.java
@@ -290,7 +290,10 @@ public abstract class HopRunBase implements Runnable,
IHasHopMetadataProvider {
PipelineEngineFactory.createPipelineEngine(
variables, pipelineRunConfigurationName, metadataProvider,
pipelineMeta);
pipeline.getPipelineMeta().setInternalHopVariables(pipeline);
- pipeline.initializeFrom(null);
+ // Do not call initializeFrom(null) here: the factory already
initialized the engine from
+ // hop-run's variables (project + environment). Re-initializing from
null would drop those
+ // values (e.g. HOSTNAME, TEST_0012) and only re-apply hop-config
defaults — breaking the
+ // single-JVM IT suite runner and any hop-run of a pipeline with -e.
List<EngineCompatibilityChecker.Violation> pipelineViolations =
EngineCompatibilityChecker.checkPipeline(pipelineMeta, pipeline);
diff --git a/integration-tests/aes/dev-env-config.json
b/integration-tests/aes/dev-env-config.json
index 09dfeb99fa..471c885669 100644
--- a/integration-tests/aes/dev-env-config.json
+++ b/integration-tests/aes/dev-env-config.json
@@ -3,5 +3,13 @@
"name" : "AES2_PASSWORD",
"value" : "AES2 5SPGXV3SEPhoDRiYIxBv+wtnSC7WQYoYU6zyJx1Y75sEzHAM",
"description" : ""
+ }, {
+ "name" : "HOP_PASSWORD_ENCODER_PLUGIN",
+ "value" : "AES2",
+ "description" : "AES2 encoder required to decrypt AES2_PASSWORD in this
suite"
+ }, {
+ "name" : "HOP_AES_ENCODER_KEY",
+ "value" : "abcdef",
+ "description" : "Key matching the ciphertext stored in AES2_PASSWORD"
} ]
}
\ No newline at end of file
diff --git a/integration-tests/aes/hop-config.json
b/integration-tests/aes/hop-config.json
index 8a188da4ab..064aa482fb 100644
--- a/integration-tests/aes/hop-config.json
+++ b/integration-tests/aes/hop-config.json
@@ -167,8 +167,8 @@
},
{
"name": "HOP_PASSWORD_ENCODER_PLUGIN",
- "value": "Hop",
- "description": "Specifies the password encoder plugin to use by ID (Hop
is the default)."
+ "value": "AES2",
+ "description": "AES2 encoder required by the aes integration-test
project (matches docker env)."
},
{
"name": "HOP_SYSTEM_HOSTNAME",
diff --git a/integration-tests/scripts/run-tests.sh
b/integration-tests/scripts/run-tests.sh
index c2072ca497..ccbbb8147a 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -137,9 +137,8 @@ TMP_CONFIG_FOLDER="${HOP_CONFIG_FOLDER}"
SUREFIRE_DIR="$(cd "${CURRENT_DIR}/../surefire-reports" && pwd)"
RUNNER_PIPELINE="${CURRENT_DIR}/run-project-tests.hpl"
-# Shared hop-run parameters for both single-JVM and per-test modes
+# Shared hop-run parameters for both single-JVM and per-test modes (run
configuration added per project)
HOP_RUN_COMMON_ARGS=(
- -r "local"
-e "dev"
-p "POSTGRES_HOST=${POSTGRES_HOST}"
-p "POSTGRES_DATABASE=${POSTGRES_DATABASE}"
@@ -180,16 +179,26 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
# Create New Project
export HOP_CONFIG_FOLDER="$d"
+ # Default pipeline run configuration name used by hop-run and the suite
runner.
+ # Beam projects name their Beam engine "local" and keep a native Local
engine as "hop-local".
+ # The single-JVM suite driver (run-project-tests.hpl) must never run
under Beam.
+ PIPELINE_RUN_CONFIG="local"
+ SUITE_RUN_CONFIG="local"
+ if [ -f "$d/metadata/pipeline-run-configuration/hop-local.json" ]; then
+ SUITE_RUN_CONFIG="hop-local"
+ fi
+
# Prefer single-JVM suite runner when available (unless isolation mode
is requested)
if [ "${HOP_IT_PER_TEST_JVM}" != "true" ] && [ -f "${RUNNER_PIPELINE}"
]; then
echo ${SPACER}
- echo "Running project tests in single JVM via run-project-tests.hpl"
+ echo "Running project tests in single JVM via run-project-tests.hpl
(run config: ${SUITE_RUN_CONFIG})"
echo ${SPACER}
start_time_test=$SECONDS
$HOP_LOCATION/hop-run.sh \
+ -r "${SUITE_RUN_CONFIG}" \
"${HOP_RUN_COMMON_ARGS[@]}" \
-p "PROJECT_NAME=${PROJECT_NAME}" \
-p "IT_SUREFIRE_DIR=${SUREFIRE_DIR}" \
@@ -252,8 +261,9 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
#Start time test
start_time_test=$SECONDS
- #Run Test
+ #Run Test (use project pipeline run config, e.g. Beam "local")
$HOP_LOCATION/hop-run.sh \
+ -r "${PIPELINE_RUN_CONFIG}" \
"${HOP_RUN_COMMON_ARGS[@]}" \
-f "$hop_file" > >(tee /tmp/test_output) 2> >(tee
/tmp/test_output_err >&1)
diff --git a/integration-tests/transforms/0083-workflow-executor-run-wf.hpl
b/integration-tests/transforms/0083-workflow-executor-run-wf.hpl
index a39c2379b9..d70a870c9c 100644
--- a/integration-tests/transforms/0083-workflow-executor-run-wf.hpl
+++ b/integration-tests/transforms/0083-workflow-executor-run-wf.hpl
@@ -115,7 +115,7 @@ limitations under the License.
<group/>
<length>-1</length>
<name>expected</name>
- <nullif>ExecutionTime, ExecutionResult, ExecutionNrErrors,
ExecutionLinesRead, ExecutionLinesWritten, ExecutionLinesInput,
ExecutionLinesOutput, ExecutionLinesRejected, ExecutionLinesUpdated,
ExecutionLinesDeleted, ExecutionFilesRetrieved, ExecutionExitStatus,
ExecutionLogText, ExecutionLogChannelId</nullif>
+ <nullif>f1, ExecutionTime, ExecutionResult, ExecutionNrErrors,
ExecutionLinesRead, ExecutionLinesWritten, ExecutionLinesInput,
ExecutionLinesOutput, ExecutionLinesRejected, ExecutionLinesUpdated,
ExecutionLinesDeleted, ExecutionFilesRetrieved, ExecutionExitStatus,
ExecutionLogText, ExecutionLogChannelId</nullif>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<type>String</type>
diff --git
a/plugins/actions/pipeline/src/main/java/org/apache/hop/workflow/actions/pipeline/ActionPipeline.java
b/plugins/actions/pipeline/src/main/java/org/apache/hop/workflow/actions/pipeline/ActionPipeline.java
index 1201b5b004..55e4a52ba4 100644
---
a/plugins/actions/pipeline/src/main/java/org/apache/hop/workflow/actions/pipeline/ActionPipeline.java
+++
b/plugins/actions/pipeline/src/main/java/org/apache/hop/workflow/actions/pipeline/ActionPipeline.java
@@ -522,8 +522,9 @@ public class ActionPipeline extends ActionBase implements
Cloneable, IAction {
pipeline.setMetadataProvider(getMetadataProvider());
// Handle parameters...
+ // Factory already initialized the pipeline from this action's
variables (parent workflow
+ // + env). Do not re-initialize from null or environment variables are
dropped.
//
- pipeline.initializeFrom(null);
pipeline.copyParametersFromDefinitions(pipelineMeta);
// Pass the parameter values and activate...
diff --git
a/plugins/actions/workflow/src/main/java/org/apache/hop/workflow/actions/workflow/ActionWorkflow.java
b/plugins/actions/workflow/src/main/java/org/apache/hop/workflow/actions/workflow/ActionWorkflow.java
index 29807ab860..f26323b2b3 100644
---
a/plugins/actions/workflow/src/main/java/org/apache/hop/workflow/actions/workflow/ActionWorkflow.java
+++
b/plugins/actions/workflow/src/main/java/org/apache/hop/workflow/actions/workflow/ActionWorkflow.java
@@ -479,6 +479,14 @@ public class ActionWorkflow extends ActionBase implements
Cloneable, IAction {
String parentValue =
parentWorkflow.getParameterValue(parameterName);
if (!Utils.isEmpty(parentValue)) {
workflow.setParameterValue(parameterName, parentValue);
+ } else {
+ // Match hop-run: if the parent has a variable of the same
name (env, -p as
+ // variable, project config), use it so nested main-*.hwf
parameters do not
+ // fall back to their local-dev defaults (e.g.
HOSTNAME=localhost).
+ String parentVariable =
parentWorkflow.getVariable(parameterName);
+ if (!Utils.isEmpty(parentVariable)) {
+ workflow.setParameterValue(parameterName, parentVariable);
+ }
}
}
}
