This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 9b18f3b66e Reflection and Testing listeners leave engines other than
Hop's local… (#9565)
9b18f3b66e is described below
commit 9b18f3b66ec1d6885282b6b39250e204bc7b6b96
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 19 11:05:36 2026 +0100
Reflection and Testing listeners leave engines other than Hop's local…
(#9565)
* Reflection and Testing listeners leave engines other than Hop's local one
alone, fixes #9563
* fix remarks
---
.../pipeline/xp/PipelineStartLoggingXp.java | 9 +-
.../reflection/probe/xp/PipelineDataProbeXp.java | 9 +-
.../hop/reflection/xp/ReflectionXpEngineTest.java | 79 +++++++++++++
.../xp/ValidatePipelineUnitTestExtensionPoint.java | 5 +
.../testing/xp/WriteToDataSetExtensionPoint.java | 11 ++
.../testing/xp/UnitTestListenersEngineTest.java | 127 +++++++++++++++++++++
6 files changed, 236 insertions(+), 4 deletions(-)
diff --git
a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/pipeline/xp/PipelineStartLoggingXp.java
b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/pipeline/xp/PipelineStartLoggingXp.java
index dae81a222f..04ca9cbab3 100644
---
a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/pipeline/xp/PipelineStartLoggingXp.java
+++
b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/pipeline/xp/PipelineStartLoggingXp.java
@@ -45,15 +45,20 @@ import
org.apache.hop.reflection.pipeline.transform.PipelineLogging;
id = "PipelineStartLoggingXp",
extensionPointId = "PipelinePrepareExecution",
description = "At the start of a pipeline, handle any Pipeline Log
metadata objects")
-public class PipelineStartLoggingXp implements IExtensionPoint<Pipeline> {
+public class PipelineStartLoggingXp implements
IExtensionPoint<IPipelineEngine<PipelineMeta>> {
private static final String
CONST_ERROR_HANDLING_PIPELINE_LOG_METADATA_OBJECT =
"Error handling Pipeline Log metadata object '";
public static final String PIPELINE_LOGGING_FLAG = "PipelineLoggingActive";
@Override
- public void callExtensionPoint(ILogChannel log, IVariables variables,
Pipeline pipeline)
+ public void callExtensionPoint(
+ ILogChannel log, IVariables variables, IPipelineEngine<PipelineMeta>
engine)
throws HopException {
+ // Works on the transforms of Hop's own pipeline engine: Spark, Beam ...
have none
+ if (!(engine instanceof Pipeline pipeline)) {
+ return;
+ }
// Prevent recursive logging of the logging pipeline
//
diff --git
a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/probe/xp/PipelineDataProbeXp.java
b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/probe/xp/PipelineDataProbeXp.java
index f474f3d9d0..88031df5c7 100644
---
a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/probe/xp/PipelineDataProbeXp.java
+++
b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/probe/xp/PipelineDataProbeXp.java
@@ -49,13 +49,18 @@ import
org.apache.hop.reflection.probe.transform.PipelineDataProbe;
id = "PipelineDataProbeXp",
extensionPointId = "PipelineStartThreads",
description = "Before the start of a pipeline, after init, attach the data
probes needed")
-public class PipelineDataProbeXp implements IExtensionPoint<Pipeline> {
+public class PipelineDataProbeXp implements
IExtensionPoint<IPipelineEngine<PipelineMeta>> {
public static final String PIPELINE_DATA_PROBE_FLAG =
"PipelineDataProbeActive";
@Override
- public void callExtensionPoint(ILogChannel log, IVariables variables,
Pipeline pipeline)
+ public void callExtensionPoint(
+ ILogChannel log, IVariables variables, IPipelineEngine<PipelineMeta>
engine)
throws HopException {
+ // Works on the transforms of Hop's own pipeline engine: Spark, Beam ...
have none
+ if (!(engine instanceof Pipeline pipeline)) {
+ return;
+ }
// Prevent recursive data probing of the pipeline
//
diff --git
a/plugins/misc/reflection/src/test/java/org/apache/hop/reflection/xp/ReflectionXpEngineTest.java
b/plugins/misc/reflection/src/test/java/org/apache/hop/reflection/xp/ReflectionXpEngineTest.java
new file mode 100644
index 0000000000..6b035fa58b
--- /dev/null
+++
b/plugins/misc/reflection/src/test/java/org/apache/hop/reflection/xp/ReflectionXpEngineTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.reflection.xp;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hop.core.extension.IExtensionPoint;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.reflection.pipeline.xp.PipelineStartLoggingXp;
+import org.apache.hop.reflection.probe.xp.PipelineDataProbeXp;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The pipeline log and data probe listeners work on Hop's own pipeline
engine. Another engine
+ * firing the same extension points is left alone instead of failing.
+ *
+ * <p>The listeners are called through the raw {@link IExtensionPoint}, as the
extension point
+ * handler calls them, so the engine object is passed whatever its type.
+ */
+class ReflectionXpEngineTest {
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ void anotherPipelineEngineIsLeftAlone() {
+ IPipelineEngine<PipelineMeta> engine = mock(IPipelineEngine.class);
+
+ for (IExtensionPoint extensionPoint :
+ new IExtensionPoint[] {new PipelineStartLoggingXp(), new
PipelineDataProbeXp()}) {
+ assertDoesNotThrow(
+ () -> extensionPoint.callExtensionPoint(LogChannel.GENERAL, new
Variables(), engine),
+ extensionPoint.getClass().getSimpleName());
+ }
+ verifyNoInteractions(engine);
+ }
+
+ /** On the local engine they go on: here up to the check that stops them
logging themselves. */
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ void theLocalEngineIsHandled() throws Exception {
+ LocalPipelineEngine pipeline = mock(LocalPipelineEngine.class);
+ Map<String, Object> extensionData = new HashMap<>();
+ extensionData.put(PipelineStartLoggingXp.PIPELINE_LOGGING_FLAG, true);
+ extensionData.put(PipelineDataProbeXp.PIPELINE_DATA_PROBE_FLAG, true);
+ when(pipeline.getExtensionDataMap()).thenReturn(extensionData);
+
+ ((IExtensionPoint) new PipelineStartLoggingXp())
+ .callExtensionPoint(LogChannel.GENERAL, new Variables(), pipeline);
+ ((IExtensionPoint) new PipelineDataProbeXp())
+ .callExtensionPoint(LogChannel.GENERAL, new Variables(), pipeline);
+
+ verify(pipeline, times(2)).getExtensionDataMap();
+ }
+}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
index 4c32afe1f1..138dbf5e85 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
@@ -35,6 +35,7 @@ import org.apache.hop.lineage.model.UnitTestLineagePayload;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.testing.PipelineUnitTest;
import org.apache.hop.testing.PipelineUnitTestSetLocation;
import org.apache.hop.testing.UnitTestResult;
@@ -62,6 +63,10 @@ public class ValidatePipelineUnitTestExtensionPoint
if (!runUnitTest) {
return;
}
+ // The data set rows come from row listeners on the transforms of the
local engine
+ if (!(pipeline instanceof LocalPipelineEngine)) {
+ return;
+ }
// We should always have a unit test name here...
String unitTestName =
pipeline.getVariable(DataSetConst.VAR_UNIT_TEST_NAME);
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
index 7b7e67fa9b..f973c0a1ca 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
@@ -30,6 +30,7 @@ import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.RowDataUtil;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.engine.IEngineComponent;
import org.apache.hop.pipeline.engine.IPipelineEngine;
@@ -78,6 +79,16 @@ public class WriteToDataSetExtensionPoint
WriteToDataSetExtensionPoint.setsMap.remove(pipelineMeta.getName());
});
+ // The data set rows come from a row listener on a transform of Hop's own
engine (local,
+ // single threaded). Spark, Beam ... have no such transforms: say so
instead of writing nothing.
+ if (!(pipeline instanceof Pipeline)) {
+ log.logError(
+ "Writing rows to a data set is not supported on pipeline engine '"
+ + pipeline.getClass().getSimpleName()
+ + "': no data set was written.");
+ return;
+ }
+
try {
IHopMetadataProvider metadataProvider =
pipelineMeta.getMetadataProvider();
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/xp/UnitTestListenersEngineTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/xp/UnitTestListenersEngineTest.java
new file mode 100644
index 0000000000..45a0850974
--- /dev/null
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/xp/UnitTestListenersEngineTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing.xp;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.contains;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.testing.util.DataSetConst;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Validating a unit test uses row listeners on the transforms of Hop's local
engine, like the other
+ * unit test listeners: on another engine it does nothing, even with the unit
test flags set.
+ * Writing rows to a data set is not part of a unit test and works on any
engine with Hop's own
+ * transforms (local, single threaded). On another engine it writes nothing
but still cleans up.
+ */
+class UnitTestListenersEngineTest {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void validationLeavesAnotherEngineAlone() throws Exception {
+ PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+ IPipelineEngine<PipelineMeta> engine =
+ unitTestEngine(mock(IPipelineEngine.class), pipelineMeta);
+
+ new ValidatePipelineUnitTestExtensionPoint()
+ .callExtensionPoint(LogChannel.GENERAL, new Variables(), engine);
+
+ verify(pipelineMeta, never()).getMetadataProvider();
+ }
+
+ @Test
+ void validationRunsOnTheLocalEngine() throws Exception {
+ PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+ IPipelineEngine<PipelineMeta> engine =
+ unitTestEngine(mock(LocalPipelineEngine.class), pipelineMeta);
+
+ new ValidatePipelineUnitTestExtensionPoint()
+ .callExtensionPoint(LogChannel.GENERAL, new Variables(), engine);
+
+ // No metadata provider here, so it stops right after looking for one
+ verify(pipelineMeta).getMetadataProvider();
+ }
+
+ /** Spark, Beam ...: no transforms to listen to, but the flag and the maps
are still cleared. */
+ @Test
+ @SuppressWarnings("unchecked")
+ void writingToADataSetOnAnotherEngineOnlyCleansUp() throws Exception {
+ PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+ IPipelineEngine<PipelineMeta> engine =
+ writeToDataSetEngine(mock(IPipelineEngine.class), pipelineMeta);
+ ILogChannel log = mock(ILogChannel.class);
+
+ new WriteToDataSetExtensionPoint().callExtensionPoint(log, new
Variables(), engine);
+
+ verify(engine).addExecutionFinishedListener(any());
+ verify(pipelineMeta, never()).getMetadataProvider();
+ verify(log).logError(contains("not supported"));
+ }
+
+ /** The single threaded engine is a Pipeline, but not a LocalPipelineEngine.
*/
+ @Test
+ void writingToADataSetRunsOnTheSingleThreadedEngine() throws Exception {
+ PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+ IPipelineEngine<PipelineMeta> engine =
writeToDataSetEngine(mock(Pipeline.class), pipelineMeta);
+
+ new WriteToDataSetExtensionPoint()
+ .callExtensionPoint(LogChannel.GENERAL, new Variables(), engine);
+
+ verify(engine).addExecutionFinishedListener(any());
+ verify(pipelineMeta).getMetadataProvider();
+ }
+
+ @Test
+ void writingToADataSetRunsOnTheLocalEngine() throws Exception {
+ PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+ IPipelineEngine<PipelineMeta> engine =
+ writeToDataSetEngine(mock(LocalPipelineEngine.class), pipelineMeta);
+
+ new WriteToDataSetExtensionPoint()
+ .callExtensionPoint(LogChannel.GENERAL, new Variables(), engine);
+
+ verify(engine).addExecutionFinishedListener(any());
+ verify(pipelineMeta).getMetadataProvider();
+ }
+
+ private static IPipelineEngine<PipelineMeta> unitTestEngine(
+ IPipelineEngine<PipelineMeta> engine, PipelineMeta pipelineMeta) {
+ when(engine.getPipelineMeta()).thenReturn(pipelineMeta);
+ when(engine.getVariable(DataSetConst.VAR_RUN_UNIT_TEST)).thenReturn("Y");
+ when(engine.getVariable(DataSetConst.VAR_UNIT_TEST_NAME)).thenReturn("a
unit test");
+ return engine;
+ }
+
+ private static IPipelineEngine<PipelineMeta> writeToDataSetEngine(
+ IPipelineEngine<PipelineMeta> engine, PipelineMeta pipelineMeta) {
+ when(engine.getPipelineMeta()).thenReturn(pipelineMeta);
+
when(engine.getVariable(DataSetConst.VAR_WRITE_TO_DATASET)).thenReturn("Y");
+ return engine;
+ }
+}