This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 7e8ef281e2 XML output and Set files in result clone the row metadata
for every row, fixes #9335 (#9337)
7e8ef281e2 is described below
commit 7e8ef281e2fead553a07acbea163c9efaafc2ee4
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 20:46:24 2026 +0100
XML output and Set files in result clone the row metadata for every row,
fixes #9335 (#9337)
---
.../transforms/filestoresult/FilesToResult.java | 6 +-
.../filestoresult/FilesToResultTest.java | 149 +++++++++++++++++
.../transforms/xml/xmloutput/XmlOutput.java | 10 +-
.../xml/xmloutput/XmlOutputRowMetaTest.java | 185 +++++++++++++++++++++
4 files changed, 345 insertions(+), 5 deletions(-)
diff --git
a/plugins/transforms/filestoresult/src/main/java/org/apache/hop/pipeline/transforms/filestoresult/FilesToResult.java
b/plugins/transforms/filestoresult/src/main/java/org/apache/hop/pipeline/transforms/filestoresult/FilesToResult.java
index 36dce27625..be9494a675 100644
---
a/plugins/transforms/filestoresult/src/main/java/org/apache/hop/pipeline/transforms/filestoresult/FilesToResult.java
+++
b/plugins/transforms/filestoresult/src/main/java/org/apache/hop/pipeline/transforms/filestoresult/FilesToResult.java
@@ -72,6 +72,10 @@ public class FilesToResult extends
BaseTransform<FilesToResultMeta, FilesToResul
stopAll();
return false;
}
+
+ // Built once: the input row metadata doesn't change during the stream.
+ data.outputRowMeta = getInputRowMeta().clone();
+ meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
}
// OK, get the filename field from the row
@@ -94,8 +98,6 @@ public class FilesToResult extends
BaseTransform<FilesToResultMeta, FilesToResul
}
// Copy to any possible next transforms...
- data.outputRowMeta = getInputRowMeta().clone();
- meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
putRow(data.outputRowMeta, r); // copy row to possible alternate
// rowset(s).
diff --git
a/plugins/transforms/filestoresult/src/test/java/org/apache/hop/pipeline/transforms/filestoresult/FilesToResultTest.java
b/plugins/transforms/filestoresult/src/test/java/org/apache/hop/pipeline/transforms/filestoresult/FilesToResultTest.java
new file mode 100644
index 0000000000..09842e4314
--- /dev/null
+++
b/plugins/transforms/filestoresult/src/test/java/org/apache/hop/pipeline/transforms/filestoresult/FilesToResultTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.filestoresult;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.ResultFile;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/** Set files in result in a real pipeline: Injector, then Set files in
result, then Dummy. */
+class FilesToResultTest {
+
+ private static final Duration TIMEOUT = Duration.ofSeconds(60);
+ private static final String INPUT = "input";
+ private static final String FILES_TO_RESULT = "files to result";
+
+ @TempDir Path tempDir;
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ PluginRegistry registry = PluginRegistry.getInstance();
+ for (ITransformMeta meta :
+ List.of(new FilesToResultMeta(), new InjectorMeta(), new DummyMeta()))
{
+ if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+ registry.registerPluginClass(
+ meta.getClass().getName(), TransformPluginType.class,
Transform.class);
+ }
+ }
+ }
+
+ @Test
+ void rowsArePassedOnWithOneRowMetaAndEveryFileReachesTheResult() throws
Exception {
+ List<String> filenames = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ Path file = Files.createFile(tempDir.resolve("file-" + i + ".txt"));
+ filenames.add(file.toUri().toString());
+ }
+
+ List<IRowMeta> passedOn = new ArrayList<>();
+ Pipeline pipeline = assertTimeoutPreemptively(TIMEOUT, () ->
run(filenames, passedOn));
+
+ assertEquals(5, passedOn.size());
+ for (IRowMeta rowMeta : passedOn) {
+ assertSame(passedOn.get(0), rowMeta);
+ }
+ assertEquals(List.of("filename", "other"),
List.of(passedOn.get(0).getFieldNames()));
+
+ Set<String> resultFiles = new TreeSet<>();
+ for (ResultFile resultFile : pipeline.getResult().getResultFilesList()) {
+ resultFiles.add(resultFile.getFile().getName().getBaseName());
+ }
+ assertEquals(
+ Set.of("file-0.txt", "file-1.txt", "file-2.txt", "file-3.txt",
"file-4.txt"), resultFiles);
+ }
+
+ private static Pipeline run(List<String> filenames, List<IRowMeta> passedOn)
throws Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("files-to-result");
+
+ FilesToResultMeta filesToResultMeta = new FilesToResultMeta();
+ filesToResultMeta.setDefault();
+ filesToResultMeta.setFilenameField("filename");
+
+ TransformMeta input = addTransform(pipelineMeta, INPUT, new
InjectorMeta());
+ TransformMeta filesToResult = addTransform(pipelineMeta, FILES_TO_RESULT,
filesToResultMeta);
+ TransformMeta output = addTransform(pipelineMeta, "output", new
DummyMeta());
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(input, filesToResult));
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(filesToResult, output));
+
+ Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+ pipeline.prepareExecution();
+ pipeline
+ .getTransform(FILES_TO_RESULT, 0)
+ .addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
+ throws HopTransformException {
+ passedOn.add(rowMeta);
+ }
+ });
+ RowProducer producer = pipeline.addRowProducer(INPUT, 0);
+ pipeline.startThreads();
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("filename"));
+ rowMeta.addValueMeta(new ValueMetaString("other"));
+ for (String filename : filenames) {
+ producer.putRow(rowMeta, new Object[] {filename, "x"});
+ }
+ producer.finished();
+ pipeline.waitUntilFinished();
+ assertEquals(0, pipeline.getErrors());
+ return pipeline;
+ }
+
+ private static TransformMeta addTransform(
+ PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+ String pluginId =
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+ TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+ pipelineMeta.addTransform(transformMeta);
+ return transformMeta;
+ }
+}
diff --git
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmloutput/XmlOutput.java
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmloutput/XmlOutput.java
index 0a04fda7f0..5d2dfbef89 100644
---
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmloutput/XmlOutput.java
+++
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmloutput/XmlOutput.java
@@ -112,8 +112,12 @@ public class XmlOutput extends
BaseTransform<XmlOutputMeta, XmlOutputData> {
writeRowToFile(getInputRowMeta(), r);
- data.outputRowMeta = getInputRowMeta().clone();
- meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+ // Built once: the input row metadata doesn't change during the stream.
The first flag can't
+ // tell, writeRowToFile() has already reset it.
+ if (data.outputRowMeta == null) {
+ data.outputRowMeta = getInputRowMeta().clone();
+ meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+ }
putRow(data.outputRowMeta, r); // in case we want it to go further...
if (checkFeedback(getLinesOutput()) && isBasic()) {
@@ -200,7 +204,7 @@ public class XmlOutput extends BaseTransform<XmlOutputMeta,
XmlOutputData> {
elementName = outputField.getFieldName();
}
- if (!(valueMeta.isNull(valueData) &&
meta.getFileDetails().isOmitNullValues())) {
+ if (!(meta.getFileDetails().isOmitNullValues() &&
valueMeta.isNull(valueData))) {
writeField(valueMeta, valueData, elementName);
}
}
diff --git
a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmloutput/XmlOutputRowMetaTest.java
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmloutput/XmlOutputRowMetaTest.java
new file mode 100644
index 0000000000..af205b4218
--- /dev/null
+++
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmloutput/XmlOutputRowMetaTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.xml.xmloutput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.apache.hop.pipeline.transforms.xml.xmloutput.XmlField.ContentType;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/** XML output in a real pipeline: Injector, then XML output, then Dummy. */
+class XmlOutputRowMetaTest {
+
+ private static final Duration TIMEOUT = Duration.ofSeconds(60);
+ private static final String INPUT = "input";
+ private static final String XML_OUTPUT = "xml output";
+
+ @TempDir Path tempDir;
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ PluginRegistry registry = PluginRegistry.getInstance();
+ for (ITransformMeta meta : List.of(new XmlOutputMeta(), new
InjectorMeta(), new DummyMeta())) {
+ if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+ registry.registerPluginClass(
+ meta.getClass().getName(), TransformPluginType.class,
Transform.class);
+ }
+ }
+ }
+
+ /** The rows are passed on with one metadata object, which still has the
field lengths set. */
+ @Test
+ void rowsArePassedOnWithOneRowMeta() {
+ List<IRowMeta> passedOn =
+ assertTimeoutPreemptively(TIMEOUT, () -> run(false,
rows(5)).passedOn());
+
+ assertEquals(5, passedOn.size());
+ for (IRowMeta rowMeta : passedOn) {
+ assertSame(passedOn.get(0), rowMeta);
+ }
+ IValueMeta qty = passedOn.get(0).searchValueMeta("qty");
+ assertEquals(7, qty.getLength());
+ assertEquals(0, qty.getPrecision());
+ }
+
+ @Test
+ void omitNullValuesLeavesOutNullElements() throws Exception {
+ String xml = assertTimeoutPreemptively(TIMEOUT, () -> run(true,
rowsWithANullName()).xml());
+
+ assertEquals(1, count(xml, "<name>"), xml);
+ assertEquals(2, count(xml, "<qty>"), xml);
+ }
+
+ @Test
+ void nullElementsAreWrittenWhenNotOmitted() throws Exception {
+ String xml = assertTimeoutPreemptively(TIMEOUT, () -> run(false,
rowsWithANullName()).xml());
+
+ assertEquals(2, count(xml, "<name"), xml);
+ assertEquals(2, count(xml, "<qty>"), xml);
+ }
+
+ private record Run(List<IRowMeta> passedOn, String xml) {}
+
+ private Run run(boolean omitNullValues, List<Object[]> rows) throws
Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("xml-output");
+
+ XmlOutputMeta xmlOutputMeta = new XmlOutputMeta();
+
xmlOutputMeta.getFileDetails().setFileName(tempDir.resolve("out").toString());
+ xmlOutputMeta.getFileDetails().setExtension("xml");
+ xmlOutputMeta.getFileDetails().setOmitNullValues(omitNullValues);
+ List<XmlField> fields = new ArrayList<>();
+ fields.add(new XmlField(ContentType.Element, "name", "name",
IValueMeta.TYPE_STRING, -1, -1));
+ fields.add(new XmlField(ContentType.Element, "qty", "qty",
IValueMeta.TYPE_INTEGER, 7, 0));
+ xmlOutputMeta.setOutputFields(fields);
+
+ TransformMeta input = addTransform(pipelineMeta, INPUT, new
InjectorMeta());
+ TransformMeta xmlOutput = addTransform(pipelineMeta, XML_OUTPUT,
xmlOutputMeta);
+ TransformMeta output = addTransform(pipelineMeta, "output", new
DummyMeta());
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(input, xmlOutput));
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(xmlOutput, output));
+
+ Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+ pipeline.prepareExecution();
+ List<IRowMeta> passedOn = new ArrayList<>();
+ pipeline
+ .getTransform(XML_OUTPUT, 0)
+ .addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
+ throws HopTransformException {
+ passedOn.add(rowMeta);
+ }
+ });
+ RowProducer producer = pipeline.addRowProducer(INPUT, 0);
+ pipeline.startThreads();
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ rowMeta.addValueMeta(new ValueMetaInteger("qty"));
+ for (Object[] row : rows) {
+ producer.putRow(rowMeta, row);
+ }
+ producer.finished();
+ pipeline.waitUntilFinished();
+ assertEquals(0, pipeline.getErrors());
+
+ String xml = Files.readString(tempDir.resolve("out.xml"),
StandardCharsets.UTF_8);
+ return new Run(passedOn, xml);
+ }
+
+ private static List<Object[]> rows(int nrRows) {
+ List<Object[]> rows = new ArrayList<>();
+ for (long i = 0; i < nrRows; i++) {
+ rows.add(new Object[] {"name-" + i, i});
+ }
+ return rows;
+ }
+
+ private static List<Object[]> rowsWithANullName() {
+ return List.of(new Object[] {"a", 1L}, new Object[] {null, 2L});
+ }
+
+ private static int count(String text, String part) {
+ int count = 0;
+ for (int i = text.indexOf(part); i >= 0; i = text.indexOf(part, i +
part.length())) {
+ count++;
+ }
+ return count;
+ }
+
+ private static TransformMeta addTransform(
+ PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+ String pluginId =
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+ TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+ pipelineMeta.addTransform(transformMeta);
+ return transformMeta;
+ }
+}